MIPRO compile ERROR

Question

MIPRO compile ERROR

litetoooooom opened this issue 2 months ago · 1 comments

import time
import dspy
from dspy.predict import Retry
from dspy.datasets import HotPotQA

from dspy.teleprompt import BootstrapFewShotWithRandomSearch
from dsp.utils import EM, normalize_text
from dspy.primitives.assertions import assert_transform_module, backtrack_handler

from tal_llm import OllamaLocal

llm = OllamaLocal(model="qwen:4b-chat-v1.5-fp16", model_type="chat")

dspy.settings.configure(lm=llm, max_tokens=2000)

colbertv2_wiki17_abstracts = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')
dspy.settings.configure(rm=colbertv2_wiki17_abstracts)


import dspy
from dspy.datasets.gsm8k import GSM8K, gsm8k_metric

gsm8k = GSM8K()
gsm8k_trainset, gsm8k_devset = gsm8k.train[:10], gsm8k.dev[:10]

class CoT(dspy.Module):
    def __init__(self):
        super().__init__()
        self.prog = dspy.ChainOfThought("question -> answer")
    
    def forward(self, question):
        return self.prog(question=question)

from dspy.teleprompt import BootstrapFewShot, MIPRO, COPRO

config = dict(num_candidates=4)
teleprompter = MIPRO(metric=gsm8k_metric, **config)

"""
display_progress=False,
display_table=False,
display=True,
max_errors=5,
return_outputs=False,
"""
kwargs = dict(
    num_threads=1, 
    display_progress=True, 
    display_table=True, 
    display=True,
    return_outputs=True)

optimized_cot = teleprompter.compile(
  CoT(), 
  trainset=gsm8k_trainset, 
  num_trials=3, 
  max_bootstrapped_demos=3, 
  max_labeled_demos=5, 
  eval_kwargs=kwargs, 
  requires_permission_to_run=False
  )

Average Metric: 8 [/](https://file+.vscode-resource.vscode-cdn.net/) 10  (80.0): 100%|██████████| 10/10 [01:14<00:00,  7.50s/it]
[W 2024-07-25 15:24:15,363] Trial 0 failed with parameters: {'5344131728_predictor_instruction': 1, '5344131728_predictor_demos': 3} because of the following error: TypeError("unsupported operand type(s) for +=: 'int' and 'tuple'").
Traceback (most recent call last):
  File "/opt/anaconda3/envs/dspy/lib/python3.10/site-packages/optuna/study/_optimize.py", line 196, in _run_trial
    value_or_values = func(trial)
  File "/opt/anaconda3/envs/dspy/lib/python3.10/site-packages/dspy/teleprompt/mipro_optimizer.py", line 554, in objective
    total_score += split_score * len(split_trainset)
TypeError: unsupported operand type(s) for +=: 'int' and 'tuple'
[W 2024-07-25 15:24:15,365] Trial 0 failed with value None.

more detail error information

TypeError                                 Traceback (most recent call last)
Cell In[8], [line 53](vscode-notebook-cell:?execution_count=8&line=53)
     [39](vscode-notebook-cell:?execution_count=8&line=39) """
     [40](vscode-notebook-cell:?execution_count=8&line=40) display_progress=False,
     [41](vscode-notebook-cell:?execution_count=8&line=41) display_table=False,
   (...)
     [44](vscode-notebook-cell:?execution_count=8&line=44) return_outputs=False,
     [45](vscode-notebook-cell:?execution_count=8&line=45) """
     [46](vscode-notebook-cell:?execution_count=8&line=46) kwargs = dict(
     [47](vscode-notebook-cell:?execution_count=8&line=47)     num_threads=1, 
     [48](vscode-notebook-cell:?execution_count=8&line=48)     display_progress=True, 
     [49](vscode-notebook-cell:?execution_count=8&line=49)     display_table=True, 
     [50](vscode-notebook-cell:?execution_count=8&line=50)     display=True,
     [51](vscode-notebook-cell:?execution_count=8&line=51)     return_outputs=True)
---> [53](vscode-notebook-cell:?execution_count=8&line=53) optimized_cot = teleprompter.compile(
     [54](vscode-notebook-cell:?execution_count=8&line=54)   CoT(), 
     [55](vscode-notebook-cell:?execution_count=8&line=55)   trainset=gsm8k_trainset, 
     [56](vscode-notebook-cell:?execution_count=8&line=56)   num_trials=3, 
     [57](vscode-notebook-cell:?execution_count=8&line=57)   max_bootstrapped_demos=3, 
     [58](vscode-notebook-cell:?execution_count=8&line=58)   max_labeled_demos=5, 
     [59](vscode-notebook-cell:?execution_count=8&line=59)   eval_kwargs=kwargs, 
     [60](vscode-notebook-cell:?execution_count=8&line=60)   requires_permission_to_run=False
     [61](vscode-notebook-cell:?execution_count=8&line=61)   )

File [/opt/anaconda3/envs/dspy/lib/python3.10/site-packages/dspy/teleprompt/mipro_optimizer.py:593](https://file+.vscode-resource.vscode-cdn.net/opt/anaconda3/envs/dspy/lib/python3.10/site-packages/dspy/teleprompt/mipro_optimizer.py:593), in MIPRO.compile(self, student, trainset, num_trials, max_bootstrapped_demos, max_labeled_demos, eval_kwargs, seed, view_data, view_examples, requires_permission_to_run)
    [591](https://file+.vscode-resource.vscode-cdn.net/opt/anaconda3/envs/dspy/lib/python3.10/site-packages/dspy/teleprompt/mipro_optimizer.py:591) sampler = optuna.samplers.TPESampler(seed=seed)
    [592](https://file+.vscode-resource.vscode-cdn.net/opt/anaconda3/envs/dspy/lib/python3.10/site-packages/dspy/teleprompt/mipro_optimizer.py:592) study = optuna.create_study(direction="maximize", sampler=sampler)
--> [593](https://file+.vscode-resource.vscode-cdn.net/opt/anaconda3/envs/dspy/lib/python3.10/site-packages/dspy/teleprompt/mipro_optimizer.py:593) _score = study.optimize(objective_function, n_trials=num_trials)
    [595](https://file+.vscode-resource.vscode-cdn.net/opt/anaconda3/envs/dspy/lib/python3.10/site-packages/dspy/teleprompt/mipro_optimizer.py:595) if best_program is not None and self.track_stats:
    [596](https://file+.vscode-resource.vscode-cdn.net/opt/anaconda3/envs/dspy/lib/python3.10/site-packages/dspy/teleprompt/mipro_optimizer.py:596)     best_program.trial_logs = trial_logs

File [/opt/anaconda3/envs/dspy/lib/python3.10/site-packages/optuna/study/study.py:451](https://file+.vscode-resource.vscode-cdn.net/opt/anaconda3/envs/dspy/lib/python3.10/site-packages/optuna/study/study.py:451), in Study.optimize(self, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)
    [348](https://file+.vscode-resource.vscode-cdn.net/opt/anaconda3/envs/dspy/lib/python3.10/site-packages/optuna/study/study.py:348) def optimize(
    [349](https://file+.vscode-resource.vscode-cdn.net/opt/anaconda3/envs/dspy/lib/python3.10/site-packages/optuna/study/study.py:349)     self,
    [350](https://file+.vscode-resource.vscode-cdn.net/opt/anaconda3/envs/dspy/lib/python3.10/site-packages/optuna/study/study.py:350)     func: ObjectiveFuncType,
   (...)
    [357](https://file+.vscode-resource.vscode-cdn.net/opt/anaconda3/envs/dspy/lib/python3.10/site-packages/optuna/study/study.py:357)     show_progress_bar: bool = False,
    [358](https://file+.vscode-resource.vscode-cdn.net/opt/anaconda3/envs/dspy/lib/python3.10/site-packages/optuna/study/study.py:358) ) -> None:
    [359](https://file+.vscode-resource.vscode-cdn.net/opt/anaconda3/envs/dspy/lib/python3.10/site-packages/optuna/study/study.py:359)     """Optimize an objective function.
    [360](https://file+.vscode-resource.vscode-cdn.net/opt/anaconda3/envs/dspy/lib/python3.10/site-packages/optuna/study/study.py:360) 
    [361](https://file+.vscode-resource.vscode-cdn.net/opt/anaconda3/envs/dspy/lib/python3.10/site-packages/optuna/study/study.py:361)     Optimization is done by choosing a suitable set of hyperparameter values from a given
   (...)
    [449](https://file+.vscode-resource.vscode-cdn.net/opt/anaconda3/envs/dspy/lib/python3.10/site-packages/optuna/study/study.py:449)             If nested invocation of this method occurs.
    [450](https://file+.vscode-resource.vscode-cdn.net/opt/anaconda3/envs/dspy/lib/python3.10/site-packages/optuna/study/study.py:450)     """
...
--> [554](https://file+.vscode-resource.vscode-cdn.net/opt/anaconda3/envs/dspy/lib/python3.10/site-packages/dspy/teleprompt/mipro_optimizer.py:554) total_score += split_score * len(split_trainset)
    [555](https://file+.vscode-resource.vscode-cdn.net/opt/anaconda3/envs/dspy/lib/python3.10/site-packages/dspy/teleprompt/mipro_optimizer.py:555) curr_weighted_avg_score = total_score [/](https://file+.vscode-resource.vscode-cdn.net/) min((i + 1) * 100, len(trainset))
    [556](https://file+.vscode-resource.vscode-cdn.net/opt/anaconda3/envs/dspy/lib/python3.10/site-packages/dspy/teleprompt/mipro_optimizer.py:556) if self.verbose:

TypeError: unsupported operand type(s) for +=: 'int' and 'tuple'

litetoooooom commented 2 months ago

M