Fix DPO dataset builder for helpsteer3 and ultrafeedback (#46)

Author: janos-perczel

tinker_cookbook/recipes/preference/datasets.py

@@ -174,7 +174,7 @@ def example_to_labeled_comparison(self, example: dict) -> LabeledComparison | No
             completion_B=[{"role": "assistant", "content": response2}],
         )
         return LabeledComparison(
-            comparison=comparison, label="A" if overall_preference > 0 else "B"
+            comparison=comparison, label="A" if overall_preference < 0 else "B"
         )
 
 

tinker_cookbook/recipes/preference/dpo/train.py

@@ -10,9 +10,6 @@
 from tinker_cookbook.preference.dpo_datasets import (
     DPODatasetBuilderFromComparisons,
 )
-from tinker_cookbook.preference.preference_datasets import (
-    ChatDatasetBuilderFromComparisons,
-)
 from tinker_cookbook.recipes.preference.datasets import (
     HelpSteer3ComparisonBuilder,
     HHHComparisonBuilder,
@@ -69,11 +66,11 @@ def get_dataset_builder(
             common_config=common_config, comparison_builder=HHHComparisonBuilder()
         )
     elif dataset == "helpsteer3":
-        return ChatDatasetBuilderFromComparisons(
+        return DPODatasetBuilderFromComparisons(
             common_config=common_config, comparison_builder=HelpSteer3ComparisonBuilder()
         )
     elif dataset == "ultrafeedback":
-        return ChatDatasetBuilderFromComparisons(
+        return DPODatasetBuilderFromComparisons(
             common_config=common_config, comparison_builder=UltraFeedbackComparisonBuilder()
         )
     else: