From cc367279ba731a6d5cfa5154f13883e228ad0db0 Mon Sep 17 00:00:00 2001 From: Keerthan Vasist Date: Tue, 10 Oct 2023 18:13:54 -0700 Subject: [PATCH] fix: update assert for eval score to be float or int (instead of just float) --- src/amazon_fmeval/eval_algorithms/__init__.py | 4 ++-- src/amazon_fmeval/eval_algorithms/util.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/amazon_fmeval/eval_algorithms/__init__.py b/src/amazon_fmeval/eval_algorithms/__init__.py index 4c9f18fa..9ccd8435 100644 --- a/src/amazon_fmeval/eval_algorithms/__init__.py +++ b/src/amazon_fmeval/eval_algorithms/__init__.py @@ -5,7 +5,7 @@ from functional import seq -from amazon_fmeval.constants import MIME_TYPE_JSON, MIME_TYPE_JSONLINES +from amazon_fmeval.constants import MIME_TYPE_JSONLINES from amazon_fmeval.data_loaders.data_config import DataConfig @@ -264,7 +264,7 @@ class ModelTask(Enum): WOMENS_CLOTHING_ECOMMERCE_REVIEWS: DataConfig( dataset_name=WOMENS_CLOTHING_ECOMMERCE_REVIEWS, dataset_uri="dummy link", - dataset_mime_type=MIME_TYPE_JSON, + dataset_mime_type=MIME_TYPE_JSONLINES, model_input_location="Review Text", target_output_location="Recommended IND", model_output_location=None, diff --git a/src/amazon_fmeval/eval_algorithms/util.py b/src/amazon_fmeval/eval_algorithms/util.py index 01464aac..c35cca82 100644 --- a/src/amazon_fmeval/eval_algorithms/util.py +++ b/src/amazon_fmeval/eval_algorithms/util.py @@ -181,6 +181,9 @@ class EvalOutputRecord: sent_less_input_prob: Optional[str] = None sent_more_output: Optional[str] = None sent_less_output: Optional[str] = None + prompt: Optional[str] = None + sent_more_prompt: Optional[str] = None + sent_less_prompt: Optional[str] = None def __str__(self): return json.dumps(self._to_dict()) @@ -243,7 +246,7 @@ def from_row(row: Dict[str, Union[str, float]], score_names: List[str]) -> "Eval ) non_score_columns[column_name] = value else: - assert isinstance(value, float) # to satisfy Mypy + assert isinstance(value, float) or isinstance(value, int) # to satisfy Mypy scores.append(EvalScore(name=column_name, value=value)) return EvalOutputRecord(