QATestset - push to hub integration

giskard/rag/dataset_card_template.md

@@ -0,0 +1,31 @@
+---
+tags:
+- giskard
+- QATestset
+- qa-dataset
+- dataset
+---
+
+# Dataset Card for {repo_id}
+This dataset was created using the [giskard](https://github.com/Giskard-AI/giskard) library, an open-source Python framework designed to evaluate and test AI systems. Giskard helps identify performance, bias, and security issues in AI applications, supporting both LLM-based systems like RAG agents and traditional machine learning models for tabular data.
+
+This dataset is a QA (Question/Answer) dataset, containing {num_items} pairs.
+
+## Usage
+
+You can load this dataset using the following code:
+
+```python
+from giskard.rag.testset import QATestset
+test_set = QATestset.load_from_hub("{repo_id}")
+```
+
+Refer to the following tutorial to use it for evaluating your RAG engine: [RAG evaluation tutorial](https://docs.giskard.ai/en/stable/open_source/testset_generation/rag_evaluation/index.html).
+
+## Configuration
+
+The configuration relative to the dataset generation:
+
+```bash
+{config}
+```

giskard/rag/testset.py

@@ -1,15 +1,27 @@
+from pathlib import Path
+import tempfile
 from typing import Any, Dict, Optional, Sequence
 
 import json
 from dataclasses import dataclass
 
 import pandas as pd
 
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+    from huggingface_hub import CommitInfo
+
+
 from ..core.suite import Suite
 from ..datasets.base import Dataset
 from ..testing.tests.llm import test_llm_correctness
 
 
+_HUB_IMPORT_ERROR = ImportError(
+    "`datasets` and `huggingface_hub` are required to push to the Hugging Face Hub. Please install them with `pip install datasets huggingface_hub`"
+)
+
+
 @dataclass
 class QuestionSample:
     id: str
@@ -110,6 +122,90 @@ def load(cls, path):
         dataframe = pd.read_json(path, orient="records", lines=True)
         return cls.from_pandas(dataframe)
 
+    def push_to_hub(
+        self,
+        repo_id: str,
+        token: str | None = None,
+        private: bool = False,
+        **kwargs: Any,
+    ) -> "CommitInfo":
+        """Push the QATestset to the Hugging Face Hub.
+
+        Parameters
+        ----------
+        repo_id : str
+            The repository ID on the Hugging Face Hub.
+        token : str, optional
+            Authentication token for private repositories. Defaults to None.
+        private : bool
+            Whether to create a private repository. Defaults to False.
+        **kwargs : Any
+            Additional arguments passed to Dataset.push_to_hub().
+
+        Returns
+        -------
+        CommitInfo
+            The commit information.
+        """
+
+        try:
+            from datasets import Dataset as HFDataset
+            from huggingface_hub import DatasetCard
+        except ImportError:
+            raise _HUB_IMPORT_ERROR
+
+        # Conversion to Dataset from the datasets library
+        dataset = HFDataset.from_pandas(self._dataframe)
+        dataset.push_to_hub(repo_id, token=token, private=private, **kwargs)
+
+        # Load the dataset card template
+        template_path = Path(__file__).parent / "dataset_card_template.md"
+        template = template_path.read_text()
+
+        # Make and push the dataset card
+        # global _default_llm_model
+        config = {
+            "metadata": {
+                "model": "gpt-4o"
+            }
+        }
+        content = template.format(repo_id=repo_id, num_items=len(self._dataframe), config=json.dumps(config, indent=4))
+        return DatasetCard(content=content).push_to_hub(repo_id=repo_id, token=token, repo_type="dataset")
+
+    @classmethod
+    def load_from_hub(cls, repo_id: str, token: str | None = None, **kwargs: Any) -> "QATestset":
+        """
+        Load an instance of the class from the Hugging Face Hub.
+
+        Parameters
+        ----------
+        repo_id : str
+            The repository ID on the Hugging Face Hub.
+        token : str, optional
+            Authentication token for private repositories. Defaults to None.
+        **kwargs : Any
+            Additional arguments passed to `load_dataset`.
+
+        Returns
+        -------
+        QATestset
+            An instance of the class itself loaded from the Hub.
+
+        Raises
+        ------
+        ImportError
+            If required dependencies are not installed.
+        """
+        try:
+            from datasets import load_dataset
+        except ImportError:
+            raise _HUB_IMPORT_ERROR
+
+        # Load dataset and extract items
+        dataset = load_dataset(repo_id, token=token, split="train", **kwargs)
+        dataframe = pd.DataFrame(dataset)
+        return cls.from_pandas(dataframe)
+
     def to_test_suite(self, name=None, slicing_metadata: Optional[Sequence[str]] = None):
         """
         Convert the testset to a Giskard test suite.