Metrics

ragbits.evaluate.metrics.base.MetricSet

MetricSet(*metrics: Metric[EvaluationResultT])

Bases: WithConstructionConfig, Generic[EvaluationResultT]

Represents a set of metrics.

Initialize the metric set.

PARAMETER	DESCRIPTION
metrics	The metrics. TYPE: Metric[EvaluationResultT] DEFAULT: ()

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/base.py

def __init__(self, *metrics: Metric[EvaluationResultT]) -> None:
    """
    Initialize the metric set.

    Args:
        metrics: The metrics.
    """
    self.metrics = metrics

configuration_key class-attribute

configuration_key: str = 'metrics'

default_module class-attribute

default_module: ModuleType | None = metrics

metrics instance-attribute

metrics = metrics

subclass_from_config classmethod

subclass_from_config(config: ObjectConstructionConfig) -> Self

Initializes the class with the provided configuration. May return a subclass of the class, if requested by the configuration.

PARAMETER	DESCRIPTION
config	A model containing configuration details for the class. TYPE: ObjectConstructionConfig

RETURNS	DESCRIPTION
Self	An instance of the class initialized with the provided configuration.

RAISES	DESCRIPTION
InvalidConfigError	The class can't be found or is not a subclass of the current class.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def subclass_from_config(cls, config: ObjectConstructionConfig) -> Self:
    """
    Initializes the class with the provided configuration. May return a subclass of the class,
    if requested by the configuration.

    Args:
        config: A model containing configuration details for the class.

    Returns:
        An instance of the class initialized with the provided configuration.

    Raises:
        InvalidConfigError: The class can't be found or is not a subclass of the current class.
    """
    subclass = import_by_path(config.type, cls.default_module)
    if not issubclass(subclass, cls):
        raise InvalidConfigError(f"{subclass} is not a subclass of {cls}")

    return subclass.from_config(config.config)

subclass_from_factory classmethod

subclass_from_factory(factory_path: str) -> Self

Creates the class using the provided factory function. May return a subclass of the class, if requested by the factory. Supports both synchronous and asynchronous factory functions.

PARAMETER	DESCRIPTION
factory_path	A string representing the path to the factory function in the format of "module.submodule:factory_name". TYPE: str

RETURNS	DESCRIPTION
Self	An instance of the class initialized with the provided factory function.

RAISES	DESCRIPTION
InvalidConfigError	The factory can't be found or the object returned is not a subclass of the current class.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def subclass_from_factory(cls, factory_path: str) -> Self:
    """
    Creates the class using the provided factory function. May return a subclass of the class,
    if requested by the factory. Supports both synchronous and asynchronous factory functions.

    Args:
        factory_path: A string representing the path to the factory function
            in the format of "module.submodule:factory_name".

    Returns:
        An instance of the class initialized with the provided factory function.

    Raises:
        InvalidConfigError: The factory can't be found or the object returned
            is not a subclass of the current class.
    """
    factory = import_by_path(factory_path, cls.default_module)

    if asyncio.iscoroutinefunction(factory):
        try:
            loop = asyncio.get_running_loop()
            obj = asyncio.run_coroutine_threadsafe(factory, loop).result()
        except RuntimeError:
            obj = asyncio.run(factory())
    else:
        obj = factory()

    if not isinstance(obj, cls):
        raise InvalidConfigError(f"The object returned by factory {factory_path} is not an instance of {cls}")

    return obj

preferred_subclass classmethod

preferred_subclass(config: CoreConfig, factory_path_override: str | None = None, yaml_path_override: Path | None = None) -> Self

Tries to create an instance by looking at project's component preferences, either from YAML or from the factory. Takes optional overrides for both, which takes a higher precedence.

PARAMETER	DESCRIPTION
config	The CoreConfig instance containing preferred factory and configuration details. TYPE: CoreConfig
factory_path_override	A string representing the path to the factory function in the format of "module.submodule:factory_name". TYPE: str | None DEFAULT: None
yaml_path_override	A string representing the path to the YAML file containing the Ragstack instance configuration. TYPE: Path | None DEFAULT: None

RAISES	DESCRIPTION
InvalidConfigError	If the default factory or configuration can't be found.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def preferred_subclass(
    cls, config: CoreConfig, factory_path_override: str | None = None, yaml_path_override: Path | None = None
) -> Self:
    """
    Tries to create an instance by looking at project's component preferences, either from YAML
    or from the factory. Takes optional overrides for both, which takes a higher precedence.

    Args:
        config: The CoreConfig instance containing preferred factory and configuration details.
        factory_path_override: A string representing the path to the factory function
            in the format of "module.submodule:factory_name".
        yaml_path_override: A string representing the path to the YAML file containing
            the Ragstack instance configuration.

    Raises:
        InvalidConfigError: If the default factory or configuration can't be found.
    """
    if yaml_path_override:
        preferences = get_config_from_yaml(yaml_path_override)
        if type_config := preferences.get(cls.configuration_key):
            return cls.subclass_from_config(ObjectConstructionConfig.model_validate(type_config))

    if factory_path_override:
        return cls.subclass_from_factory(factory_path_override)

    if preferred_factory := config.component_preference_factories.get(cls.configuration_key):
        return cls.subclass_from_factory(preferred_factory)

    if preferred_config := config.preferred_instances_config.get(cls.configuration_key):
        return cls.subclass_from_config(ObjectConstructionConfig.model_validate(preferred_config))

    raise NoPreferredConfigError(f"Could not find preferred factory or configuration for {cls.configuration_key}")

from_config classmethod

from_config(config: dict) -> Self

Create an instance of MetricSet from a configuration dictionary.

PARAMETER	DESCRIPTION
config	A dictionary containing configuration settings for the metric set. TYPE: dict

RETURNS	DESCRIPTION
Self	An instance of the metric set class initialized with the provided configuration.

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/base.py

@classmethod
def from_config(cls, config: dict) -> Self:
    """
    Create an instance of `MetricSet` from a configuration dictionary.

    Args:
        config: A dictionary containing configuration settings for the metric set.

    Returns:
        An instance of the metric set class initialized with the provided configuration.
    """
    return cls(*[Metric.subclass_from_config(metric_config) for metric_config in config.values()])

compute async

compute(results: list[EvaluationResultT]) -> dict

Compute the metrics.

PARAMETER	DESCRIPTION
results	The evaluation results. TYPE: list[EvaluationResultT]

RETURNS	DESCRIPTION
dict	The computed metrics.

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/base.py

async def compute(self, results: list[EvaluationResultT]) -> dict:
    """
    Compute the metrics.

    Args:
        results: The evaluation results.

    Returns:
        The computed metrics.
    """
    metric_results = await asyncio.gather(*[metric.compute(results) for metric in self.metrics])
    return {
        name: metric.weight * value
        for metric, result in zip(self.metrics, metric_results, strict=False)
        for name, value in result.items()
    }

ragbits.evaluate.metrics.base.Metric

Metric(weight: float = 1.0)

Bases: WithConstructionConfig, Generic[EvaluationResultT], ABC

Base class for metrics.

Initialize the metric.

PARAMETER	DESCRIPTION
weight	Metric value weight in the final score, used during optimization. TYPE: float DEFAULT: 1.0

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/base.py

def __init__(self, weight: float = 1.0) -> None:
    """
    Initialize the metric.

    Args:
        weight: Metric value weight in the final score, used during optimization.
    """
    super().__init__()
    self.weight = weight

default_module class-attribute

default_module: ModuleType | None = metrics

configuration_key class-attribute

configuration_key: str = 'metric'

weight instance-attribute

weight = weight

subclass_from_config classmethod

subclass_from_config(config: ObjectConstructionConfig) -> Self

Initializes the class with the provided configuration. May return a subclass of the class, if requested by the configuration.

PARAMETER	DESCRIPTION
config	A model containing configuration details for the class. TYPE: ObjectConstructionConfig

RETURNS	DESCRIPTION
Self	An instance of the class initialized with the provided configuration.

RAISES	DESCRIPTION
InvalidConfigError	The class can't be found or is not a subclass of the current class.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def subclass_from_config(cls, config: ObjectConstructionConfig) -> Self:
    """
    Initializes the class with the provided configuration. May return a subclass of the class,
    if requested by the configuration.

    Args:
        config: A model containing configuration details for the class.

    Returns:
        An instance of the class initialized with the provided configuration.

    Raises:
        InvalidConfigError: The class can't be found or is not a subclass of the current class.
    """
    subclass = import_by_path(config.type, cls.default_module)
    if not issubclass(subclass, cls):
        raise InvalidConfigError(f"{subclass} is not a subclass of {cls}")

    return subclass.from_config(config.config)

subclass_from_factory classmethod

subclass_from_factory(factory_path: str) -> Self

Creates the class using the provided factory function. May return a subclass of the class, if requested by the factory. Supports both synchronous and asynchronous factory functions.

PARAMETER	DESCRIPTION
factory_path	A string representing the path to the factory function in the format of "module.submodule:factory_name". TYPE: str

RETURNS	DESCRIPTION
Self	An instance of the class initialized with the provided factory function.

RAISES	DESCRIPTION
InvalidConfigError	The factory can't be found or the object returned is not a subclass of the current class.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def subclass_from_factory(cls, factory_path: str) -> Self:
    """
    Creates the class using the provided factory function. May return a subclass of the class,
    if requested by the factory. Supports both synchronous and asynchronous factory functions.

    Args:
        factory_path: A string representing the path to the factory function
            in the format of "module.submodule:factory_name".

    Returns:
        An instance of the class initialized with the provided factory function.

    Raises:
        InvalidConfigError: The factory can't be found or the object returned
            is not a subclass of the current class.
    """
    factory = import_by_path(factory_path, cls.default_module)

    if asyncio.iscoroutinefunction(factory):
        try:
            loop = asyncio.get_running_loop()
            obj = asyncio.run_coroutine_threadsafe(factory, loop).result()
        except RuntimeError:
            obj = asyncio.run(factory())
    else:
        obj = factory()

    if not isinstance(obj, cls):
        raise InvalidConfigError(f"The object returned by factory {factory_path} is not an instance of {cls}")

    return obj

preferred_subclass classmethod

preferred_subclass(config: CoreConfig, factory_path_override: str | None = None, yaml_path_override: Path | None = None) -> Self

Tries to create an instance by looking at project's component preferences, either from YAML or from the factory. Takes optional overrides for both, which takes a higher precedence.

PARAMETER	DESCRIPTION
config	The CoreConfig instance containing preferred factory and configuration details. TYPE: CoreConfig
factory_path_override	A string representing the path to the factory function in the format of "module.submodule:factory_name". TYPE: str | None DEFAULT: None
yaml_path_override	A string representing the path to the YAML file containing the Ragstack instance configuration. TYPE: Path | None DEFAULT: None

RAISES	DESCRIPTION
InvalidConfigError	If the default factory or configuration can't be found.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def preferred_subclass(
    cls, config: CoreConfig, factory_path_override: str | None = None, yaml_path_override: Path | None = None
) -> Self:
    """
    Tries to create an instance by looking at project's component preferences, either from YAML
    or from the factory. Takes optional overrides for both, which takes a higher precedence.

    Args:
        config: The CoreConfig instance containing preferred factory and configuration details.
        factory_path_override: A string representing the path to the factory function
            in the format of "module.submodule:factory_name".
        yaml_path_override: A string representing the path to the YAML file containing
            the Ragstack instance configuration.

    Raises:
        InvalidConfigError: If the default factory or configuration can't be found.
    """
    if yaml_path_override:
        preferences = get_config_from_yaml(yaml_path_override)
        if type_config := preferences.get(cls.configuration_key):
            return cls.subclass_from_config(ObjectConstructionConfig.model_validate(type_config))

    if factory_path_override:
        return cls.subclass_from_factory(factory_path_override)

    if preferred_factory := config.component_preference_factories.get(cls.configuration_key):
        return cls.subclass_from_factory(preferred_factory)

    if preferred_config := config.preferred_instances_config.get(cls.configuration_key):
        return cls.subclass_from_config(ObjectConstructionConfig.model_validate(preferred_config))

    raise NoPreferredConfigError(f"Could not find preferred factory or configuration for {cls.configuration_key}")

from_config classmethod

from_config(config: dict) -> Self

Initializes the class with the provided configuration.

PARAMETER	DESCRIPTION
config	A dictionary containing configuration details for the class. TYPE: dict

RETURNS	DESCRIPTION
Self	An instance of the class initialized with the provided configuration.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def from_config(cls, config: dict) -> Self:
    """
    Initializes the class with the provided configuration.

    Args:
        config: A dictionary containing configuration details for the class.

    Returns:
        An instance of the class initialized with the provided configuration.
    """
    return cls(**config)

compute abstractmethod async

compute(results: list[EvaluationResultT]) -> dict

Compute the metric.

PARAMETER	DESCRIPTION
results	The evaluation results. TYPE: list[EvaluationResultT]

RETURNS	DESCRIPTION
dict	The computed metric.

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/base.py

@abstractmethod
async def compute(self, results: list[EvaluationResultT]) -> dict:
    """
    Compute the metric.

    Args:
        results: The evaluation results.

    Returns:
        The computed metric.
    """

ragbits.evaluate.metrics.document_search.DocumentSearchMetric

DocumentSearchMetric(matching_strategy: MatchingStrategy, weight: float = 1.0)

Bases: Metric[DocumentSearchResult], ABC

Metric for document search evaluation based on Relari backend. More details can be found here.

Initialize the document search metric.

PARAMETER	DESCRIPTION
matching_strategy	Matching strategys that determine relevance. TYPE: MatchingStrategy
weight	Metric value weight in the final score, used during optimization. TYPE: float DEFAULT: 1.0

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/document_search.py

def __init__(self, matching_strategy: MatchingStrategy, weight: float = 1.0) -> None:
    """
    Initialize the document search metric.

    Args:
        matching_strategy: Matching strategys that determine relevance.
        weight: Metric value weight in the final score, used during optimization.
    """
    super().__init__(weight=weight)
    self.metric = self.metric_cls(matching_strategy)

default_module class-attribute

default_module: ModuleType | None = metrics

configuration_key class-attribute

configuration_key: str = 'metric'

weight instance-attribute

weight = weight

metric_cls instance-attribute

metric_cls: type[PrecisionRecallF1 | RankedRetrievalMetrics]

metric instance-attribute

metric = metric_cls(matching_strategy)

subclass_from_config classmethod

subclass_from_config(config: ObjectConstructionConfig) -> Self

Initializes the class with the provided configuration. May return a subclass of the class, if requested by the configuration.

PARAMETER	DESCRIPTION
config	A model containing configuration details for the class. TYPE: ObjectConstructionConfig

RETURNS	DESCRIPTION
Self	An instance of the class initialized with the provided configuration.

RAISES	DESCRIPTION
InvalidConfigError	The class can't be found or is not a subclass of the current class.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def subclass_from_config(cls, config: ObjectConstructionConfig) -> Self:
    """
    Initializes the class with the provided configuration. May return a subclass of the class,
    if requested by the configuration.

    Args:
        config: A model containing configuration details for the class.

    Returns:
        An instance of the class initialized with the provided configuration.

    Raises:
        InvalidConfigError: The class can't be found or is not a subclass of the current class.
    """
    subclass = import_by_path(config.type, cls.default_module)
    if not issubclass(subclass, cls):
        raise InvalidConfigError(f"{subclass} is not a subclass of {cls}")

    return subclass.from_config(config.config)

subclass_from_factory classmethod

subclass_from_factory(factory_path: str) -> Self

Creates the class using the provided factory function. May return a subclass of the class, if requested by the factory. Supports both synchronous and asynchronous factory functions.

PARAMETER	DESCRIPTION
factory_path	A string representing the path to the factory function in the format of "module.submodule:factory_name". TYPE: str

RETURNS	DESCRIPTION
Self	An instance of the class initialized with the provided factory function.

RAISES	DESCRIPTION
InvalidConfigError	The factory can't be found or the object returned is not a subclass of the current class.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def subclass_from_factory(cls, factory_path: str) -> Self:
    """
    Creates the class using the provided factory function. May return a subclass of the class,
    if requested by the factory. Supports both synchronous and asynchronous factory functions.

    Args:
        factory_path: A string representing the path to the factory function
            in the format of "module.submodule:factory_name".

    Returns:
        An instance of the class initialized with the provided factory function.

    Raises:
        InvalidConfigError: The factory can't be found or the object returned
            is not a subclass of the current class.
    """
    factory = import_by_path(factory_path, cls.default_module)

    if asyncio.iscoroutinefunction(factory):
        try:
            loop = asyncio.get_running_loop()
            obj = asyncio.run_coroutine_threadsafe(factory, loop).result()
        except RuntimeError:
            obj = asyncio.run(factory())
    else:
        obj = factory()

    if not isinstance(obj, cls):
        raise InvalidConfigError(f"The object returned by factory {factory_path} is not an instance of {cls}")

    return obj

preferred_subclass classmethod

preferred_subclass(config: CoreConfig, factory_path_override: str | None = None, yaml_path_override: Path | None = None) -> Self

Tries to create an instance by looking at project's component preferences, either from YAML or from the factory. Takes optional overrides for both, which takes a higher precedence.

PARAMETER	DESCRIPTION
config	The CoreConfig instance containing preferred factory and configuration details. TYPE: CoreConfig
factory_path_override	A string representing the path to the factory function in the format of "module.submodule:factory_name". TYPE: str | None DEFAULT: None
yaml_path_override	A string representing the path to the YAML file containing the Ragstack instance configuration. TYPE: Path | None DEFAULT: None

RAISES	DESCRIPTION
InvalidConfigError	If the default factory or configuration can't be found.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def preferred_subclass(
    cls, config: CoreConfig, factory_path_override: str | None = None, yaml_path_override: Path | None = None
) -> Self:
    """
    Tries to create an instance by looking at project's component preferences, either from YAML
    or from the factory. Takes optional overrides for both, which takes a higher precedence.

    Args:
        config: The CoreConfig instance containing preferred factory and configuration details.
        factory_path_override: A string representing the path to the factory function
            in the format of "module.submodule:factory_name".
        yaml_path_override: A string representing the path to the YAML file containing
            the Ragstack instance configuration.

    Raises:
        InvalidConfigError: If the default factory or configuration can't be found.
    """
    if yaml_path_override:
        preferences = get_config_from_yaml(yaml_path_override)
        if type_config := preferences.get(cls.configuration_key):
            return cls.subclass_from_config(ObjectConstructionConfig.model_validate(type_config))

    if factory_path_override:
        return cls.subclass_from_factory(factory_path_override)

    if preferred_factory := config.component_preference_factories.get(cls.configuration_key):
        return cls.subclass_from_factory(preferred_factory)

    if preferred_config := config.preferred_instances_config.get(cls.configuration_key):
        return cls.subclass_from_config(ObjectConstructionConfig.model_validate(preferred_config))

    raise NoPreferredConfigError(f"Could not find preferred factory or configuration for {cls.configuration_key}")

from_config classmethod

from_config(config: dict) -> Self

Create an instance of DocumentSearchMetric from a configuration dictionary.

PARAMETER	DESCRIPTION
config	A dictionary containing configuration settings for the metric. TYPE: dict

RETURNS	DESCRIPTION
Self	An instance of the metric class initialized with the provided configuration.

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/document_search.py

@classmethod
def from_config(cls, config: dict) -> Self:
    """
    Create an instance of `DocumentSearchMetric` from a configuration dictionary.

    Args:
        config: A dictionary containing configuration settings for the metric.

    Returns:
        An instance of the metric class initialized with the provided configuration.
    """
    matching_strategy_cls = getattr(
        importlib.import_module("continuous_eval.metrics.retrieval.matching_strategy"),
        config["matching_strategy"]["type"],
    )
    matching_strategy = matching_strategy_cls(**config["matching_strategy"]["config"])
    return cls(matching_strategy=matching_strategy, weight=config.get("weight", 1.0))

compute async

compute(results: list[DocumentSearchResult]) -> dict

Compute the metric.

PARAMETER	DESCRIPTION
results	The evaluation results. TYPE: list[DocumentSearchResult]

RETURNS	DESCRIPTION
dict	The computed metric.

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/document_search.py

async def compute(self, results: list[DocumentSearchResult]) -> dict:
    """
    Compute the metric.

    Args:
        results: The evaluation results.

    Returns:
        The computed metric.
    """
    return self.metric.aggregate(
        [
            self.metric(
                [
                    element.text_representation
                    for element in result.predicted_elements
                    if element.text_representation
                ],
                result.reference_passages,
            )
            for result in results
            if result.reference_passages is not None
        ]
    )

ragbits.evaluate.metrics.document_search.DocumentSearchPrecisionRecallF1

DocumentSearchPrecisionRecallF1(matching_strategy: MatchingStrategy, weight: float = 1.0)

Bases: DocumentSearchMetric

Precision, recall, and F1 score for context retrieval. More details can be found here.

Initialize the document search metric.

PARAMETER	DESCRIPTION
matching_strategy	Matching strategys that determine relevance. TYPE: MatchingStrategy
weight	Metric value weight in the final score, used during optimization. TYPE: float DEFAULT: 1.0

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/document_search.py

def __init__(self, matching_strategy: MatchingStrategy, weight: float = 1.0) -> None:
    """
    Initialize the document search metric.

    Args:
        matching_strategy: Matching strategys that determine relevance.
        weight: Metric value weight in the final score, used during optimization.
    """
    super().__init__(weight=weight)
    self.metric = self.metric_cls(matching_strategy)

default_module class-attribute

default_module: ModuleType | None = metrics

configuration_key class-attribute

configuration_key: str = 'metric'

weight instance-attribute

weight = weight

metric instance-attribute

metric = metric_cls(matching_strategy)

metric_cls class-attribute instance-attribute

metric_cls = PrecisionRecallF1

subclass_from_config classmethod

subclass_from_config(config: ObjectConstructionConfig) -> Self

Initializes the class with the provided configuration. May return a subclass of the class, if requested by the configuration.

PARAMETER	DESCRIPTION
config	A model containing configuration details for the class. TYPE: ObjectConstructionConfig

RETURNS	DESCRIPTION
Self	An instance of the class initialized with the provided configuration.

RAISES	DESCRIPTION
InvalidConfigError	The class can't be found or is not a subclass of the current class.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def subclass_from_config(cls, config: ObjectConstructionConfig) -> Self:
    """
    Initializes the class with the provided configuration. May return a subclass of the class,
    if requested by the configuration.

    Args:
        config: A model containing configuration details for the class.

    Returns:
        An instance of the class initialized with the provided configuration.

    Raises:
        InvalidConfigError: The class can't be found or is not a subclass of the current class.
    """
    subclass = import_by_path(config.type, cls.default_module)
    if not issubclass(subclass, cls):
        raise InvalidConfigError(f"{subclass} is not a subclass of {cls}")

    return subclass.from_config(config.config)

subclass_from_factory classmethod

subclass_from_factory(factory_path: str) -> Self

Creates the class using the provided factory function. May return a subclass of the class, if requested by the factory. Supports both synchronous and asynchronous factory functions.

PARAMETER	DESCRIPTION
factory_path	A string representing the path to the factory function in the format of "module.submodule:factory_name". TYPE: str

RETURNS	DESCRIPTION
Self	An instance of the class initialized with the provided factory function.

RAISES	DESCRIPTION
InvalidConfigError	The factory can't be found or the object returned is not a subclass of the current class.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def subclass_from_factory(cls, factory_path: str) -> Self:
    """
    Creates the class using the provided factory function. May return a subclass of the class,
    if requested by the factory. Supports both synchronous and asynchronous factory functions.

    Args:
        factory_path: A string representing the path to the factory function
            in the format of "module.submodule:factory_name".

    Returns:
        An instance of the class initialized with the provided factory function.

    Raises:
        InvalidConfigError: The factory can't be found or the object returned
            is not a subclass of the current class.
    """
    factory = import_by_path(factory_path, cls.default_module)

    if asyncio.iscoroutinefunction(factory):
        try:
            loop = asyncio.get_running_loop()
            obj = asyncio.run_coroutine_threadsafe(factory, loop).result()
        except RuntimeError:
            obj = asyncio.run(factory())
    else:
        obj = factory()

    if not isinstance(obj, cls):
        raise InvalidConfigError(f"The object returned by factory {factory_path} is not an instance of {cls}")

    return obj

preferred_subclass classmethod

preferred_subclass(config: CoreConfig, factory_path_override: str | None = None, yaml_path_override: Path | None = None) -> Self

Tries to create an instance by looking at project's component preferences, either from YAML or from the factory. Takes optional overrides for both, which takes a higher precedence.

PARAMETER	DESCRIPTION
config	The CoreConfig instance containing preferred factory and configuration details. TYPE: CoreConfig
factory_path_override	A string representing the path to the factory function in the format of "module.submodule:factory_name". TYPE: str | None DEFAULT: None
yaml_path_override	A string representing the path to the YAML file containing the Ragstack instance configuration. TYPE: Path | None DEFAULT: None

RAISES	DESCRIPTION
InvalidConfigError	If the default factory or configuration can't be found.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def preferred_subclass(
    cls, config: CoreConfig, factory_path_override: str | None = None, yaml_path_override: Path | None = None
) -> Self:
    """
    Tries to create an instance by looking at project's component preferences, either from YAML
    or from the factory. Takes optional overrides for both, which takes a higher precedence.

    Args:
        config: The CoreConfig instance containing preferred factory and configuration details.
        factory_path_override: A string representing the path to the factory function
            in the format of "module.submodule:factory_name".
        yaml_path_override: A string representing the path to the YAML file containing
            the Ragstack instance configuration.

    Raises:
        InvalidConfigError: If the default factory or configuration can't be found.
    """
    if yaml_path_override:
        preferences = get_config_from_yaml(yaml_path_override)
        if type_config := preferences.get(cls.configuration_key):
            return cls.subclass_from_config(ObjectConstructionConfig.model_validate(type_config))

    if factory_path_override:
        return cls.subclass_from_factory(factory_path_override)

    if preferred_factory := config.component_preference_factories.get(cls.configuration_key):
        return cls.subclass_from_factory(preferred_factory)

    if preferred_config := config.preferred_instances_config.get(cls.configuration_key):
        return cls.subclass_from_config(ObjectConstructionConfig.model_validate(preferred_config))

    raise NoPreferredConfigError(f"Could not find preferred factory or configuration for {cls.configuration_key}")

from_config classmethod

from_config(config: dict) -> Self

Create an instance of DocumentSearchMetric from a configuration dictionary.

PARAMETER	DESCRIPTION
config	A dictionary containing configuration settings for the metric. TYPE: dict

RETURNS	DESCRIPTION
Self	An instance of the metric class initialized with the provided configuration.

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/document_search.py

@classmethod
def from_config(cls, config: dict) -> Self:
    """
    Create an instance of `DocumentSearchMetric` from a configuration dictionary.

    Args:
        config: A dictionary containing configuration settings for the metric.

    Returns:
        An instance of the metric class initialized with the provided configuration.
    """
    matching_strategy_cls = getattr(
        importlib.import_module("continuous_eval.metrics.retrieval.matching_strategy"),
        config["matching_strategy"]["type"],
    )
    matching_strategy = matching_strategy_cls(**config["matching_strategy"]["config"])
    return cls(matching_strategy=matching_strategy, weight=config.get("weight", 1.0))

compute async

compute(results: list[DocumentSearchResult]) -> dict

Compute the metric.

PARAMETER	DESCRIPTION
results	The evaluation results. TYPE: list[DocumentSearchResult]

RETURNS	DESCRIPTION
dict	The computed metric.

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/document_search.py

async def compute(self, results: list[DocumentSearchResult]) -> dict:
    """
    Compute the metric.

    Args:
        results: The evaluation results.

    Returns:
        The computed metric.
    """
    return self.metric.aggregate(
        [
            self.metric(
                [
                    element.text_representation
                    for element in result.predicted_elements
                    if element.text_representation
                ],
                result.reference_passages,
            )
            for result in results
            if result.reference_passages is not None
        ]
    )

ragbits.evaluate.metrics.document_search.DocumentSearchRankedRetrievalMetrics

DocumentSearchRankedRetrievalMetrics(matching_strategy: MatchingStrategy, weight: float = 1.0)

Bases: DocumentSearchMetric

Rank-aware metrics takes into account the order in which the contexts are retrieved. More details can be found here.

Initialize the document search metric.

PARAMETER	DESCRIPTION
matching_strategy	Matching strategys that determine relevance. TYPE: MatchingStrategy
weight	Metric value weight in the final score, used during optimization. TYPE: float DEFAULT: 1.0

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/document_search.py

def __init__(self, matching_strategy: MatchingStrategy, weight: float = 1.0) -> None:
    """
    Initialize the document search metric.

    Args:
        matching_strategy: Matching strategys that determine relevance.
        weight: Metric value weight in the final score, used during optimization.
    """
    super().__init__(weight=weight)
    self.metric = self.metric_cls(matching_strategy)

default_module class-attribute

default_module: ModuleType | None = metrics

configuration_key class-attribute

configuration_key: str = 'metric'

weight instance-attribute

weight = weight

metric instance-attribute

metric = metric_cls(matching_strategy)

metric_cls class-attribute instance-attribute

metric_cls = RankedRetrievalMetrics

subclass_from_config classmethod

subclass_from_config(config: ObjectConstructionConfig) -> Self

Initializes the class with the provided configuration. May return a subclass of the class, if requested by the configuration.

PARAMETER	DESCRIPTION
config	A model containing configuration details for the class. TYPE: ObjectConstructionConfig

RETURNS	DESCRIPTION
Self	An instance of the class initialized with the provided configuration.

RAISES	DESCRIPTION
InvalidConfigError	The class can't be found or is not a subclass of the current class.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def subclass_from_config(cls, config: ObjectConstructionConfig) -> Self:
    """
    Initializes the class with the provided configuration. May return a subclass of the class,
    if requested by the configuration.

    Args:
        config: A model containing configuration details for the class.

    Returns:
        An instance of the class initialized with the provided configuration.

    Raises:
        InvalidConfigError: The class can't be found or is not a subclass of the current class.
    """
    subclass = import_by_path(config.type, cls.default_module)
    if not issubclass(subclass, cls):
        raise InvalidConfigError(f"{subclass} is not a subclass of {cls}")

    return subclass.from_config(config.config)

subclass_from_factory classmethod

subclass_from_factory(factory_path: str) -> Self

Creates the class using the provided factory function. May return a subclass of the class, if requested by the factory. Supports both synchronous and asynchronous factory functions.

PARAMETER	DESCRIPTION
factory_path	A string representing the path to the factory function in the format of "module.submodule:factory_name". TYPE: str

RETURNS	DESCRIPTION
Self	An instance of the class initialized with the provided factory function.

RAISES	DESCRIPTION
InvalidConfigError	The factory can't be found or the object returned is not a subclass of the current class.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def subclass_from_factory(cls, factory_path: str) -> Self:
    """
    Creates the class using the provided factory function. May return a subclass of the class,
    if requested by the factory. Supports both synchronous and asynchronous factory functions.

    Args:
        factory_path: A string representing the path to the factory function
            in the format of "module.submodule:factory_name".

    Returns:
        An instance of the class initialized with the provided factory function.

    Raises:
        InvalidConfigError: The factory can't be found or the object returned
            is not a subclass of the current class.
    """
    factory = import_by_path(factory_path, cls.default_module)

    if asyncio.iscoroutinefunction(factory):
        try:
            loop = asyncio.get_running_loop()
            obj = asyncio.run_coroutine_threadsafe(factory, loop).result()
        except RuntimeError:
            obj = asyncio.run(factory())
    else:
        obj = factory()

    if not isinstance(obj, cls):
        raise InvalidConfigError(f"The object returned by factory {factory_path} is not an instance of {cls}")

    return obj

preferred_subclass classmethod

preferred_subclass(config: CoreConfig, factory_path_override: str | None = None, yaml_path_override: Path | None = None) -> Self

Tries to create an instance by looking at project's component preferences, either from YAML or from the factory. Takes optional overrides for both, which takes a higher precedence.

PARAMETER	DESCRIPTION
config	The CoreConfig instance containing preferred factory and configuration details. TYPE: CoreConfig
factory_path_override	A string representing the path to the factory function in the format of "module.submodule:factory_name". TYPE: str | None DEFAULT: None
yaml_path_override	A string representing the path to the YAML file containing the Ragstack instance configuration. TYPE: Path | None DEFAULT: None

RAISES	DESCRIPTION
InvalidConfigError	If the default factory or configuration can't be found.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def preferred_subclass(
    cls, config: CoreConfig, factory_path_override: str | None = None, yaml_path_override: Path | None = None
) -> Self:
    """
    Tries to create an instance by looking at project's component preferences, either from YAML
    or from the factory. Takes optional overrides for both, which takes a higher precedence.

    Args:
        config: The CoreConfig instance containing preferred factory and configuration details.
        factory_path_override: A string representing the path to the factory function
            in the format of "module.submodule:factory_name".
        yaml_path_override: A string representing the path to the YAML file containing
            the Ragstack instance configuration.

    Raises:
        InvalidConfigError: If the default factory or configuration can't be found.
    """
    if yaml_path_override:
        preferences = get_config_from_yaml(yaml_path_override)
        if type_config := preferences.get(cls.configuration_key):
            return cls.subclass_from_config(ObjectConstructionConfig.model_validate(type_config))

    if factory_path_override:
        return cls.subclass_from_factory(factory_path_override)

    if preferred_factory := config.component_preference_factories.get(cls.configuration_key):
        return cls.subclass_from_factory(preferred_factory)

    if preferred_config := config.preferred_instances_config.get(cls.configuration_key):
        return cls.subclass_from_config(ObjectConstructionConfig.model_validate(preferred_config))

    raise NoPreferredConfigError(f"Could not find preferred factory or configuration for {cls.configuration_key}")

from_config classmethod

from_config(config: dict) -> Self

Create an instance of DocumentSearchMetric from a configuration dictionary.

PARAMETER	DESCRIPTION
config	A dictionary containing configuration settings for the metric. TYPE: dict

RETURNS	DESCRIPTION
Self	An instance of the metric class initialized with the provided configuration.

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/document_search.py

@classmethod
def from_config(cls, config: dict) -> Self:
    """
    Create an instance of `DocumentSearchMetric` from a configuration dictionary.

    Args:
        config: A dictionary containing configuration settings for the metric.

    Returns:
        An instance of the metric class initialized with the provided configuration.
    """
    matching_strategy_cls = getattr(
        importlib.import_module("continuous_eval.metrics.retrieval.matching_strategy"),
        config["matching_strategy"]["type"],
    )
    matching_strategy = matching_strategy_cls(**config["matching_strategy"]["config"])
    return cls(matching_strategy=matching_strategy, weight=config.get("weight", 1.0))

compute async

compute(results: list[DocumentSearchResult]) -> dict

Compute the metric.

PARAMETER	DESCRIPTION
results	The evaluation results. TYPE: list[DocumentSearchResult]

RETURNS	DESCRIPTION
dict	The computed metric.

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/document_search.py

async def compute(self, results: list[DocumentSearchResult]) -> dict:
    """
    Compute the metric.

    Args:
        results: The evaluation results.

    Returns:
        The computed metric.
    """
    return self.metric.aggregate(
        [
            self.metric(
                [
                    element.text_representation
                    for element in result.predicted_elements
                    if element.text_representation
                ],
                result.reference_passages,
            )
            for result in results
            if result.reference_passages is not None
        ]
    )

ragbits.evaluate.metrics.question_answer.QuestionAnswerMetric

QuestionAnswerMetric(llm: LLM, batch_size: int = 15, weight: float = 1.0)

Bases: Generic[MetricT], Metric[QuestionAnswerResult], ABC

Metric for question answer evaluation based on Relari backend. More details can be found here.

Initialize the agent metric.

PARAMETER	DESCRIPTION
llm	Judge LLM instance. TYPE: LLM
batch_size	Batch size for metric computation. TYPE: int DEFAULT: 15
weight	Metric value weight in the final score, used during optimization. TYPE: float DEFAULT: 1.0

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/question_answer.py

def __init__(self, llm: LLM, batch_size: int = 15, weight: float = 1.0) -> None:
    """
    Initialize the agent metric.

    Args:
        llm: Judge LLM instance.
        batch_size: Batch size for metric computation.
        weight: Metric value weight in the final score, used during optimization.
    """
    super().__init__(weight=weight)
    self.llm = llm
    self.batch_size = batch_size

default_module class-attribute

default_module: ModuleType | None = metrics

configuration_key class-attribute

configuration_key: str = 'metric'

weight instance-attribute

weight = weight

metric_cls instance-attribute

metric_cls: type[MetricT]

llm instance-attribute

llm = llm

batch_size instance-attribute

batch_size = batch_size

subclass_from_config classmethod

subclass_from_config(config: ObjectConstructionConfig) -> Self

Initializes the class with the provided configuration. May return a subclass of the class, if requested by the configuration.

PARAMETER	DESCRIPTION
config	A model containing configuration details for the class. TYPE: ObjectConstructionConfig

RETURNS	DESCRIPTION
Self	An instance of the class initialized with the provided configuration.

RAISES	DESCRIPTION
InvalidConfigError	The class can't be found or is not a subclass of the current class.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def subclass_from_config(cls, config: ObjectConstructionConfig) -> Self:
    """
    Initializes the class with the provided configuration. May return a subclass of the class,
    if requested by the configuration.

    Args:
        config: A model containing configuration details for the class.

    Returns:
        An instance of the class initialized with the provided configuration.

    Raises:
        InvalidConfigError: The class can't be found or is not a subclass of the current class.
    """
    subclass = import_by_path(config.type, cls.default_module)
    if not issubclass(subclass, cls):
        raise InvalidConfigError(f"{subclass} is not a subclass of {cls}")

    return subclass.from_config(config.config)

subclass_from_factory classmethod

subclass_from_factory(factory_path: str) -> Self

Creates the class using the provided factory function. May return a subclass of the class, if requested by the factory. Supports both synchronous and asynchronous factory functions.

PARAMETER	DESCRIPTION
factory_path	A string representing the path to the factory function in the format of "module.submodule:factory_name". TYPE: str

RETURNS	DESCRIPTION
Self	An instance of the class initialized with the provided factory function.

RAISES	DESCRIPTION
InvalidConfigError	The factory can't be found or the object returned is not a subclass of the current class.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def subclass_from_factory(cls, factory_path: str) -> Self:
    """
    Creates the class using the provided factory function. May return a subclass of the class,
    if requested by the factory. Supports both synchronous and asynchronous factory functions.

    Args:
        factory_path: A string representing the path to the factory function
            in the format of "module.submodule:factory_name".

    Returns:
        An instance of the class initialized with the provided factory function.

    Raises:
        InvalidConfigError: The factory can't be found or the object returned
            is not a subclass of the current class.
    """
    factory = import_by_path(factory_path, cls.default_module)

    if asyncio.iscoroutinefunction(factory):
        try:
            loop = asyncio.get_running_loop()
            obj = asyncio.run_coroutine_threadsafe(factory, loop).result()
        except RuntimeError:
            obj = asyncio.run(factory())
    else:
        obj = factory()

    if not isinstance(obj, cls):
        raise InvalidConfigError(f"The object returned by factory {factory_path} is not an instance of {cls}")

    return obj

preferred_subclass classmethod

preferred_subclass(config: CoreConfig, factory_path_override: str | None = None, yaml_path_override: Path | None = None) -> Self

Tries to create an instance by looking at project's component preferences, either from YAML or from the factory. Takes optional overrides for both, which takes a higher precedence.

PARAMETER	DESCRIPTION
config	The CoreConfig instance containing preferred factory and configuration details. TYPE: CoreConfig
factory_path_override	A string representing the path to the factory function in the format of "module.submodule:factory_name". TYPE: str | None DEFAULT: None
yaml_path_override	A string representing the path to the YAML file containing the Ragstack instance configuration. TYPE: Path | None DEFAULT: None

RAISES	DESCRIPTION
InvalidConfigError	If the default factory or configuration can't be found.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def preferred_subclass(
    cls, config: CoreConfig, factory_path_override: str | None = None, yaml_path_override: Path | None = None
) -> Self:
    """
    Tries to create an instance by looking at project's component preferences, either from YAML
    or from the factory. Takes optional overrides for both, which takes a higher precedence.

    Args:
        config: The CoreConfig instance containing preferred factory and configuration details.
        factory_path_override: A string representing the path to the factory function
            in the format of "module.submodule:factory_name".
        yaml_path_override: A string representing the path to the YAML file containing
            the Ragstack instance configuration.

    Raises:
        InvalidConfigError: If the default factory or configuration can't be found.
    """
    if yaml_path_override:
        preferences = get_config_from_yaml(yaml_path_override)
        if type_config := preferences.get(cls.configuration_key):
            return cls.subclass_from_config(ObjectConstructionConfig.model_validate(type_config))

    if factory_path_override:
        return cls.subclass_from_factory(factory_path_override)

    if preferred_factory := config.component_preference_factories.get(cls.configuration_key):
        return cls.subclass_from_factory(preferred_factory)

    if preferred_config := config.preferred_instances_config.get(cls.configuration_key):
        return cls.subclass_from_config(ObjectConstructionConfig.model_validate(preferred_config))

    raise NoPreferredConfigError(f"Could not find preferred factory or configuration for {cls.configuration_key}")

from_config classmethod

from_config(config: dict) -> Self

Create an instance of QuestionAnswerMetric from a configuration dictionary.

PARAMETER	DESCRIPTION
config	A dictionary containing configuration settings for the metric. TYPE: dict

RETURNS	DESCRIPTION
Self	An instance of the metric class initialized with the provided configuration.

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/question_answer.py

@classmethod
def from_config(cls, config: dict) -> Self:
    """
    Create an instance of `QuestionAnswerMetric` from a configuration dictionary.

    Args:
        config: A dictionary containing configuration settings for the metric.

    Returns:
        An instance of the metric class initialized with the provided configuration.
    """
    config["llm"] = LLM.from_config(config["llm"])
    config["batch_size"] = config.get("batch_size", 15)
    config["weight"] = config.get("weight", 1.0)
    return super().from_config(config)

compute async

compute(results: list[QuestionAnswerResult[QuestionAnswerPromptOutputT]]) -> dict

Compute the metric.

PARAMETER	DESCRIPTION
results	The evaluation results. TYPE: list[QuestionAnswerResult[QuestionAnswerPromptOutputT]]

RETURNS	DESCRIPTION
dict	The computed metric.

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/question_answer.py

async def compute(self, results: list[QuestionAnswerResult[QuestionAnswerPromptOutputT]]) -> dict:
    """
    Compute the metric.

    Args:
        results: The evaluation results.

    Returns:
        The computed metric.
    """
    metric = self.metric_cls(_MetricLMM(self.llm, loop=asyncio.get_running_loop()))
    metric_results = chain.from_iterable(
        [
            await asyncio.gather(*[asyncio.to_thread(self._call_metric, metric, result) for result in batch])
            for batch in batched(results, self.batch_size)
        ]
    )
    return metric.aggregate(list(metric_results))

ragbits.evaluate.metrics.question_answer.QuestionAnswerAnswerCorrectness

QuestionAnswerAnswerCorrectness(llm: LLM, batch_size: int = 15, weight: float = 1.0)

Bases: QuestionAnswerMetric[LLMBasedAnswerCorrectness]

Metric checking answer correctness based on LLM. More details can be found here.

Initialize the agent metric.

PARAMETER	DESCRIPTION
llm	Judge LLM instance. TYPE: LLM
batch_size	Batch size for metric computation. TYPE: int DEFAULT: 15
weight	Metric value weight in the final score, used during optimization. TYPE: float DEFAULT: 1.0

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/question_answer.py

def __init__(self, llm: LLM, batch_size: int = 15, weight: float = 1.0) -> None:
    """
    Initialize the agent metric.

    Args:
        llm: Judge LLM instance.
        batch_size: Batch size for metric computation.
        weight: Metric value weight in the final score, used during optimization.
    """
    super().__init__(weight=weight)
    self.llm = llm
    self.batch_size = batch_size

default_module class-attribute

default_module: ModuleType | None = metrics

configuration_key class-attribute

configuration_key: str = 'metric'

weight instance-attribute

weight = weight

llm instance-attribute

llm = llm

batch_size instance-attribute

batch_size = batch_size

metric_cls class-attribute instance-attribute

metric_cls: type[LLMBasedAnswerCorrectness] = LLMBasedAnswerCorrectness

subclass_from_config classmethod

subclass_from_config(config: ObjectConstructionConfig) -> Self

Initializes the class with the provided configuration. May return a subclass of the class, if requested by the configuration.

PARAMETER	DESCRIPTION
config	A model containing configuration details for the class. TYPE: ObjectConstructionConfig

RETURNS	DESCRIPTION
Self	An instance of the class initialized with the provided configuration.

RAISES	DESCRIPTION
InvalidConfigError	The class can't be found or is not a subclass of the current class.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def subclass_from_config(cls, config: ObjectConstructionConfig) -> Self:
    """
    Initializes the class with the provided configuration. May return a subclass of the class,
    if requested by the configuration.

    Args:
        config: A model containing configuration details for the class.

    Returns:
        An instance of the class initialized with the provided configuration.

    Raises:
        InvalidConfigError: The class can't be found or is not a subclass of the current class.
    """
    subclass = import_by_path(config.type, cls.default_module)
    if not issubclass(subclass, cls):
        raise InvalidConfigError(f"{subclass} is not a subclass of {cls}")

    return subclass.from_config(config.config)

subclass_from_factory classmethod

subclass_from_factory(factory_path: str) -> Self

Creates the class using the provided factory function. May return a subclass of the class, if requested by the factory. Supports both synchronous and asynchronous factory functions.

PARAMETER	DESCRIPTION
factory_path	A string representing the path to the factory function in the format of "module.submodule:factory_name". TYPE: str

RETURNS	DESCRIPTION
Self	An instance of the class initialized with the provided factory function.

RAISES	DESCRIPTION
InvalidConfigError	The factory can't be found or the object returned is not a subclass of the current class.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def subclass_from_factory(cls, factory_path: str) -> Self:
    """
    Creates the class using the provided factory function. May return a subclass of the class,
    if requested by the factory. Supports both synchronous and asynchronous factory functions.

    Args:
        factory_path: A string representing the path to the factory function
            in the format of "module.submodule:factory_name".

    Returns:
        An instance of the class initialized with the provided factory function.

    Raises:
        InvalidConfigError: The factory can't be found or the object returned
            is not a subclass of the current class.
    """
    factory = import_by_path(factory_path, cls.default_module)

    if asyncio.iscoroutinefunction(factory):
        try:
            loop = asyncio.get_running_loop()
            obj = asyncio.run_coroutine_threadsafe(factory, loop).result()
        except RuntimeError:
            obj = asyncio.run(factory())
    else:
        obj = factory()

    if not isinstance(obj, cls):
        raise InvalidConfigError(f"The object returned by factory {factory_path} is not an instance of {cls}")

    return obj

preferred_subclass classmethod

preferred_subclass(config: CoreConfig, factory_path_override: str | None = None, yaml_path_override: Path | None = None) -> Self

Tries to create an instance by looking at project's component preferences, either from YAML or from the factory. Takes optional overrides for both, which takes a higher precedence.

PARAMETER	DESCRIPTION
config	The CoreConfig instance containing preferred factory and configuration details. TYPE: CoreConfig
factory_path_override	A string representing the path to the factory function in the format of "module.submodule:factory_name". TYPE: str | None DEFAULT: None
yaml_path_override	A string representing the path to the YAML file containing the Ragstack instance configuration. TYPE: Path | None DEFAULT: None

RAISES	DESCRIPTION
InvalidConfigError	If the default factory or configuration can't be found.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def preferred_subclass(
    cls, config: CoreConfig, factory_path_override: str | None = None, yaml_path_override: Path | None = None
) -> Self:
    """
    Tries to create an instance by looking at project's component preferences, either from YAML
    or from the factory. Takes optional overrides for both, which takes a higher precedence.

    Args:
        config: The CoreConfig instance containing preferred factory and configuration details.
        factory_path_override: A string representing the path to the factory function
            in the format of "module.submodule:factory_name".
        yaml_path_override: A string representing the path to the YAML file containing
            the Ragstack instance configuration.

    Raises:
        InvalidConfigError: If the default factory or configuration can't be found.
    """
    if yaml_path_override:
        preferences = get_config_from_yaml(yaml_path_override)
        if type_config := preferences.get(cls.configuration_key):
            return cls.subclass_from_config(ObjectConstructionConfig.model_validate(type_config))

    if factory_path_override:
        return cls.subclass_from_factory(factory_path_override)

    if preferred_factory := config.component_preference_factories.get(cls.configuration_key):
        return cls.subclass_from_factory(preferred_factory)

    if preferred_config := config.preferred_instances_config.get(cls.configuration_key):
        return cls.subclass_from_config(ObjectConstructionConfig.model_validate(preferred_config))

    raise NoPreferredConfigError(f"Could not find preferred factory or configuration for {cls.configuration_key}")

from_config classmethod

from_config(config: dict) -> Self

Create an instance of QuestionAnswerMetric from a configuration dictionary.

PARAMETER	DESCRIPTION
config	A dictionary containing configuration settings for the metric. TYPE: dict

RETURNS	DESCRIPTION
Self	An instance of the metric class initialized with the provided configuration.

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/question_answer.py

@classmethod
def from_config(cls, config: dict) -> Self:
    """
    Create an instance of `QuestionAnswerMetric` from a configuration dictionary.

    Args:
        config: A dictionary containing configuration settings for the metric.

    Returns:
        An instance of the metric class initialized with the provided configuration.
    """
    config["llm"] = LLM.from_config(config["llm"])
    config["batch_size"] = config.get("batch_size", 15)
    config["weight"] = config.get("weight", 1.0)
    return super().from_config(config)

compute async

compute(results: list[QuestionAnswerResult[QuestionAnswerPromptOutputT]]) -> dict

Compute the metric.

PARAMETER	DESCRIPTION
results	The evaluation results. TYPE: list[QuestionAnswerResult[QuestionAnswerPromptOutputT]]

RETURNS	DESCRIPTION
dict	The computed metric.

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/question_answer.py

async def compute(self, results: list[QuestionAnswerResult[QuestionAnswerPromptOutputT]]) -> dict:
    """
    Compute the metric.

    Args:
        results: The evaluation results.

    Returns:
        The computed metric.
    """
    metric = self.metric_cls(_MetricLMM(self.llm, loop=asyncio.get_running_loop()))
    metric_results = chain.from_iterable(
        [
            await asyncio.gather(*[asyncio.to_thread(self._call_metric, metric, result) for result in batch])
            for batch in batched(results, self.batch_size)
        ]
    )
    return metric.aggregate(list(metric_results))

ragbits.evaluate.metrics.question_answer.QuestionAnswerAnswerFaithfulness

QuestionAnswerAnswerFaithfulness(llm: LLM, batch_size: int = 15, weight: float = 1.0)

Bases: QuestionAnswerMetric[LLMBasedFaithfulness]

Metric checking answer faithfulness based on LLM. More details can be found here.

Initialize the agent metric.

PARAMETER	DESCRIPTION
llm	Judge LLM instance. TYPE: LLM
batch_size	Batch size for metric computation. TYPE: int DEFAULT: 15
weight	Metric value weight in the final score, used during optimization. TYPE: float DEFAULT: 1.0

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/question_answer.py

def __init__(self, llm: LLM, batch_size: int = 15, weight: float = 1.0) -> None:
    """
    Initialize the agent metric.

    Args:
        llm: Judge LLM instance.
        batch_size: Batch size for metric computation.
        weight: Metric value weight in the final score, used during optimization.
    """
    super().__init__(weight=weight)
    self.llm = llm
    self.batch_size = batch_size

default_module class-attribute

default_module: ModuleType | None = metrics

configuration_key class-attribute

configuration_key: str = 'metric'

weight instance-attribute

weight = weight

llm instance-attribute

llm = llm

batch_size instance-attribute

batch_size = batch_size

metric_cls class-attribute instance-attribute

metric_cls: type[LLMBasedFaithfulness] = LLMBasedFaithfulness

subclass_from_config classmethod

subclass_from_config(config: ObjectConstructionConfig) -> Self

Initializes the class with the provided configuration. May return a subclass of the class, if requested by the configuration.

PARAMETER	DESCRIPTION
config	A model containing configuration details for the class. TYPE: ObjectConstructionConfig

RETURNS	DESCRIPTION
Self	An instance of the class initialized with the provided configuration.

RAISES	DESCRIPTION
InvalidConfigError	The class can't be found or is not a subclass of the current class.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def subclass_from_config(cls, config: ObjectConstructionConfig) -> Self:
    """
    Initializes the class with the provided configuration. May return a subclass of the class,
    if requested by the configuration.

    Args:
        config: A model containing configuration details for the class.

    Returns:
        An instance of the class initialized with the provided configuration.

    Raises:
        InvalidConfigError: The class can't be found or is not a subclass of the current class.
    """
    subclass = import_by_path(config.type, cls.default_module)
    if not issubclass(subclass, cls):
        raise InvalidConfigError(f"{subclass} is not a subclass of {cls}")

    return subclass.from_config(config.config)

subclass_from_factory classmethod

subclass_from_factory(factory_path: str) -> Self

Creates the class using the provided factory function. May return a subclass of the class, if requested by the factory. Supports both synchronous and asynchronous factory functions.

PARAMETER	DESCRIPTION
factory_path	A string representing the path to the factory function in the format of "module.submodule:factory_name". TYPE: str

RETURNS	DESCRIPTION
Self	An instance of the class initialized with the provided factory function.

RAISES	DESCRIPTION
InvalidConfigError	The factory can't be found or the object returned is not a subclass of the current class.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def subclass_from_factory(cls, factory_path: str) -> Self:
    """
    Creates the class using the provided factory function. May return a subclass of the class,
    if requested by the factory. Supports both synchronous and asynchronous factory functions.

    Args:
        factory_path: A string representing the path to the factory function
            in the format of "module.submodule:factory_name".

    Returns:
        An instance of the class initialized with the provided factory function.

    Raises:
        InvalidConfigError: The factory can't be found or the object returned
            is not a subclass of the current class.
    """
    factory = import_by_path(factory_path, cls.default_module)

    if asyncio.iscoroutinefunction(factory):
        try:
            loop = asyncio.get_running_loop()
            obj = asyncio.run_coroutine_threadsafe(factory, loop).result()
        except RuntimeError:
            obj = asyncio.run(factory())
    else:
        obj = factory()

    if not isinstance(obj, cls):
        raise InvalidConfigError(f"The object returned by factory {factory_path} is not an instance of {cls}")

    return obj

preferred_subclass classmethod

preferred_subclass(config: CoreConfig, factory_path_override: str | None = None, yaml_path_override: Path | None = None) -> Self

Tries to create an instance by looking at project's component preferences, either from YAML or from the factory. Takes optional overrides for both, which takes a higher precedence.

PARAMETER	DESCRIPTION
config	The CoreConfig instance containing preferred factory and configuration details. TYPE: CoreConfig
factory_path_override	A string representing the path to the factory function in the format of "module.submodule:factory_name". TYPE: str | None DEFAULT: None
yaml_path_override	A string representing the path to the YAML file containing the Ragstack instance configuration. TYPE: Path | None DEFAULT: None

RAISES	DESCRIPTION
InvalidConfigError	If the default factory or configuration can't be found.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def preferred_subclass(
    cls, config: CoreConfig, factory_path_override: str | None = None, yaml_path_override: Path | None = None
) -> Self:
    """
    Tries to create an instance by looking at project's component preferences, either from YAML
    or from the factory. Takes optional overrides for both, which takes a higher precedence.

    Args:
        config: The CoreConfig instance containing preferred factory and configuration details.
        factory_path_override: A string representing the path to the factory function
            in the format of "module.submodule:factory_name".
        yaml_path_override: A string representing the path to the YAML file containing
            the Ragstack instance configuration.

    Raises:
        InvalidConfigError: If the default factory or configuration can't be found.
    """
    if yaml_path_override:
        preferences = get_config_from_yaml(yaml_path_override)
        if type_config := preferences.get(cls.configuration_key):
            return cls.subclass_from_config(ObjectConstructionConfig.model_validate(type_config))

    if factory_path_override:
        return cls.subclass_from_factory(factory_path_override)

    if preferred_factory := config.component_preference_factories.get(cls.configuration_key):
        return cls.subclass_from_factory(preferred_factory)

    if preferred_config := config.preferred_instances_config.get(cls.configuration_key):
        return cls.subclass_from_config(ObjectConstructionConfig.model_validate(preferred_config))

    raise NoPreferredConfigError(f"Could not find preferred factory or configuration for {cls.configuration_key}")

from_config classmethod

from_config(config: dict) -> Self

Create an instance of QuestionAnswerMetric from a configuration dictionary.

PARAMETER	DESCRIPTION
config	A dictionary containing configuration settings for the metric. TYPE: dict

RETURNS	DESCRIPTION
Self	An instance of the metric class initialized with the provided configuration.

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/question_answer.py

@classmethod
def from_config(cls, config: dict) -> Self:
    """
    Create an instance of `QuestionAnswerMetric` from a configuration dictionary.

    Args:
        config: A dictionary containing configuration settings for the metric.

    Returns:
        An instance of the metric class initialized with the provided configuration.
    """
    config["llm"] = LLM.from_config(config["llm"])
    config["batch_size"] = config.get("batch_size", 15)
    config["weight"] = config.get("weight", 1.0)
    return super().from_config(config)

compute async

compute(results: list[QuestionAnswerResult[QuestionAnswerPromptOutputT]]) -> dict

Compute the metric.

PARAMETER	DESCRIPTION
results	The evaluation results. TYPE: list[QuestionAnswerResult[QuestionAnswerPromptOutputT]]

RETURNS	DESCRIPTION
dict	The computed metric.

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/question_answer.py

async def compute(self, results: list[QuestionAnswerResult[QuestionAnswerPromptOutputT]]) -> dict:
    """
    Compute the metric.

    Args:
        results: The evaluation results.

    Returns:
        The computed metric.
    """
    metric = self.metric_cls(_MetricLMM(self.llm, loop=asyncio.get_running_loop()))
    metric_results = chain.from_iterable(
        [
            await asyncio.gather(*[asyncio.to_thread(self._call_metric, metric, result) for result in batch])
            for batch in batched(results, self.batch_size)
        ]
    )
    return metric.aggregate(list(metric_results))

ragbits.evaluate.metrics.question_answer.QuestionAnswerAnswerRelevance

QuestionAnswerAnswerRelevance(llm: LLM, batch_size: int = 15, weight: float = 1.0)

Bases: QuestionAnswerMetric[LLMBasedAnswerRelevance]

Metric checking answer relevance based on LLM. More details can be found here.

Initialize the agent metric.

PARAMETER	DESCRIPTION
llm	Judge LLM instance. TYPE: LLM
batch_size	Batch size for metric computation. TYPE: int DEFAULT: 15
weight	Metric value weight in the final score, used during optimization. TYPE: float DEFAULT: 1.0

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/question_answer.py

def __init__(self, llm: LLM, batch_size: int = 15, weight: float = 1.0) -> None:
    """
    Initialize the agent metric.

    Args:
        llm: Judge LLM instance.
        batch_size: Batch size for metric computation.
        weight: Metric value weight in the final score, used during optimization.
    """
    super().__init__(weight=weight)
    self.llm = llm
    self.batch_size = batch_size

default_module class-attribute

default_module: ModuleType | None = metrics

configuration_key class-attribute

configuration_key: str = 'metric'

weight instance-attribute

weight = weight

llm instance-attribute

llm = llm

batch_size instance-attribute

batch_size = batch_size

metric_cls class-attribute instance-attribute

metric_cls: type[LLMBasedAnswerRelevance] = LLMBasedAnswerRelevance

subclass_from_config classmethod

subclass_from_config(config: ObjectConstructionConfig) -> Self

Initializes the class with the provided configuration. May return a subclass of the class, if requested by the configuration.

PARAMETER	DESCRIPTION
config	A model containing configuration details for the class. TYPE: ObjectConstructionConfig

RETURNS	DESCRIPTION
Self	An instance of the class initialized with the provided configuration.

RAISES	DESCRIPTION
InvalidConfigError	The class can't be found or is not a subclass of the current class.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def subclass_from_config(cls, config: ObjectConstructionConfig) -> Self:
    """
    Initializes the class with the provided configuration. May return a subclass of the class,
    if requested by the configuration.

    Args:
        config: A model containing configuration details for the class.

    Returns:
        An instance of the class initialized with the provided configuration.

    Raises:
        InvalidConfigError: The class can't be found or is not a subclass of the current class.
    """
    subclass = import_by_path(config.type, cls.default_module)
    if not issubclass(subclass, cls):
        raise InvalidConfigError(f"{subclass} is not a subclass of {cls}")

    return subclass.from_config(config.config)

subclass_from_factory classmethod

subclass_from_factory(factory_path: str) -> Self

Creates the class using the provided factory function. May return a subclass of the class, if requested by the factory. Supports both synchronous and asynchronous factory functions.

PARAMETER	DESCRIPTION
factory_path	A string representing the path to the factory function in the format of "module.submodule:factory_name". TYPE: str

RETURNS	DESCRIPTION
Self	An instance of the class initialized with the provided factory function.

RAISES	DESCRIPTION
InvalidConfigError	The factory can't be found or the object returned is not a subclass of the current class.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def subclass_from_factory(cls, factory_path: str) -> Self:
    """
    Creates the class using the provided factory function. May return a subclass of the class,
    if requested by the factory. Supports both synchronous and asynchronous factory functions.

    Args:
        factory_path: A string representing the path to the factory function
            in the format of "module.submodule:factory_name".

    Returns:
        An instance of the class initialized with the provided factory function.

    Raises:
        InvalidConfigError: The factory can't be found or the object returned
            is not a subclass of the current class.
    """
    factory = import_by_path(factory_path, cls.default_module)

    if asyncio.iscoroutinefunction(factory):
        try:
            loop = asyncio.get_running_loop()
            obj = asyncio.run_coroutine_threadsafe(factory, loop).result()
        except RuntimeError:
            obj = asyncio.run(factory())
    else:
        obj = factory()

    if not isinstance(obj, cls):
        raise InvalidConfigError(f"The object returned by factory {factory_path} is not an instance of {cls}")

    return obj

preferred_subclass classmethod

preferred_subclass(config: CoreConfig, factory_path_override: str | None = None, yaml_path_override: Path | None = None) -> Self

Tries to create an instance by looking at project's component preferences, either from YAML or from the factory. Takes optional overrides for both, which takes a higher precedence.

PARAMETER	DESCRIPTION
config	The CoreConfig instance containing preferred factory and configuration details. TYPE: CoreConfig
factory_path_override	A string representing the path to the factory function in the format of "module.submodule:factory_name". TYPE: str | None DEFAULT: None
yaml_path_override	A string representing the path to the YAML file containing the Ragstack instance configuration. TYPE: Path | None DEFAULT: None

RAISES	DESCRIPTION
InvalidConfigError	If the default factory or configuration can't be found.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def preferred_subclass(
    cls, config: CoreConfig, factory_path_override: str | None = None, yaml_path_override: Path | None = None
) -> Self:
    """
    Tries to create an instance by looking at project's component preferences, either from YAML
    or from the factory. Takes optional overrides for both, which takes a higher precedence.

    Args:
        config: The CoreConfig instance containing preferred factory and configuration details.
        factory_path_override: A string representing the path to the factory function
            in the format of "module.submodule:factory_name".
        yaml_path_override: A string representing the path to the YAML file containing
            the Ragstack instance configuration.

    Raises:
        InvalidConfigError: If the default factory or configuration can't be found.
    """
    if yaml_path_override:
        preferences = get_config_from_yaml(yaml_path_override)
        if type_config := preferences.get(cls.configuration_key):
            return cls.subclass_from_config(ObjectConstructionConfig.model_validate(type_config))

    if factory_path_override:
        return cls.subclass_from_factory(factory_path_override)

    if preferred_factory := config.component_preference_factories.get(cls.configuration_key):
        return cls.subclass_from_factory(preferred_factory)

    if preferred_config := config.preferred_instances_config.get(cls.configuration_key):
        return cls.subclass_from_config(ObjectConstructionConfig.model_validate(preferred_config))

    raise NoPreferredConfigError(f"Could not find preferred factory or configuration for {cls.configuration_key}")

from_config classmethod

from_config(config: dict) -> Self

Create an instance of QuestionAnswerMetric from a configuration dictionary.

PARAMETER	DESCRIPTION
config	A dictionary containing configuration settings for the metric. TYPE: dict

RETURNS	DESCRIPTION
Self	An instance of the metric class initialized with the provided configuration.

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/question_answer.py

@classmethod
def from_config(cls, config: dict) -> Self:
    """
    Create an instance of `QuestionAnswerMetric` from a configuration dictionary.

    Args:
        config: A dictionary containing configuration settings for the metric.

    Returns:
        An instance of the metric class initialized with the provided configuration.
    """
    config["llm"] = LLM.from_config(config["llm"])
    config["batch_size"] = config.get("batch_size", 15)
    config["weight"] = config.get("weight", 1.0)
    return super().from_config(config)

compute async

compute(results: list[QuestionAnswerResult[QuestionAnswerPromptOutputT]]) -> dict

Compute the metric.

PARAMETER	DESCRIPTION
results	The evaluation results. TYPE: list[QuestionAnswerResult[QuestionAnswerPromptOutputT]]

RETURNS	DESCRIPTION
dict	The computed metric.

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/question_answer.py

async def compute(self, results: list[QuestionAnswerResult[QuestionAnswerPromptOutputT]]) -> dict:
    """
    Compute the metric.

    Args:
        results: The evaluation results.

    Returns:
        The computed metric.
    """
    metric = self.metric_cls(_MetricLMM(self.llm, loop=asyncio.get_running_loop()))
    metric_results = chain.from_iterable(
        [
            await asyncio.gather(*[asyncio.to_thread(self._call_metric, metric, result) for result in batch])
            for batch in batched(results, self.batch_size)
        ]
    )
    return metric.aggregate(list(metric_results))

ragbits.evaluate.metrics.question_answer.QuestionAnswerAnswerConsistency

QuestionAnswerAnswerConsistency(llm: LLM, batch_size: int = 15, weight: float = 1.0)

Bases: QuestionAnswerMetric[LLMBasedStyleConsistency]

Metric checking answer relevance based on LLM. More details can be found here.

Initialize the agent metric.

PARAMETER	DESCRIPTION
llm	Judge LLM instance. TYPE: LLM
batch_size	Batch size for metric computation. TYPE: int DEFAULT: 15
weight	Metric value weight in the final score, used during optimization. TYPE: float DEFAULT: 1.0

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/question_answer.py

def __init__(self, llm: LLM, batch_size: int = 15, weight: float = 1.0) -> None:
    """
    Initialize the agent metric.

    Args:
        llm: Judge LLM instance.
        batch_size: Batch size for metric computation.
        weight: Metric value weight in the final score, used during optimization.
    """
    super().__init__(weight=weight)
    self.llm = llm
    self.batch_size = batch_size

default_module class-attribute

default_module: ModuleType | None = metrics

configuration_key class-attribute

configuration_key: str = 'metric'

weight instance-attribute

weight = weight

llm instance-attribute

llm = llm

batch_size instance-attribute

batch_size = batch_size

metric_cls class-attribute instance-attribute

metric_cls: type[LLMBasedStyleConsistency] = LLMBasedStyleConsistency

subclass_from_config classmethod

subclass_from_config(config: ObjectConstructionConfig) -> Self

Initializes the class with the provided configuration. May return a subclass of the class, if requested by the configuration.

PARAMETER	DESCRIPTION
config	A model containing configuration details for the class. TYPE: ObjectConstructionConfig

RETURNS	DESCRIPTION
Self	An instance of the class initialized with the provided configuration.

RAISES	DESCRIPTION
InvalidConfigError	The class can't be found or is not a subclass of the current class.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def subclass_from_config(cls, config: ObjectConstructionConfig) -> Self:
    """
    Initializes the class with the provided configuration. May return a subclass of the class,
    if requested by the configuration.

    Args:
        config: A model containing configuration details for the class.

    Returns:
        An instance of the class initialized with the provided configuration.

    Raises:
        InvalidConfigError: The class can't be found or is not a subclass of the current class.
    """
    subclass = import_by_path(config.type, cls.default_module)
    if not issubclass(subclass, cls):
        raise InvalidConfigError(f"{subclass} is not a subclass of {cls}")

    return subclass.from_config(config.config)

subclass_from_factory classmethod

subclass_from_factory(factory_path: str) -> Self

Creates the class using the provided factory function. May return a subclass of the class, if requested by the factory. Supports both synchronous and asynchronous factory functions.

PARAMETER	DESCRIPTION
factory_path	A string representing the path to the factory function in the format of "module.submodule:factory_name". TYPE: str

RETURNS	DESCRIPTION
Self	An instance of the class initialized with the provided factory function.

RAISES	DESCRIPTION
InvalidConfigError	The factory can't be found or the object returned is not a subclass of the current class.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def subclass_from_factory(cls, factory_path: str) -> Self:
    """
    Creates the class using the provided factory function. May return a subclass of the class,
    if requested by the factory. Supports both synchronous and asynchronous factory functions.

    Args:
        factory_path: A string representing the path to the factory function
            in the format of "module.submodule:factory_name".

    Returns:
        An instance of the class initialized with the provided factory function.

    Raises:
        InvalidConfigError: The factory can't be found or the object returned
            is not a subclass of the current class.
    """
    factory = import_by_path(factory_path, cls.default_module)

    if asyncio.iscoroutinefunction(factory):
        try:
            loop = asyncio.get_running_loop()
            obj = asyncio.run_coroutine_threadsafe(factory, loop).result()
        except RuntimeError:
            obj = asyncio.run(factory())
    else:
        obj = factory()

    if not isinstance(obj, cls):
        raise InvalidConfigError(f"The object returned by factory {factory_path} is not an instance of {cls}")

    return obj

preferred_subclass classmethod

preferred_subclass(config: CoreConfig, factory_path_override: str | None = None, yaml_path_override: Path | None = None) -> Self

Tries to create an instance by looking at project's component preferences, either from YAML or from the factory. Takes optional overrides for both, which takes a higher precedence.

PARAMETER	DESCRIPTION
config	The CoreConfig instance containing preferred factory and configuration details. TYPE: CoreConfig
factory_path_override	A string representing the path to the factory function in the format of "module.submodule:factory_name". TYPE: str | None DEFAULT: None
yaml_path_override	A string representing the path to the YAML file containing the Ragstack instance configuration. TYPE: Path | None DEFAULT: None

RAISES	DESCRIPTION
InvalidConfigError	If the default factory or configuration can't be found.

Source code in packages/ragbits-core/src/ragbits/core/utils/config_handling.py

@classmethod
def preferred_subclass(
    cls, config: CoreConfig, factory_path_override: str | None = None, yaml_path_override: Path | None = None
) -> Self:
    """
    Tries to create an instance by looking at project's component preferences, either from YAML
    or from the factory. Takes optional overrides for both, which takes a higher precedence.

    Args:
        config: The CoreConfig instance containing preferred factory and configuration details.
        factory_path_override: A string representing the path to the factory function
            in the format of "module.submodule:factory_name".
        yaml_path_override: A string representing the path to the YAML file containing
            the Ragstack instance configuration.

    Raises:
        InvalidConfigError: If the default factory or configuration can't be found.
    """
    if yaml_path_override:
        preferences = get_config_from_yaml(yaml_path_override)
        if type_config := preferences.get(cls.configuration_key):
            return cls.subclass_from_config(ObjectConstructionConfig.model_validate(type_config))

    if factory_path_override:
        return cls.subclass_from_factory(factory_path_override)

    if preferred_factory := config.component_preference_factories.get(cls.configuration_key):
        return cls.subclass_from_factory(preferred_factory)

    if preferred_config := config.preferred_instances_config.get(cls.configuration_key):
        return cls.subclass_from_config(ObjectConstructionConfig.model_validate(preferred_config))

    raise NoPreferredConfigError(f"Could not find preferred factory or configuration for {cls.configuration_key}")

from_config classmethod

from_config(config: dict) -> Self

Create an instance of QuestionAnswerMetric from a configuration dictionary.

PARAMETER	DESCRIPTION
config	A dictionary containing configuration settings for the metric. TYPE: dict

RETURNS	DESCRIPTION
Self	An instance of the metric class initialized with the provided configuration.

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/question_answer.py

@classmethod
def from_config(cls, config: dict) -> Self:
    """
    Create an instance of `QuestionAnswerMetric` from a configuration dictionary.

    Args:
        config: A dictionary containing configuration settings for the metric.

    Returns:
        An instance of the metric class initialized with the provided configuration.
    """
    config["llm"] = LLM.from_config(config["llm"])
    config["batch_size"] = config.get("batch_size", 15)
    config["weight"] = config.get("weight", 1.0)
    return super().from_config(config)

compute async

compute(results: list[QuestionAnswerResult[QuestionAnswerPromptOutputT]]) -> dict

Compute the metric.

PARAMETER	DESCRIPTION
results	The evaluation results. TYPE: list[QuestionAnswerResult[QuestionAnswerPromptOutputT]]

RETURNS	DESCRIPTION
dict	The computed metric.

Source code in packages/ragbits-evaluate/src/ragbits/evaluate/metrics/question_answer.py

async def compute(self, results: list[QuestionAnswerResult[QuestionAnswerPromptOutputT]]) -> dict:
    """
    Compute the metric.

    Args:
        results: The evaluation results.

    Returns:
        The computed metric.
    """
    metric = self.metric_cls(_MetricLMM(self.llm, loop=asyncio.get_running_loop()))
    metric_results = chain.from_iterable(
        [
            await asyncio.gather(*[asyncio.to_thread(self._call_metric, metric, result) for result in batch])
            for batch in batched(results, self.batch_size)
        ]
    )
    return metric.aggregate(list(metric_results))