Metrics

ragbits.core.audit.metrics.set_metric_handlers

set_metric_handlers(handlers: Handler | list[Handler]) -> None

Set the global metric handlers.

PARAMETER	DESCRIPTION
handlers	List of metric handlers to be used. TYPE: Handler | list[Handler]

RAISES	DESCRIPTION
ValueError	If handler is not found.
TypeError	If handler type is invalid.

Source code in packages/ragbits-core/src/ragbits/core/audit/metrics/__init__.py

def set_metric_handlers(handlers: Handler | list[Handler]) -> None:
    """
    Set the global metric handlers.

    Args:
        handlers: List of metric handlers to be used.

    Raises:
        ValueError: If handler is not found.
        TypeError: If handler type is invalid.
    """
    global _metric_handlers  # noqa: PLW0602

    if isinstance(handlers, Handler):
        handlers = [handlers]

    for handler in handlers:
        if isinstance(handler, MetricHandler):
            _metric_handlers.append(handler)
        elif isinstance(handler, str):
            if handler == "otel":
                from ragbits.core.audit.metrics.otel import OtelMetricHandler

                if not any(isinstance(item, OtelMetricHandler) for item in _metric_handlers):
                    _metric_handlers.append(OtelMetricHandler())
            else:
                raise ValueError(f"Not found handler: {handler}")
        else:
            raise TypeError(f"Invalid handler type: {type(handler)}")

ragbits.core.audit.metrics.clear_metric_handlers

clear_metric_handlers() -> None

Clear all metric handlers.

Source code in packages/ragbits-core/src/ragbits/core/audit/metrics/__init__.py

def clear_metric_handlers() -> None:
    """
    Clear all metric handlers.
    """
    global _metric_handlers  # noqa: PLW0602
    _metric_handlers.clear()

ragbits.core.audit.metrics.create_histogram

create_histogram(name: str, unit: str = '', description: str = '') -> str

Create a histogram metric.

PARAMETER	DESCRIPTION
name	The histogram metric name. TYPE: str
unit	The histogram metric unit. TYPE: str DEFAULT: ''
description	The histogram metric description. TYPE: str DEFAULT: ''

RETURNS	DESCRIPTION
str	The initialized histogram metric.

Source code in packages/ragbits-core/src/ragbits/core/audit/metrics/__init__.py

def create_histogram(name: str, unit: str = "", description: str = "") -> str:
    """
    Create a histogram metric.

    Args:
        name: The histogram metric name.
        unit: The histogram metric unit.
        description: The histogram metric description.

    Returns:
        The initialized histogram metric.
    """
    for handler in _metric_handlers:
        handler.register_histogram(name=name, unit=unit, description=description)
    return name

ragbits.core.audit.metrics.record

record(metric: HistogramMetric | str, value: int | float, **attributes: Any) -> None

Record a histogram metric using the global metric handlers.

PARAMETER	DESCRIPTION
metric	The histogram metric name to record. TYPE: HistogramMetric | str
value	The value to record. TYPE: int | float
attributes	Additional metadata for the metric. TYPE: Any DEFAULT: {}

Source code in packages/ragbits-core/src/ragbits/core/audit/metrics/__init__.py

def record(metric: HistogramMetric | str, value: int | float, **attributes: Any) -> None:  # noqa: ANN401
    """
    Record a histogram metric using the global metric handlers.

    Args:
        metric: The histogram metric name to record.
        value: The value to record.
        attributes: Additional metadata for the metric.
    """
    for handler in _metric_handlers:
        handler.record_histogram(metric=metric, value=value, attributes=attributes)

ragbits.core.audit.metrics.base.HistogramMetric

Bases: Enum

Histogram metric types that can be recorded.

PROMPT_THROUGHPUT class-attribute instance-attribute

PROMPT_THROUGHPUT = auto()

TOKEN_THROUGHPUT class-attribute instance-attribute

TOKEN_THROUGHPUT = auto()

INPUT_TOKENS class-attribute instance-attribute

INPUT_TOKENS = auto()

TIME_TO_FIRST_TOKEN class-attribute instance-attribute

TIME_TO_FIRST_TOKEN = auto()

ragbits.core.audit.metrics.base.MetricHandler

MetricHandler(metric_prefix: str = 'ragbits')

Bases: Generic[HistogramT], ABC

Base class for all metric handlers.

Initialize the MetricHandler instance.

PARAMETER	DESCRIPTION
metric_prefix	Prefix for all metric names. TYPE: str DEFAULT: 'ragbits'

Source code in packages/ragbits-core/src/ragbits/core/audit/metrics/base.py

def __init__(self, metric_prefix: str = "ragbits") -> None:
    """
    Initialize the MetricHandler instance.

    Args:
        metric_prefix: Prefix for all metric names.
    """
    super().__init__()
    self._metric_prefix = metric_prefix
    self._histogram_metrics: dict[str, HistogramT] = {}

create_histogram abstractmethod

create_histogram(name: str, unit: str = '', description: str = '') -> HistogramT

Create a histogram metric.

PARAMETER	DESCRIPTION
name	The histogram metric name. TYPE: str
unit	The histogram metric unit. TYPE: str DEFAULT: ''
description	The histogram metric description. TYPE: str DEFAULT: ''

RETURNS	DESCRIPTION
HistogramT	The initialized histogram metric.

Source code in packages/ragbits-core/src/ragbits/core/audit/metrics/base.py

@abstractmethod
def create_histogram(self, name: str, unit: str = "", description: str = "") -> HistogramT:
    """
    Create a histogram metric.

    Args:
        name: The histogram metric name.
        unit: The histogram metric unit.
        description: The histogram metric description.

    Returns:
        The initialized histogram metric.
    """

record abstractmethod

record(metric: HistogramT, value: int | float, attributes: dict | None = None) -> None

Record the value for a specified histogram metric.

PARAMETER	DESCRIPTION
metric	The histogram metric to record. TYPE: HistogramT
value	The value to record for the metric. TYPE: int | float
attributes	Additional metadata for the metric. TYPE: dict | None DEFAULT: None

Source code in packages/ragbits-core/src/ragbits/core/audit/metrics/base.py

@abstractmethod
def record(self, metric: HistogramT, value: int | float, attributes: dict | None = None) -> None:
    """
    Record the value for a specified histogram metric.

    Args:
        metric: The histogram metric to record.
        value: The value to record for the metric.
        attributes: Additional metadata for the metric.
    """

register_histogram

register_histogram(name: str, unit: str = '', description: str = '') -> None

Register a histogram metric.

PARAMETER	DESCRIPTION
name	The histogram metric name. TYPE: str
unit	The histogram metric unit. TYPE: str DEFAULT: ''
description	The histogram metric description. TYPE: str DEFAULT: ''

RETURNS	DESCRIPTION
None	The registered histogram metric.

Source code in packages/ragbits-core/src/ragbits/core/audit/metrics/base.py

def register_histogram(self, name: str, unit: str = "", description: str = "") -> None:
    """
    Register a histogram metric.

    Args:
        name: The histogram metric name.
        unit: The histogram metric unit.
        description: The histogram metric description.

    Returns:
        The registered histogram metric.
    """
    self._histogram_metrics[name] = self.create_histogram(
        name=f"{self._metric_prefix}_{name}",
        unit=unit,
        description=description,
    )

record_histogram

record_histogram(metric: HistogramMetric | str, value: int | float, attributes: dict | None = None) -> None

Record the value for a specified histogram metric.

PARAMETER	DESCRIPTION
metric	The histogram metric name to record. TYPE: HistogramMetric | str
value	The value to record for the metric. TYPE: int | float
attributes	Additional metadata for the metric. TYPE: dict | None DEFAULT: None

Source code in packages/ragbits-core/src/ragbits/core/audit/metrics/base.py

def record_histogram(
    self, metric: HistogramMetric | str, value: int | float, attributes: dict | None = None
) -> None:
    """
    Record the value for a specified histogram metric.

    Args:
        metric: The histogram metric name to record.
        value: The value to record for the metric.
        attributes: Additional metadata for the metric.
    """
    if histogram_metric := HISTOGRAM_METRICS.get(metric):  # type: ignore
        metric_name = histogram_metric.name
        if metric_name not in self._histogram_metrics:
            self.register_histogram(
                name=metric_name,
                unit=histogram_metric.unit,
                description=histogram_metric.description,
            )
    else:
        metric_name = str(metric)
        if metric_name not in self._histogram_metrics:
            self.register_histogram(metric_name)

    self.record(
        metric=self._histogram_metrics[metric_name],
        value=value,
        attributes=attributes,
    )

ragbits.core.audit.metrics.otel.OtelMetricHandler

OtelMetricHandler(provider: MeterProvider | None = None, metric_prefix: str = 'ragbits')

Bases: MetricHandler[Histogram]

OpenTelemetry metric handler.

Initialize the OtelMetricHandler instance.

PARAMETER	DESCRIPTION
provider	The meter provider to use. TYPE: MeterProvider | None DEFAULT: None
metric_prefix	Prefix for all metric names. TYPE: str DEFAULT: 'ragbits'

Source code in packages/ragbits-core/src/ragbits/core/audit/metrics/otel.py

def __init__(self, provider: MeterProvider | None = None, metric_prefix: str = "ragbits") -> None:
    """
    Initialize the OtelMetricHandler instance.

    Args:
        provider: The meter provider to use.
        metric_prefix: Prefix for all metric names.
    """
    super().__init__(metric_prefix=metric_prefix)
    self._meter = get_meter(name=__name__, meter_provider=provider)

register_histogram

register_histogram(name: str, unit: str = '', description: str = '') -> None

Register a histogram metric.

PARAMETER	DESCRIPTION
name	The histogram metric name. TYPE: str
unit	The histogram metric unit. TYPE: str DEFAULT: ''
description	The histogram metric description. TYPE: str DEFAULT: ''

RETURNS	DESCRIPTION
None	The registered histogram metric.

Source code in packages/ragbits-core/src/ragbits/core/audit/metrics/base.py

def register_histogram(self, name: str, unit: str = "", description: str = "") -> None:
    """
    Register a histogram metric.

    Args:
        name: The histogram metric name.
        unit: The histogram metric unit.
        description: The histogram metric description.

    Returns:
        The registered histogram metric.
    """
    self._histogram_metrics[name] = self.create_histogram(
        name=f"{self._metric_prefix}_{name}",
        unit=unit,
        description=description,
    )

record_histogram

record_histogram(metric: HistogramMetric | str, value: int | float, attributes: dict | None = None) -> None

Record the value for a specified histogram metric.

PARAMETER	DESCRIPTION
metric	The histogram metric name to record. TYPE: HistogramMetric | str
value	The value to record for the metric. TYPE: int | float
attributes	Additional metadata for the metric. TYPE: dict | None DEFAULT: None

Source code in packages/ragbits-core/src/ragbits/core/audit/metrics/base.py

def record_histogram(
    self, metric: HistogramMetric | str, value: int | float, attributes: dict | None = None
) -> None:
    """
    Record the value for a specified histogram metric.

    Args:
        metric: The histogram metric name to record.
        value: The value to record for the metric.
        attributes: Additional metadata for the metric.
    """
    if histogram_metric := HISTOGRAM_METRICS.get(metric):  # type: ignore
        metric_name = histogram_metric.name
        if metric_name not in self._histogram_metrics:
            self.register_histogram(
                name=metric_name,
                unit=histogram_metric.unit,
                description=histogram_metric.description,
            )
    else:
        metric_name = str(metric)
        if metric_name not in self._histogram_metrics:
            self.register_histogram(metric_name)

    self.record(
        metric=self._histogram_metrics[metric_name],
        value=value,
        attributes=attributes,
    )

create_histogram

create_histogram(name: str, unit: str = '', description: str = '') -> Histogram

Create a histogram metric.

PARAMETER	DESCRIPTION
name	The histogram metric name. TYPE: str
unit	The histogram metric unit. TYPE: str DEFAULT: ''
description	The histogram metric description. TYPE: str DEFAULT: ''

RETURNS	DESCRIPTION
Histogram	The initialized histogram metric.

Source code in packages/ragbits-core/src/ragbits/core/audit/metrics/otel.py

def create_histogram(self, name: str, unit: str = "", description: str = "") -> Histogram:
    """
    Create a histogram metric.

    Args:
        name: The histogram metric name.
        unit: The histogram metric unit.
        description: The histogram metric description.

    Returns:
        The initialized histogram metric.
    """
    return self._meter.create_histogram(name=name, unit=unit, description=description)

record

record(metric: Histogram, value: int | float, attributes: dict | None = None) -> None

Record the value for a specified histogram metric.

PARAMETER	DESCRIPTION
metric	The histogram metric to record. TYPE: Histogram
value	The value to record for the metric. TYPE: int | float
attributes	Additional metadata for the metric. TYPE: dict | None DEFAULT: None

Source code in packages/ragbits-core/src/ragbits/core/audit/metrics/otel.py

def record(self, metric: Histogram, value: int | float, attributes: dict | None = None) -> None:  # noqa: PLR6301
    """
    Record the value for a specified histogram metric.

    Args:
        metric: The histogram metric to record.
        value: The value to record for the metric.
        attributes: Additional metadata for the metric.
    """
    metric.record(value, attributes=attributes)