Python API reference

from paperlab_space import run

def run(
    config: RunConfig,
    data: Dataset | MappedDataset | None = None,
    *,
    backend: Backend | None = None,
    on_progress: ProgressCallback | None = None,
    previous: RunResults | None = None,
    checkpoint: str | Path | None = None,
) -> RunResults

from paperlab_space import arun

async def arun(
    config: RunConfig,
    data: Dataset | MappedDataset | None = None,
    *,
    backend: Backend | None = None,
    on_progress: ProgressCallback | None = None,
    previous: RunResults | None = None,
    checkpoint: str | Path | None = None,
    sleep: Sleep = asyncio.sleep,
    clock: Clock = time.monotonic,
) -> RunResults

from paperlab_space import estimate

def estimate(
    config: RunConfig,
    data: Dataset | MappedDataset,
    *,
    backend: Backend | None = None,
    latency_s: float = 3.0,
) -> Estimate

from paperlab_space.runner import Estimate

Estimate(models: list[ModelEstimate])

from paperlab_space import Progress

Progress(
    model: str,
    done: int,
    failed: int,
    total: int,
    record_id: str,
    row: RowOutput | None = None,
)

from paperlab_space import LiteLLMBackend

LiteLLMBackend(
    api_keys: Mapping[str, str] | None = None,
    api_bases: Mapping[str, str] | None = None,
)

async def complete(
    model: ModelSpec,
    messages: Messages,
    *,
    seed: int | None,
    timeout: float,
) -> Completion

def price(model: ModelSpec) -> tuple[float, float] | None

from paperlab_space import RunConfig

def model_seed(model: ModelSpec) -> int | None

def from_dict(data: dict[str, Any]) -> RunConfig

def from_yaml(path: str | Path) -> RunConfig

def to_dict() -> dict[str, Any]

def to_yaml(path: str | Path | None = None) -> str

from paperlab_space import ModelSpec

from paperlab_space import MetricSpec

from paperlab_space import DatasetSpec

from paperlab_space import RetrySettings

from paperlab_space import ColumnMapping

from paperlab_space import PromptTemplate

PromptTemplate(user: str | None = None, /, **data: Any)

>>> PromptTemplate(user="Translate to English: {sentence}").render({"sentence": "สวัสดี"})
[{'role': 'user', 'content': 'Translate to English: สวัสดี'}]

def check(columns: Iterable[str]) -> None

def render(fields: Mapping[str, str | None]) -> list[dict[str, str]]

from paperlab_space import load_dataset

def load_dataset(path: str | Path, sheet: str | int | None = None) -> Dataset

from paperlab_space import list_sheets

def list_sheets(path: str | Path) -> list[str]

from paperlab_space import Dataset

Dataset(
    columns: list[str],
    rows: list[Row],
    source: str | None = None,
    sheet: str | None = None,
    sha256: str | None = None,
    encoding: str | None = None,
    warnings: list[str] = [],
)

def column(name: str) -> list[str | None]

def map(mapping: ColumnMapping) -> MappedDataset

def from_records(
    records: Sequence[dict[str, Any]],
    source: str | None = None,
) -> Dataset

from paperlab_space import MappedDataset

MappedDataset(
    records: list[Record],
    mapping: ColumnMapping,
    dataset: Dataset,
)

def from_dataset(dataset: Dataset, mapping: ColumnMapping) -> MappedDataset

def validate() -> ValidationReport

def preview(n: int = 5) -> list[dict[str, Any]]

def to_jsonl(path: str | Path | None = None) -> str

from paperlab_space.data import Record

Record(
    id: str,
    fields: dict[str, str | None],
    expected: list[str] | None,
    row_number: int,
)

def to_json() -> dict[str, Any]

from paperlab_space.data import ValidationReport

ValidationReport(issues: list[Issue], n_records: int)

from paperlab_space.data import Issue

Issue(
    level: Level,
    code: str,
    message: str,
    rows: list[int] = [],
    column: str | None = None,
)

from paperlab_space import RunResults

RunResults(
    config: RunConfig,
    data: MappedDataset,
    runs: dict[str, ModelRun],
    scores: dict[str, dict[str, MetricResult]] = {},
    started_at: str | None = None,
    finished_at: str | None = None,
    environment: dict[str, str] = ...,
)

def output(model: str, record_id: str) -> RowOutput

def to_dict() -> dict[str, Any]

def save_json(path: str | Path) -> None

def from_dict(data: dict[str, Any]) -> RunResults

def load_json(path: str | Path) -> RunResults

from paperlab_space import ModelRun

ModelRun(model: ModelSpec, rows: list[RowOutput])

from paperlab_space import RowOutput

RowOutput(
    record_id: str,
    output: str | None,
    error: str | None = None,
    attempts: int = 0,
    latency_s: float | None = None,
    input_tokens: int | None = None,
    output_tokens: int | None = None,
    cost_usd: float | None = None,
    model_version: str | None = None,
)

from paperlab_space import MetricResult

MetricResult(
    name: str,
    value: float | None,
    per_row: list[float | None] | None = None,
    extra: dict[str, float] = {},
    n_scored: int = 0,
    higher_is_better: bool = True,
)

from paperlab_space import score

def score(results: RunResults) -> dict[str, dict[str, MetricResult]]

from paperlab_space import available_metrics

def available_metrics() -> dict[str, type[Metric]]

from paperlab_space import register

def register(cls: type[Metric]) -> type[Metric]

from paperlab_space.metrics import Metric

def compute(predictions: Predictions, references: References) -> MetricResult

def result(value: float | None, **kwargs: Any) -> MetricResult

from paperlab_space.metrics import Normalizer

Normalizer(
    nfkc: bool = True,
    ignore_case: bool = True,
    trim_punctuation: bool = True,
    ignore_punctuation: bool = False,
    remove_articles: bool = False,
)

from paperlab_space.metrics import tokenize

def tokenize(text: str) -> list[str]

from paperlab_space.metrics import extract_label

def extract_label(text: str, labels: Sequence[str], normalize: Normalizer) -> str

from paperlab_space.export import write_all

def write_all(results: RunResults, out_dir: str | Path) -> dict[str, Path]

from paperlab_space.export import write_excel

def write_excel(results: RunResults, path: str | Path) -> Path

from paperlab_space.export import write_latex

def write_latex(results: RunResults, path: str | Path, **options: Any) -> Path

from paperlab_space.export import latex_table

def latex_table(
    results: RunResults,
    *,
    metrics: Sequence[str] | None = None,
    caption: str | None = None,
    label: str | None = 'tab:results',
    decimals: int = 1,
    percent: bool = True,
    bold_best: bool = True,
    floating: bool = True,
) -> str

from paperlab_space.export import write_repro

def write_repro(results: RunResults, path: str | Path) -> Path

from paperlab_space.export import reproducibility

def reproducibility(results: RunResults) -> dict[str, Any]

from paperlab_space.runner import Backend

async def complete(
    model: ModelSpec,
    messages: Messages,
    *,
    seed: int | None,
    timeout: float,
) -> Completion

def price(model: ModelSpec) -> tuple[float, float] | None

from paperlab_space.runner import Completion

Completion(
    text: str,
    input_tokens: int | None = None,
    output_tokens: int | None = None,
    cost_usd: float | None = None,
    model_version: str | None = None,
)

from paperlab_space.runner import RetryableError

RetryableError(message: str, retry_after: float | None = None)

from paperlab_space.runner import ProviderError

ProviderError(message: str, fatal: bool = False)

from paperlab_space import PaperLabError

from paperlab_space import ConfigError

from paperlab_space import DataError

from paperlab_space import TemplateError