archivebox.services.runner

Module Contents

Classes

CrawlRunner

Functions

_bus_name
_runner_short_id
_runner_label
_runner_console_line
_count_selected_hooks
_enable_requested_plugins
_is_nonfatal_setup_hook
_runner_task_context
_is_external_task_cancelled
_run_event_now
ensure_background_runner
run_crawl
_run_crawl_locked
_run_binary
run_binary
run_snapshot_maintenance
run_due_crawl
_run_due_crawl_locked
run_due_snapshot
_run_due_snapshot_locked
run_due_binary
_run_install
run_install
_first_due_id
_run_due_crawl_status
_run_due_snapshot_query
_run_due_snapshot_id
_run_due_binary
run_pending_crawls

API

archivebox.services.runner._bus_name(prefix: str, identifier: str) → str

archivebox.services.runner._runner_short_id(identifier) → str

archivebox.services.runner._runner_label(value: str, *, reserve: int) → str

archivebox.services.runner._runner_console_line(*, crawl=None, crawl_id=None, snapshot=None, status: str = 'STARTED') → None

archivebox.services.runner._count_selected_hooks(catalog: abx_dl.catalog.PluginCatalog, selected_plugins: list[str] | None) → int

archivebox.services.runner._enable_requested_plugins(config: dict[str, Any], plugins: abx_dl.catalog.PluginCatalog) → None

archivebox.services.runner._is_nonfatal_setup_hook(plugin_name: str, hook_name: str) → bool

archivebox.services.runner._runner_task_context() → contextvars.Context

archivebox.services.runner._is_external_task_cancelled(error: asyncio.CancelledError) → bool

async archivebox.services.runner._run_event_now(event, timeout: float | None = None)

archivebox.services.runner.ensure_background_runner() → bool

class archivebox.services.runner.CrawlRunner(crawl, *, snapshot_ids: list[str] | None = None, selected_plugins: list[str] | None = None, process_discovered_snapshots_inline: bool = True, show_progress: bool = True, interactive_interrupts: bool = False, config_overrides: dict[str, Any] | None = None)

Initialization

_request_abort_from_signal(_sig: signal.Signals) → None

async crawl_is_cancelled() → bool

async crawl_is_paused() → bool

async watch_for_cancelled_crawl(parent_event: abxbus.BaseEvent, *, poll_interval: float = 1.0) → None

property allow_maintenance_on_inactive_crawl: bool

Run explicitly targeted plugin work on already-sealed snapshots.

async run() → None

async enqueue_snapshot(snapshot_id: str, crawl_start_event: abx_dl.events.CrawlStartEvent | None = None) → None

async stop_snapshot_tasks() → None

async wait_for_snapshot_tasks(*, enqueue_projected: bool = True) → None

async heartbeat_active_leases() → None

async drain_snapshot_tasks() → None

async enqueue_pending_snapshots_from_projection() → None

load_run_state() → list[str]

create_initial_snapshots() → list

finalize_run_state() → None

_create_live_ui() → abx_dl.cli.LiveBusUI | None

load_snapshot_payload(snapshot_id: str) → dict[str, Any]

async enqueue_discovered_snapshots_from_outputs(snapshot_payload: dict[str, Any]) → None

async run_crawl(root_snapshot_id: str, snapshot_ids: list[str]) → None

async run_snapshot(snapshot_id: str, crawl_start_event: abx_dl.events.CrawlStartEvent | None = None) → None

seal_snapshot_due_to_limit(snapshot_id: str) → None

archivebox.services.runner.run_crawl(crawl_id: str, *, snapshot_ids: list[str] | None = None, selected_plugins: list[str] | None = None, process_discovered_snapshots_inline: bool = True, show_progress: bool = True, interactive_interrupts: bool = False, config_overrides: dict[str, Any] | None = None) → None

archivebox.services.runner._run_crawl_locked(crawl_id: str, *, snapshot_ids: list[str] | None = None, selected_plugins: list[str] | None = None, process_discovered_snapshots_inline: bool = True, show_progress: bool = True, interactive_interrupts: bool = False, config_overrides: dict[str, Any] | None = None) → None

async archivebox.services.runner._run_binary(binary_id: str) → None

archivebox.services.runner.run_binary(binary_id: str) → None

archivebox.services.runner.run_snapshot_maintenance(snapshot_id: str, *, output_dir: pathlib.Path | None = None) → bool

archivebox.services.runner.run_due_crawl(crawl, *, lock_seconds: int, interactive_interrupts: bool = False) → bool

archivebox.services.runner._run_due_crawl_locked(crawl, *, lock_seconds: int, interactive_interrupts: bool = False) → bool

archivebox.services.runner.run_due_snapshot(snapshot, *, lock_seconds: int, interactive_interrupts: bool = False, runtime_config=None) → bool

archivebox.services.runner._run_due_snapshot_locked(snapshot, *, lock_seconds: int, interactive_interrupts: bool = False, runtime_config=None) → bool

archivebox.services.runner.run_due_binary(binary, *, lock_seconds: int) → bool

async archivebox.services.runner._run_install(plugin_names: list[str] | None = None) → None

archivebox.services.runner.run_install(*, plugin_names: list[str] | None = None) → None

archivebox.services.runner._first_due_id(queryset)

archivebox.services.runner._run_due_crawl_status(status: str, *, crawl_id: str | None, lock_seconds: int, interactive_interrupts: bool) → bool

archivebox.services.runner._run_due_snapshot_query(queryset, *, lock_seconds: int, interactive_interrupts: bool, runtime_config) → bool

archivebox.services.runner._run_due_snapshot_id(snapshot_id, *, lock_seconds: int, interactive_interrupts: bool, runtime_config) → bool

archivebox.services.runner._run_due_binary() → bool

archivebox.services.runner.run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None, maintenance_only: bool = False, interactive_interrupts: bool = False) → int