archivebox.services.runner

Module Contents

Classes

CrawlRunner

Functions

_bus_name

_count_selected_hooks

_normalize_runtime_config

_emit_machine_config

ensure_background_runner

run_crawl

_run_binary

run_binary

_run_install

run_install

recover_orphaned_crawls

recover_orphaned_snapshots

run_pending_crawls

API

archivebox.services.runner._bus_name(prefix: str, identifier: str) → str[source]
archivebox.services.runner._count_selected_hooks(plugins: dict[str, abx_dl.models.Plugin], selected_plugins: list[str] | None) → int[source]
archivebox.services.runner._normalize_runtime_config(config: dict[str, Any]) → dict[str, Any][source]
async archivebox.services.runner._emit_machine_config(bus, *, config: dict[str, Any], derived_config: dict[str, Any]) → None[source]
archivebox.services.runner.ensure_background_runner(*, allow_under_pytest: bool = False) → bool[source]
class archivebox.services.runner.CrawlRunner(crawl, *, snapshot_ids: list[str] | None = None, selected_plugins: list[str] | None = None, process_discovered_snapshots_inline: bool = True)[source]

Initialization

MAX_CONCURRENT_SNAPSHOTS[source]

8

async run() → None[source]
async enqueue_snapshot(snapshot_id: str) → None[source]
async wait_for_snapshot_tasks() → None[source]
load_run_state() → list[str][source]
finalize_run_state() → None[source]
_create_live_ui() → archivebox.services.live_ui.LiveBusUI | None[source]
load_snapshot_payload(snapshot_id: str) → dict[str, Any][source]
async enqueue_discovered_snapshots_from_outputs(snapshot_payload: dict[str, Any]) → None[source]
async run_crawl_setup(snapshot_id: str) → None[source]
async run_crawl_cleanup(snapshot_id: str) → None[source]
async run_snapshot(snapshot_id: str) → None[source]
seal_snapshot_due_to_limit(snapshot_id: str) → None[source]
archivebox.services.runner.run_crawl(crawl_id: str, *, snapshot_ids: list[str] | None = None, selected_plugins: list[str] | None = None, process_discovered_snapshots_inline: bool = True) → None[source]
async archivebox.services.runner._run_binary(binary_id: str) → None[source]
archivebox.services.runner.run_binary(binary_id: str) → None[source]
async archivebox.services.runner._run_install(plugin_names: list[str] | None = None) → None[source]
archivebox.services.runner.run_install(*, plugin_names: list[str] | None = None) → None[source]
archivebox.services.runner.recover_orphaned_crawls() → int[source]
archivebox.services.runner.recover_orphaned_snapshots() → int[source]
archivebox.services.runner.run_pending_crawls(*, daemon: bool = False, crawl_id: str | None = None) → int[source]