archivebox.workers.worker

Module Contents

Classes

WorkerType

OrchestratorWorker

FileSystemWorker

CrawlWorker

SnapshotWorker

ArchiveResultWorker

Functions

get_worker_type

Data

WORKER_TYPES

API

class archivebox.workers.worker.WorkerType[source]
name: ClassVar[str][source]

None

listens_to: ClassVar[str][source]

None

outputs: ClassVar[list[str]][source]

None

poll_interval: ClassVar[int][source]

1

event_queue() → django.db.models.QuerySet[workers.models.Event][source]
classmethod fork(wait_for_first_event=False, exit_on_idle=True) → workers.models.Process[source]
processes() → django.db.models.QuerySet[workers.models.Process][source]
classmethod run(wait_for_first_event=False, exit_on_idle=True)[source]
classmethod process_next_event() → Iterable[workers.models.EventDict][source]
classmethod process_idle_tick() → Iterable[workers.models.EventDict][source]
classmethod receive(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
static on_IDLE() → Iterable[workers.models.EventDict][source]
static mark_event_claimed(event: workers.models.Event)[source]
static mark_event_succeeded(event: workers.models.Event, output_events: Iterable[workers.models.EventDict])[source]
static mark_event_failed(event: workers.models.Event, output_events: Iterable[workers.models.EventDict] = (), error: BaseException | None = None)[source]
class archivebox.workers.worker.OrchestratorWorker[source]

Bases: archivebox.workers.worker.WorkerType

name[source]

‘orchestrator’

listens_to[source]

‘PROC_’

outputs[source]

[‘PROC_’]

static on_PROC_IDLE() → Iterable[workers.models.EventDict][source]
static on_PROC_LAUNCH(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
static on_PROC_EXIT(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
static on_PROC_KILL(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
class archivebox.workers.worker.FileSystemWorker[source]

Bases: archivebox.workers.worker.WorkerType

name[source]

‘filesystem’

listens_to[source]

‘FS_’

outputs[source]

[‘FS_’]

static on_FS_IDLE(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
static on_FS_WRITE(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
static on_FS_APPEND(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
static on_FS_DELETE(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
static on_FS_RSYNC(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
class archivebox.workers.worker.CrawlWorker[source]

Bases: archivebox.workers.worker.WorkerType

name[source]

‘crawl’

listens_to[source]

‘CRAWL_’

outputs[source]

[‘CRAWL_’, ‘FS_’, ‘SNAPSHOT_’]

static on_CRAWL_IDLE(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
static on_CRAWL_CREATE(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
static on_CRAWL_UPDATE(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
static on_CRAWL_UPDATED(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
static on_CRAWL_SEAL(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
static on_CRAWL_START(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
class archivebox.workers.worker.SnapshotWorker[source]

Bases: archivebox.workers.worker.WorkerType

name[source]

‘snapshot’

listens_to[source]

‘SNAPSHOT_’

outputs[source]

[‘SNAPSHOT_’, ‘FS_’]

static on_SNAPSHOT_IDLE(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
static on_SNAPSHOT_CREATE(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
static on_SNAPSHOT_SEAL(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
static on_SNAPSHOT_START(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
class archivebox.workers.worker.ArchiveResultWorker[source]

Bases: archivebox.workers.worker.WorkerType

name[source]

‘archiveresult’

listens_to[source]

‘ARCHIVERESULT_’

outputs[source]

[‘ARCHIVERESULT_’, ‘FS_’]

static on_ARCHIVERESULT_UPDATE(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
static on_ARCHIVERESULT_UPDATED(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
static on_ARCHIVERESULT_CREATE(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
static on_ARCHIVERESULT_SEAL(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
static on_ARCHIVERESULT_START(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
static on_ARCHIVERESULT_IDLE(event: workers.models.Event) → Iterable[workers.models.EventDict][source]
archivebox.workers.worker.WORKER_TYPES[source]

None

archivebox.workers.worker.get_worker_type(name: str) → Type[archivebox.workers.worker.WorkerType][source]