palimpsest-artifact

WARC++ serialization: records, capture groups, reader/writer. Content-addressed outputs compatible with ISO 28500.

RecordType

#![allow(unused)]
fn main() {
#[non_exhaustive]
pub enum RecordType {
    // Standard (ISO 28500)
    Warcinfo, Request, Response, Resource, Metadata,
    // Palimpsest extensions
    Envelope, DomSnapshot, ResourceGraph, Timing,
}

impl RecordType {
    pub fn is_standard(&self) -> bool;
}
}

WarcRecord

#![allow(unused)]
fn main() {
pub struct WarcRecord {
    pub record_type: RecordType,
    pub record_id: RecordId,
    pub content_hash: ContentHash,
    pub content_type: String,
    pub content_length: u64,
    pub target_uri: Option<String>,
    pub payload: Bytes,
}

impl WarcRecord {
    pub fn new(record_type: RecordType, content_type: String, payload: Bytes) -> Self;
    pub fn verify_integrity(&self) -> bool;
}
}

RecordId

#![allow(unused)]
fn main() {
pub struct RecordId(String);

impl RecordId {
    pub fn from_content(content_hash: &ContentHash, record_type: &RecordType) -> Self;
    pub fn as_str(&self) -> &str;
}
}

Deterministic — derived from content hash + record type, not random UUID.

CaptureGroup

#![allow(unused)]
fn main() {
pub struct CaptureGroup {
    pub group_hash: ContentHash,
    pub url: Url,
    pub captured_at: CaptureInstant,
    pub crawl_context: CrawlContextId,
    pub envelope: WarcRecord,
    pub request: WarcRecord,
    pub response: WarcRecord,
    pub dom_snapshot: Option<DomSnapshot>,
    pub resource_graph: Option<ResourceGraph>,
    pub timing: Option<TimingBreakdown>,
}
}

Built via CaptureGroupBuilder (fluent builder with required fields validation).

WARC Writer/Reader

#![allow(unused)]
fn main() {
pub async fn write_warc_file(path: &Path, records: &[WarcRecord]) -> Result<(), WarcWriteError>;
pub fn parse_warc_records(data: &[u8]) -> Result<Vec<WarcRecord>, WarcWriteError>;
}

Key Invariant

All record IDs and content hashes are deterministic. Same content = same hash = same record ID.