From fbe75307f27f5c8c88548dfe0c01535731cb8f81 Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Thu, 9 Jul 2026 07:58:52 -0700 Subject: [PATCH 01/37] Add manifest-first streaming tar writer for tile extracts The tar format makes every output offset computable before writing a single byte, given a manifest of (GraphId, size) pairs. ExtractLayout is that pure computation; write_extract_from_source streams the archive into any Write in a single pass (no Seek, no temp files), and write_extract drives it from a source TarballTileProvider. The canonical archive order is ascending by (level, tile index); note that GraphId's Ord deliberately orders levels in reverse, so the writer sorts its own manifest and never relies on caller order. Output is byte-deterministic: fixed ustar header metadata (mtime 0, uid/gid 0, mode 644) and spec-compliant entry paths. Also adds the minimal read accessors this needs: tile sizes from the tarball provider's index, and offset/size/graph ID getters on TileIndexBinEntry. Also add the extract module, which this writer is the first part of. Co-Authored-By: Claude Fable 5 --- valhalla-graphtile/src/extract/mod.rs | 25 + valhalla-graphtile/src/extract/writer.rs | 691 ++++++++++++++++++ valhalla-graphtile/src/lib.rs | 1 + .../src/tile_provider/tarball.rs | 32 +- 4 files changed, 748 insertions(+), 1 deletion(-) create mode 100644 valhalla-graphtile/src/extract/mod.rs create mode 100644 valhalla-graphtile/src/extract/writer.rs diff --git a/valhalla-graphtile/src/extract/mod.rs b/valhalla-graphtile/src/extract/mod.rs new file mode 100644 index 0000000..013e0f4 --- /dev/null +++ b/valhalla-graphtile/src/extract/mod.rs @@ -0,0 +1,25 @@ +//! # Offline tile extract generation +//! +//! This module writes *sub-extracts* of a full Valhalla tile extract: +//! smaller, fully spec-compliant tarballs which contain a subset of the source tiles. +//! The output is interchangeable with `valhalla_build_extract` output, +//! so it is consumable by both stock Valhalla +//! and [`TarballTileProvider`](crate::tile_provider::TarballTileProvider). +//! +//! # Writing +//! +//! The writer streams the archive in a single pass: +//! an `index.bin` entry first +//! (see [`serialize_index_bin`](crate::tile_provider::serialize_index_bin)), +//! followed by the graph tiles sorted ascending by (level, tile index) +//! (note that this differs from the `Ord` impl of [`GraphId`](crate::GraphId), +//! which deliberately orders levels in reverse). +//! Output is deterministic: +//! the same source extract and tile set always produce byte-identical output. +//! See [`ExtractLayout`] and [`write_extract_from_source`] for the details. + +mod writer; + +pub use writer::{ + ExtractLayout, ExtractWriteError, TileBytesSource, write_extract, write_extract_from_source, +}; diff --git a/valhalla-graphtile/src/extract/writer.rs b/valhalla-graphtile/src/extract/writer.rs new file mode 100644 index 0000000..f22f0c6 --- /dev/null +++ b/valhalla-graphtile/src/extract/writer.rs @@ -0,0 +1,691 @@ +//! Manifest-first, single-pass streaming tar writer for tile extracts. +//! +//! The tar format makes every output offset computable *before writing a single byte*, +//! given a manifest of `(GraphId, size)` pairs +//! (which we always have, because the source provider's tile index stores sizes): +//! +//! - `index.bin` is one [`TileIndexBinEntry`] per tile, +//! padded to a 512-byte boundary, preceded by its 512-byte tar header. +//! - Each tile contributes a 512-byte tar header plus its payload padded to 512 bytes. +//! +//! [`ExtractLayout`] is that pure computation; +//! [`write_extract_from_source`] streams the corresponding archive into any [`Write`], +//! with no `Seek`, no temp files, and no buffering of the whole archive. +//! +//! # Archive layout +//! +//! 1. `index.bin` is always the first entry. +//! Valhalla C++ treats it as optional for local tarballs +//! but requires it as the first entry for remotely-served tarballs, +//! and Valinor's own reader requires it unconditionally. +//! Writing it first satisfies every consumer +//! and keeps HTTP range-request serving possible. +//! 2. Graph tiles follow, sorted ascending by (level, tile index), +//! with spec-compliant entry paths (`level/xxx/yyy….gph`) +//! so that index-less consumers scanning the tar can still resolve them. +//! Note that this canonical order differs from the `Ord` impl of [`GraphId`], +//! which deliberately orders levels in reverse. +//! 3. The archive ends with the standard two-block end-of-archive marker. +//! +//! Output is byte-deterministic: +//! tar header metadata is fixed (mtime 0, uid/gid 0, mode 644, ustar format), +//! and the writer sorts its own manifest, so caller order never leaks into the output. + +use crate::GraphId; +use crate::graph_id::InvalidGraphIdError; +use crate::tile_provider::{ + GraphTileProviderError, InvalidIndexBinError, TarballTileProvider, TileIndexBinEntry, + serialize_index_bin, +}; +use std::io::Write; +use tar::{EntryType, Header}; +use thiserror::Error; + +/// The tar block size; every entry is a one-block header plus a payload padded to a block. +const TAR_BLOCK_SIZE: u64 = 512; + +/// A zeroed block (of [`TAR_BLOCK_SIZE`] bytes) +/// for payload padding and the end-of-archive marker. +const ZERO_BLOCK: [u8; 512] = [0; 512]; + +/// The entry name of the tile index (always the first entry in the archive). +const INDEX_ENTRY_NAME: &str = "index.bin"; + +/// An error while writing a tile extract. +#[derive(Debug, Error)] +pub enum ExtractWriteError { + /// An extract containing no tiles is meaningless + /// (and its `index.bin` would be malformed). + #[error("the extract manifest contains no tiles")] + EmptyManifest, + /// The manifest contains the same tile more than once. + #[error("the extract manifest contains tile {0} more than once")] + DuplicateTile(GraphId), + /// A graph ID in the manifest cannot be converted into a tar entry path. + #[error("invalid graph ID: {0}")] + InvalidGraphId(#[from] InvalidGraphIdError), + /// The byte source produced a tile whose length differs from the manifest size. + /// + /// Sizes must match exactly: + /// every downstream offset was computed from the manifest before any bytes were written, + /// so a mismatch would silently corrupt the rest of the archive. + #[error( + "the byte source produced {actual} bytes for tile {graph_id}, but the manifest size is {expected}" + )] + SizeMismatch { + /// The base graph ID of the offending tile. + graph_id: GraphId, + /// The size recorded in the manifest. + expected: u32, + /// The length of the bytes actually produced by the source. + actual: usize, + }, + /// The manifest contains an ID which cannot be indexed, + /// or a tile offset that `index.bin` cannot record. + #[error("invalid index.bin: {0}")] + InvalidIndexBin(#[from] InvalidIndexBinError), + /// The byte source failed to produce a tile. + #[error("tile provider error: {0}")] + Provider(#[from] GraphTileProviderError), + /// Writing to the destination failed. + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), +} + +/// A source of tile bytes for extract writing. +/// +/// The writer core ([`write_extract_from_source`]) is generic over this trait, +/// so any byte lookup can drive it: +/// a memory-mapped tarball in production, or an in-memory map in tests. +/// +/// The callback style (rather than returning a slice) +/// lets implementations hand out bytes borrowed from per-call state, +/// such as a tile pointer into a memory map. +pub trait TileBytesSource { + /// Calls `process` with the raw bytes of the tile containing the given graph ID. + /// + /// # Errors + /// + /// This fails if the tile does not exist in the source, + /// or if fetching its bytes fails. + fn with_tile_bytes( + &self, + graph_id: GraphId, + process: F, + ) -> Result + where + F: FnOnce(&[u8]) -> T; +} + +impl TileBytesSource for TarballTileProvider { + #[inline] + fn with_tile_bytes( + &self, + graph_id: GraphId, + process: F, + ) -> Result + where + F: FnOnce(&[u8]) -> T, + { + let tile_pointer = self.get_pointer_for_tile_containing(graph_id)?; + // SAFETY: We assume that the tile pointer is live and points to an actual graph tile location in the tarball. + // That is to say, we assume the tile is well-formed (correct index) and that the underlying memory map + // has not changed in a way that would render the tile invalid. + let tile_bytes = unsafe { tile_pointer.as_tile_bytes() }; + Ok(process(tile_bytes)) + } +} + +/// The precomputed layout of an extract tarball. +/// +/// This is the pure "manifest → offsets" half of the writer: +/// given `(GraphId, size)` pairs, +/// it computes the byte offset of every tile payload in the output archive +/// and the exact total archive length, +/// without performing any I/O. +/// +/// The layout sorts the manifest into the canonical archive order +/// (ascending by level, then tile index), +/// so the index entries are in the exact order +/// their tiles appear in the archive. +#[derive(Debug)] +pub struct ExtractLayout { + /// One index entry per tile, in canonical archive order. + entries: Vec, + /// The total length of the archive in bytes, including the end-of-archive marker. + total_len: u64, +} + +impl ExtractLayout { + /// Computes the archive layout for a manifest of `(GraphId, size)` pairs. + /// + /// The manifest may be in any order (the layout sorts a copy into canonical order), + /// but every graph ID must be a *base* tile ID + /// (see [`GraphId::tile_base_id`]), + /// and each tile may appear at most once. + /// + /// # Errors + /// + /// This fails if the manifest is empty, contains duplicate tiles, + /// or contains an ID which is not a valid base tile ID. + pub fn from_manifest(manifest: &[(GraphId, u32)]) -> Result { + if manifest.is_empty() { + return Err(ExtractWriteError::EmptyManifest); + } + + let mut sorted = manifest.to_vec(); + sorted.sort_unstable_by_key(|(graph_id, _)| (graph_id.level(), graph_id.tile_id())); + + // The index entry for a tile records the offset of its *payload*, + // so the first tile payload lands one header block past the index entry: + // the index header block, the padded index payload, and the tile's own header block. + let index_len = (size_of::() * sorted.len()) as u64; + let mut header_offset = TAR_BLOCK_SIZE + index_len.next_multiple_of(TAR_BLOCK_SIZE); + + let mut entries = Vec::with_capacity(sorted.len()); + for (i, (graph_id, size)) in sorted.iter().enumerate() { + // Equal (level, tile index) pairs are adjacent after the sort. + if i > 0 && sorted[i - 1].0 == *graph_id { + return Err(ExtractWriteError::DuplicateTile(*graph_id)); + } + + let payload_offset = header_offset + TAR_BLOCK_SIZE; + entries.push(TileIndexBinEntry::new(*graph_id, payload_offset, *size)?); + header_offset = payload_offset + u64::from(*size).next_multiple_of(TAR_BLOCK_SIZE); + } + + Ok(Self { + entries, + // The archive ends with two zero blocks (the standard end-of-archive marker). + total_len: header_offset + 2 * TAR_BLOCK_SIZE, + }) + } + + /// The index entries of the archive, in canonical archive order + /// (ascending by level, then tile index). + /// + /// Serializing these with + /// [`serialize_index_bin`](crate::tile_provider::serialize_index_bin) + /// yields exactly the `index.bin` payload of the archive. + #[inline] + pub fn index_entries(&self) -> &[TileIndexBinEntry] { + &self.entries + } + + /// The exact total length of the archive in bytes, + /// including the end-of-archive marker. + /// + /// This is always a multiple of 512, + /// and is known before a single byte is written, + /// which makes it possible to set a `Content-Length` up front + /// when streaming an extract over HTTP. + #[inline] + pub fn total_len(&self) -> u64 { + self.total_len + } +} + +/// Writes a tile extract sourced from an existing tarball. +/// +/// This is a convenience wrapper over [`write_extract_from_source`]: +/// the manifest is built by resolving each ID in `tile_ids` +/// (normalized to its base tile ID) +/// against the source provider's tile index, +/// and tile bytes are read from the provider's memory map. +/// See [`write_extract_from_source`] for the full output format contract. +/// +/// Returns the total number of bytes written. +/// +/// # Errors +/// +/// This fails if `tile_ids` is empty, contains duplicate tiles, +/// or references a tile which is not present in the source tarball. +/// It also fails if writing to the destination fails. +pub fn write_extract( + provider: &TarballTileProvider, + tile_ids: &[GraphId], + dest: W, +) -> Result { + let manifest = tile_ids + .iter() + .map(|graph_id| { + let base_id = graph_id.tile_base_id(); + provider + .tile_size(base_id) + .map(|size| (base_id, size)) + .ok_or(GraphTileProviderError::TileDoesNotExist) + }) + .collect::, _>>()?; + + write_extract_from_source(&manifest, provider, dest) +} + +/// Writes a tile extract as a tar archive into the destination. +/// +/// The archive is streamed sequentially in a single pass +/// (no `Seek`, no temp files, no buffering of the whole archive): +/// the `index.bin` entry first, +/// then one entry per manifest tile in canonical archive order +/// (ascending by level, then tile index), +/// and finally the two-block end-of-archive marker. +/// The manifest may be in any order; the writer sorts its own copy. +/// +/// Output is byte-deterministic: +/// the same manifest and tile bytes always produce byte-identical output. +/// +/// Returns the total number of bytes written, +/// which always equals [`ExtractLayout::total_len`] for the same manifest. +/// +/// # Errors +/// +/// This fails if the layout cannot be computed from the manifest +/// (see [`ExtractLayout::from_manifest`]), +/// if the source fails to produce a tile, +/// if a tile's length differs from its manifest size, +/// or if writing to the destination fails. +/// +/// The destination contents are unspecified after an error; +/// streaming destinations cannot roll back partially written entries. +pub fn write_extract_from_source( + manifest: &[(GraphId, u32)], + source: &S, + mut dest: W, +) -> Result +where + S: TileBytesSource, + W: Write, +{ + let layout = ExtractLayout::from_manifest(manifest)?; + + let index_bytes = serialize_index_bin(layout.index_entries())?; + write_entry(&mut dest, INDEX_ENTRY_NAME, index_bytes)?; + + for entry in layout.index_entries() { + // Infallible in practice: the layout was constructed from valid base tile IDs. + let graph_id = entry.graph_id()?; + let path = tar_entry_path(graph_id)?; + let expected = entry.size(); + + source.with_tile_bytes(graph_id, |tile_bytes| { + if tile_bytes.len() != expected as usize { + return Err(ExtractWriteError::SizeMismatch { + graph_id, + expected, + actual: tile_bytes.len(), + }); + } + write_entry(&mut dest, &path, tile_bytes).map_err(ExtractWriteError::from) + })??; + } + + // The standard end-of-archive marker: two zero blocks. + dest.write_all(&ZERO_BLOCK)?; + dest.write_all(&ZERO_BLOCK)?; + dest.flush()?; + + Ok(layout.total_len()) +} + +/// Writes a single tar entry: a 512-byte header, the payload, and zero padding to a block boundary. +fn write_entry(dest: &mut W, path: &str, payload: &[u8]) -> std::io::Result<()> { + // Determinism requires every header field to be fixed: + // ustar format, mtime 0, uid/gid 0 (with empty user/group names), mode 644. + let mut header = Header::new_ustar(); + header.set_entry_type(EntryType::Regular); + header.set_path(path)?; + header.set_size(payload.len() as u64); + header.set_mode(0o644); + header.set_uid(0); + header.set_gid(0); + header.set_mtime(0); + header.set_cksum(); + + dest.write_all(header.as_bytes())?; + dest.write_all(payload)?; + + let padding = payload.len().next_multiple_of(ZERO_BLOCK.len()) - payload.len(); + dest.write_all(&ZERO_BLOCK[..padding]) +} + +/// Constructs the spec-compliant tar entry path for a tile. +/// +/// The first component is the bare level digit; +/// the remaining components are the tile index, +/// zero-padded to the level's width and split into 3-digit groups +/// (e.g. `2/000/762/485.gph`). +/// Tar entry names always use `/` separators, regardless of platform. +fn tar_entry_path(graph_id: GraphId) -> Result { + let path = graph_id.file_path("gph")?; + let components: Vec<_> = path + .iter() + .map(|component| component.to_string_lossy()) + .collect(); + Ok(components.join("/")) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tile_provider::parse_index_bin; + use std::collections::HashMap; + use std::io::Read; + use tar::Archive; + + /// Helper to construct a base tile [`GraphId`]. + fn tile(level: u8, tile_id: u64) -> GraphId { + GraphId::try_from_components(level, tile_id, 0).expect("valid tile ID in test") + } + + /// An in-memory byte source for driving the writer without any I/O. + struct MemorySource(HashMap>); + + impl MemorySource { + fn from_manifest(manifest: &[(GraphId, u32)]) -> Self { + let tiles = manifest + .iter() + .map(|(graph_id, size)| { + // Distinctive fill bytes so tiles cannot be confused with each other + // (or with zero padding). + let fill = + 0x11 * (1 + u8::try_from(graph_id.tile_id() % 13).expect("13 fits in u8")); + (*graph_id, vec![fill; *size as usize]) + }) + .collect(); + Self(tiles) + } + } + + impl TileBytesSource for MemorySource { + fn with_tile_bytes( + &self, + graph_id: GraphId, + process: F, + ) -> Result + where + F: FnOnce(&[u8]) -> T, + { + self.0 + .get(&graph_id.tile_base_id()) + .map(|tile_bytes| process(tile_bytes)) + .ok_or(GraphTileProviderError::TileDoesNotExist) + } + } + + /// A manifest which is deliberately *not* in canonical order + /// (and not in `GraphId` order either), + /// with sizes exercising all padding cases: + /// an exact block multiple, a one-byte remainder, and a one-byte shortfall. + fn scrambled_manifest() -> Vec<(GraphId, u32)> { + vec![ + (tile(2, 762_485), 100), + (tile(0, 3015), 512), + (tile(2, 762_484), 1024), + (tile(1, 47_701), 513), + (tile(1, 47_700), 511), + ] + } + + #[test] + fn test_tar_entry_path() { + // Levels 0 and 1 pad the tile index to 6 digits (two components); + // level 2 pads to 9 digits (three components). + assert_eq!(tar_entry_path(tile(0, 3015)).unwrap(), "0/003/015.gph"); + assert_eq!(tar_entry_path(tile(0, 49)).unwrap(), "0/000/049.gph"); + assert_eq!(tar_entry_path(tile(1, 47_701)).unwrap(), "1/047/701.gph"); + assert_eq!(tar_entry_path(tile(1, 0)).unwrap(), "1/000/000.gph"); + assert_eq!( + tar_entry_path(tile(2, 762_485)).unwrap(), + "2/000/762/485.gph" + ); + assert_eq!(tar_entry_path(tile(2, 2)).unwrap(), "2/000/000/002.gph"); + assert_eq!( + tar_entry_path(tile(2, 1_036_799)).unwrap(), + "2/001/036/799.gph" + ); + } + + #[test] + fn test_layout_offsets() { + let layout = ExtractLayout::from_manifest(&scrambled_manifest()) + .expect("the manifest is well-formed"); + + // 5 tiles × 16 bytes = 80 bytes of index, padded to one block; + // with the index header, the first tile header lands at 1024. + // + // All offsets were computed by hand from the layout rules: + // each tile is a 512-byte header plus its payload padded to 512. + let expected = [ + // (graph_id, payload offset, size), in canonical (level, tile index) order + (tile(0, 3015), 1536, 512), // exactly one block; no padding + (tile(1, 47_700), 2560, 511), // padded to one block + (tile(1, 47_701), 3584, 513), // padded to two blocks + (tile(2, 762_484), 5120, 1024), // exactly two blocks; no padding + (tile(2, 762_485), 6656, 100), // padded to one block + ]; + + assert_eq!(layout.index_entries().len(), expected.len()); + for (entry, (graph_id, offset, size)) in layout.index_entries().iter().zip(expected) { + assert_eq!(entry.graph_id().unwrap(), graph_id); + assert_eq!(entry.offset(), offset); + assert_eq!(entry.size(), size); + } + + // Last tile payload ends at 6656 + 512 (padded), plus the 1024-byte end-of-archive marker. + assert_eq!(layout.total_len(), 7168 + 1024); + assert!(layout.total_len().is_multiple_of(512)); + } + + #[test] + fn test_layout_rejects_empty_manifest() { + assert!(matches!( + ExtractLayout::from_manifest(&[]), + Err(ExtractWriteError::EmptyManifest) + )); + } + + #[test] + fn test_layout_rejects_duplicate_tiles() { + let manifest = [(tile(1, 47_701), 100), (tile(1, 47_701), 100)]; + assert!(matches!( + ExtractLayout::from_manifest(&manifest), + Err(ExtractWriteError::DuplicateTile(graph_id)) if graph_id == tile(1, 47_701) + )); + } + + #[test] + fn test_layout_rejects_non_base_tile_ids() { + let with_feature_index = + GraphId::try_from_components(1, 47_701, 7).expect("valid graph ID in test"); + assert!(matches!( + ExtractLayout::from_manifest(&[(with_feature_index, 100)]), + Err(ExtractWriteError::InvalidIndexBin( + InvalidIndexBinError::FeatureIndex(_) + )) + )); + } + + #[test] + fn test_write_rejects_size_mismatch() { + let manifest = [(tile(0, 3015), 100)]; + // The source produces 99 bytes where the manifest promises 100. + let source = MemorySource::from_manifest(&[(tile(0, 3015), 99)]); + + let result = write_extract_from_source(&manifest, &source, Vec::new()); + assert!(matches!( + result, + Err(ExtractWriteError::SizeMismatch { + expected: 100, + actual: 99, + .. + }) + )); + } + + #[test] + fn test_write_rejects_missing_tiles() { + let manifest = [(tile(0, 3015), 100)]; + let source = MemorySource(HashMap::new()); + + let result = write_extract_from_source(&manifest, &source, Vec::new()); + assert!(matches!( + result, + Err(ExtractWriteError::Provider( + GraphTileProviderError::TileDoesNotExist + )) + )); + } + + #[test] + fn test_archive_invariants() { + let manifest = scrambled_manifest(); + let source = MemorySource::from_manifest(&manifest); + + let mut archive_bytes = Vec::new(); + let written = write_extract_from_source(&manifest, &source, &mut archive_bytes) + .expect("the manifest and source are well-formed"); + + assert_eq!(written, archive_bytes.len() as u64); + assert!(archive_bytes.len().is_multiple_of(512)); + + // The archive must list index.bin first, + // then the tiles in canonical (level, tile index) order under spec-compliant paths. + let mut archive = Archive::new(archive_bytes.as_slice()); + let entry_paths: Vec = archive + .entries() + .expect("the archive is readable") + .map(|entry| { + entry + .expect("every entry is readable") + .path() + .expect("every entry path is valid") + .to_string_lossy() + .into_owned() + }) + .collect(); + assert_eq!( + entry_paths, + vec![ + "index.bin", + "0/003/015.gph", + "1/047/700.gph", + "1/047/701.gph", + "2/000/762/484.gph", + "2/000/762/485.gph", + ] + ); + + // The index payload starts right after the index header + // and describes every tile payload in the archive. + let mut archive = Archive::new(archive_bytes.as_slice()); + let mut index_bytes = Vec::new(); + archive + .entries() + .expect("the archive is readable") + .next() + .expect("the archive has entries") + .expect("the index entry is readable") + .read_to_end(&mut index_bytes) + .expect("the index payload is readable"); + + let index_entries = parse_index_bin(&index_bytes).expect("the index parses"); + assert_eq!(index_entries.len(), manifest.len()); + + for index_entry in index_entries { + let offset = index_entry.offset(); + assert!(offset > 0); + assert!(offset.is_multiple_of(512)); + + // The payload at the indexed offset is byte-identical to the source tile. + let graph_id = index_entry.graph_id().expect("the entry ID is valid"); + let start = usize::try_from(offset).expect("the offset fits in memory"); + let payload = &archive_bytes[start..start + index_entry.size() as usize]; + source + .with_tile_bytes(graph_id, |tile_bytes| assert_eq!(payload, tile_bytes)) + .expect("the tile is in the source"); + } + } + + #[test] + fn test_determinism() { + let manifest = scrambled_manifest(); + let source = MemorySource::from_manifest(&manifest); + + let mut first = Vec::new(); + write_extract_from_source(&manifest, &source, &mut first).expect("first write succeeds"); + + let mut second = Vec::new(); + // A reordered manifest must not change the output either: + // the writer sorts its own copy. + let mut reversed = manifest.clone(); + reversed.reverse(); + write_extract_from_source(&reversed, &source, &mut second).expect("second write succeeds"); + + assert_eq!(first, second); + } + + // Tragically, memory maps are not supported by Miri! + #[cfg(not(miri))] + #[test] + fn test_andorra_round_trip() { + use std::path::PathBuf; + + let fixture_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("fixtures") + .join("andorra-tiles.tar"); + let source_provider = TarballTileProvider::new_readonly(fixture_path) + .expect("Unable to init the source tile provider"); + + // A hand-picked subset of the fixture, spanning all three standard levels, + // deliberately not in canonical order, + // and with one non-zero feature index to test base ID normalization. + let tile_ids = [ + tile(2, 763_926), + tile(0, 3015), + GraphId::try_from_components(1, 47_701, 42).expect("valid graph ID in test"), + tile(2, 762_485), + ]; + // A fixture tile which is *not* part of the subset. + let excluded = tile(2, 763_925); + assert!(source_provider.tile_size(excluded).is_some()); + + let mut archive_bytes = Vec::new(); + let written = write_extract(&source_provider, &tile_ids, &mut archive_bytes) + .expect("the extract writes cleanly"); + assert_eq!(written, archive_bytes.len() as u64); + + // Determinism on real data: a second run is byte-identical. + let mut second = Vec::new(); + write_extract(&source_provider, &tile_ids, &mut second).expect("second write succeeds"); + assert_eq!(archive_bytes, second); + + // Round-trip: Valinor's own reader opens the extract, + // and every tile is byte-identical to the source. + let extract_path = std::env::temp_dir().join(format!( + "valinor-extract-round-trip-{}.tar", + std::process::id() + )); + std::fs::write(&extract_path, &archive_bytes).expect("the extract is written to disk"); + + let extract_provider = TarballTileProvider::new_readonly(&extract_path) + .expect("the generated extract opens cleanly"); + + for graph_id in tile_ids { + let base_id = graph_id.tile_base_id(); + extract_provider + .with_tile_bytes(base_id, |extract_bytes| { + source_provider + .with_tile_bytes(base_id, |source_bytes| { + assert_eq!(extract_bytes, source_bytes); + }) + .expect("the tile is in the source"); + }) + .expect("the tile is in the extract"); + } + + // Tiles outside the subset did not leak into the extract. + assert!(extract_provider.tile_size(excluded).is_none()); + + drop(extract_provider); + std::fs::remove_file(&extract_path).expect("the temp extract is removed"); + } +} diff --git a/valhalla-graphtile/src/lib.rs b/valhalla-graphtile/src/lib.rs index 179c3c5..4b5416f 100644 --- a/valhalla-graphtile/src/lib.rs +++ b/valhalla-graphtile/src/lib.rs @@ -1,5 +1,6 @@ #![doc = include_str!("../README.md")] +pub mod extract; mod graph_id; pub mod graph_tile; pub mod shape_codec; diff --git a/valhalla-graphtile/src/tile_provider/tarball.rs b/valhalla-graphtile/src/tile_provider/tarball.rs index 3866fd7..0755b61 100644 --- a/valhalla-graphtile/src/tile_provider/tarball.rs +++ b/valhalla-graphtile/src/tile_provider/tarball.rs @@ -175,6 +175,18 @@ impl TarballTileProvider { pub fn tile_ids(&self) -> impl Iterator { self.tile_index.keys() } + + /// The size in bytes of the tile containing this graph ID, + /// or [`None`] if the tile is not part of the tarball archive. + /// + /// The size comes straight from the tarball's `index.bin`, + /// so no tile data is accessed. + #[inline] + pub fn tile_size(&self, graph_id: GraphId) -> Option { + self.tile_index + .get(&graph_id.tile_base_id()) + .map(|tile_offset| tile_offset.size) + } } impl GraphTileProvider for TarballTileProvider { @@ -488,7 +500,13 @@ impl TileIndexBinEntry { Ok((graph_id, offset)) } - fn graph_id(&self) -> Result { + /// The graph ID of the indexed tile. + /// + /// # Errors + /// + /// Entries parsed from a foreign `index.bin` may contain arbitrary bit patterns, + /// so this fails if the stored tile ID has non-zero feature index bits. + pub fn graph_id(&self) -> Result { // SAFETY: We know that the bit field cannot contain a value // larger than the max allowed value (it's limited to 46 bits). // Therefore, this is guaranteed to be a valid Graph ID bit pattern. @@ -497,6 +515,18 @@ impl TileIndexBinEntry { Ok(graph_id) } + + /// The byte offset of the tile payload from the beginning of the tarball. + #[inline] + pub fn offset(&self) -> u64 { + self.offset.get() + } + + /// The size of the tile in bytes. + #[inline] + pub fn size(&self) -> u32 { + self.size.get() + } } /// Makes sure that `graph_id` identifies a full tile, From eeb6844f2703b1d4222b9bef88ab9fbd22803bed Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Tue, 13 Oct 2026 20:38:58 -0700 Subject: [PATCH 02/37] Reject index.bin entries that point past the end of the archive MmapTilePointer::as_tile_bytes makes a slice from an index offset and size without a bounds check. Before this change, TarballTileProvider did not check that these ranges were inside the file, so a corrupt or truncated index.bin could make safe calls read outside the memory map. - TileIndexBinEntry::validated now also checks that offset + size fits in the archive (with checked_add, so a huge offset cannot wrap). A failure is the new InvalidIndexBinError::OutOfBounds. - The provider reads the archive length before it reads the index, so all entry checks run in one pass and an invalid index fails before the file is mapped. After the map is created, the provider makes sure that the map length is the length that it checked. - New tests: check_range at the exact edge and on overflow, validated() at the edge, a real tarball whose index points past the end of the file, and a proptest which makes sure that the read path accepts every entry that the write path creates. - tempfile is a new dev-dependency for the test tarball. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 17 +- Cargo.toml | 1 + valhalla-graphtile/Cargo.toml | 1 + .../src/tile_provider/tarball.rs | 166 ++++++++++++++++-- 4 files changed, 165 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 26b58f2..a4c24cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1048,9 +1048,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.180" +version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" [[package]] name = "libm" @@ -1071,9 +1071,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "lock_api" @@ -1665,9 +1665,9 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.3" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" dependencies = [ "bitflags 2.10.0", "errno", @@ -1926,9 +1926,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.24.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", "getrandom 0.3.4", @@ -2232,6 +2232,7 @@ dependencies = [ "self_cell", "serde", "tar", + "tempfile", "thiserror 2.0.18", "tracing", "trig-const", diff --git a/Cargo.toml b/Cargo.toml index 03469f4..d1b0706 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,6 +54,7 @@ serde = { version = "1.0.210", features = ["derive"] } serde_with = "3.14.0" serde_json = "1.0.132" tar = "0.4.44" +tempfile = "3.27.0" thiserror = "2.0.11" tokio = { version = "1.48.0", features = ["macros", "rt-multi-thread", "fs", "io-util", "sync"] } tracing = "0.1.41" diff --git a/valhalla-graphtile/Cargo.toml b/valhalla-graphtile/Cargo.toml index 62370c8..88ba8a8 100644 --- a/valhalla-graphtile/Cargo.toml +++ b/valhalla-graphtile/Cargo.toml @@ -37,6 +37,7 @@ zerocopy-derive = { workspace = true } insta = { workspace = true } proptest = { workspace = true } rand = { workspace = true } +tempfile = { workspace = true } walkdir = { workspace = true } [lints] diff --git a/valhalla-graphtile/src/tile_provider/tarball.rs b/valhalla-graphtile/src/tile_provider/tarball.rs index 0755b61..6a4b0ad 100644 --- a/valhalla-graphtile/src/tile_provider/tarball.rs +++ b/valhalla-graphtile/src/tile_provider/tarball.rs @@ -102,7 +102,11 @@ impl TarballTileProvider { ); } - let mut archive = Archive::new(File::open(&path)?); + let archive_file = File::open(&path)?; + // Get the length before the index is read, + // so that every index entry can be checked against it in a single pass. + let archive_len = archive_file.metadata()?.len(); + let mut archive = Archive::new(archive_file); let mut entries = archive.entries()?; let Some(entry) = entries.next() else { @@ -128,7 +132,7 @@ impl TarballTileProvider { let mut tile_index: HashMap = HashMap::with_capacity(index_entries.len()); for entry in index_entries { - let (graph_id, offset) = entry.validated()?; + let (graph_id, offset) = entry.validated(archive_len)?; tile_index.insert( graph_id, TileOffset { @@ -149,6 +153,16 @@ impl TarballTileProvider { Arc::new(MmapOptions::new().map_raw_read_only(&file)?) }; + // Tile pointers turn index ranges into slices without a bounds check. + // The index ranges were checked against `archive_len`, + // so the memory map must have exactly that length. + if mmap.len() as u64 != archive_len { + return Err(GraphTileProviderError::InvalidTarball(format!( + "the archive changed size while it was opened ({archive_len} bytes, then {} bytes)", + mmap.len() + ))); + } + let mut tile_cache = HashMap::new(); if CACHE { @@ -465,6 +479,15 @@ pub enum InvalidIndexBinError { "the entry for {graph_id} has offset {offset}; expected a non-zero multiple of {TAR_BLOCK_SIZE}" )] Offset { graph_id: GraphId, offset: u64 }, + #[error( + "the entry for {graph_id} has {size} bytes at offset {offset}, which is past the end of the {archive_len}-byte archive" + )] + OutOfBounds { + graph_id: GraphId, + offset: u64, + size: u32, + archive_len: u64, + }, } impl TileIndexBinEntry { @@ -491,11 +514,13 @@ impl TileIndexBinEntry { /// /// [`parse_index_bin`] does not examine the entry values, /// so the read path must call this before it uses an entry. - /// The checks are the same as in [`TileIndexBinEntry::new`]. - fn validated(&self) -> Result<(GraphId, u64), InvalidIndexBinError> { + /// The checks are the same as in [`TileIndexBinEntry::new`], + /// and the tile must also lie inside an archive of `archive_len` bytes. + fn validated(&self, archive_len: u64) -> Result<(GraphId, u64), InvalidIndexBinError> { let graph_id = self.graph_id()?; let offset = self.offset.get(); check_offset(graph_id, offset)?; + check_range(graph_id, offset, self.size.get(), archive_len)?; Ok((graph_id, offset)) } @@ -554,6 +579,28 @@ fn check_offset(graph_id: GraphId, offset: u64) -> Result<(), InvalidIndexBinErr Ok(()) } +/// Makes sure that the tile at `offset` with `size` bytes lies inside an archive of `archive_len` bytes. +fn check_range( + graph_id: GraphId, + offset: u64, + size: u32, + archive_len: u64, +) -> Result<(), InvalidIndexBinError> { + if offset + .checked_add(u64::from(size)) + .is_none_or(|end| end > archive_len) + { + return Err(InvalidIndexBinError::OutOfBounds { + graph_id, + offset, + size, + archive_len, + }); + } + + Ok(()) +} + /// Parses an `index.bin` file to enable random access. /// /// See [`TileIndexBinEntry`] for a description of the tile format. @@ -646,16 +693,34 @@ mod test { #[test] fn test_validate_parsed_entry() { + // The first fixture entry has 1,181,852 bytes at offset 23,040, + // so it ends at byte 1,204,892. + const FIRST_ENTRY_END: u64 = 23_040 + 1_181_852; + let graph_id = GraphId::try_from_components(0, 2540, 0).expect("Unable to create graph ID"); let index_entries = parse_index_bin(INDEX_BIN_FIXTURE).expect("Unable to parse fixture"); - assert_eq!(index_entries[0].validated(), Ok((graph_id, 23040))); + assert_eq!( + index_entries[0].validated(FIRST_ENTRY_END), + Ok((graph_id, 23040)), + "A tile may end exactly at the end of the archive" + ); + assert_eq!( + index_entries[0].validated(FIRST_ENTRY_END - 1), + Err(InvalidIndexBinError::OutOfBounds { + graph_id, + offset: 23040, + size: 1_181_852, + archive_len: FIRST_ENTRY_END - 1 + }), + "A tile must not end past the end of the archive" + ); // Change the low byte of the first offset (23040 = 0x5a00) to make it unaligned let mut bytes = INDEX_BIN_FIXTURE.to_vec(); bytes[0] = 0x01; let index_entries = parse_index_bin(&bytes).expect("Unable to parse modified fixture"); assert_eq!( - index_entries[0].validated(), + index_entries[0].validated(FIRST_ENTRY_END), Err(InvalidIndexBinError::Offset { graph_id, offset: 23041 @@ -668,7 +733,7 @@ mod test { bytes[..8].fill(0); let index_entries = parse_index_bin(&bytes).expect("Unable to parse modified fixture"); assert_eq!( - index_entries[0].validated(), + index_entries[0].validated(FIRST_ENTRY_END), Err(InvalidIndexBinError::Offset { graph_id, offset: 0 @@ -681,7 +746,7 @@ mod test { bytes[11] |= 0x02; let index_entries = parse_index_bin(&bytes).expect("Unable to parse modified fixture"); assert_eq!( - index_entries[0].validated(), + index_entries[0].validated(FIRST_ENTRY_END), Err(InvalidIndexBinError::FeatureIndex( graph_id .with_feature_index(1) @@ -721,6 +786,73 @@ mod test { assert_eq!(index_entries[2].size.get(), 1_809_704); } + #[test] + fn test_check_range() { + let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); + + assert_eq!(check_range(graph_id, 512, 512, 1024), Ok(())); + assert_eq!( + check_range(graph_id, 512, 513, 1024), + Err(InvalidIndexBinError::OutOfBounds { + graph_id, + offset: 512, + size: 513, + archive_len: 1024 + }), + "A tile must end at or before the end of the archive" + ); + assert_eq!( + check_range(graph_id, u64::MAX - 511, 1024, u64::MAX), + Err(InvalidIndexBinError::OutOfBounds { + graph_id, + offset: u64::MAX - 511, + size: 1024, + archive_len: u64::MAX + }), + "The end of the range must not overflow" + ); + } + + // Tragically, memory maps are not supported by Miri! + #[cfg(not(miri))] + #[test] + fn test_reject_index_past_end_of_archive() { + let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); + // The index says that the tile has 10,000 bytes, + // but the archive only contains 100 bytes of tile data. + let index_entry = + TileIndexBinEntry::new(graph_id, 1536, 10_000).expect("Unable to create index entry"); + let index_bytes = serialize_index_bin(core::slice::from_ref(&index_entry)) + .expect("Unable to serialize the index"); + + let file = tempfile::NamedTempFile::new().expect("Unable to create a temporary file"); + let mut builder = tar::Builder::new(file.as_file()); + for (path, payload) in [("index.bin", index_bytes), ("0/003/015.gph", &[0x5a; 100])] { + let mut header = tar::Header::new_ustar(); + header.set_size(payload.len() as u64); + header.set_cksum(); + builder + .append_data(&mut header, path, payload) + .expect("Unable to append a tar entry"); + } + builder.finish().expect("Unable to finish the archive"); + + let result = TarballTileProvider::new_readonly(file.path()); + assert!( + matches!( + result, + Err(GraphTileProviderError::InvalidIndexBin( + InvalidIndexBinError::OutOfBounds { + offset: 1536, + size: 10_000, + .. + } + )) + ), + "An index entry past the end of the archive must be rejected" + ); + } + #[test] fn test_tile_index_bin_entry_new() { let graph_id = GraphId::try_from_components(0, 2540, 0).expect("Unable to create graph ID"); @@ -806,8 +938,12 @@ mod test { proptest! { #[test] fn test_index_bin_roundtrip( - // Levels are 3 bits, tile IDs are 22 bits, and offsets are counted in 512-byte blocks - raw_entries in vec((0u8..=7, 0u64..1 << 22, 1u64..=u64::MAX / 512, 0u32..=u32::MAX), 1..64) + // Levels are 3 bits, tile IDs are 22 bits, and offsets are counted in 512-byte blocks. + // The offsets are small enough that the end of every tile fits in a u64. + raw_entries in vec( + (0u8..=7, 0u64..1 << 22, 1u64..=(u64::MAX - u64::from(u32::MAX)) / 512, 0u32..=u32::MAX), + 1..64 + ) ) { let entries: Vec = raw_entries .into_iter() @@ -823,9 +959,15 @@ mod test { let parsed = parse_index_bin(bytes).expect("Serialized bytes always parse"); prop_assert_eq!(parsed, entries.as_slice()); - // The read path accepts every entry that the write path creates + // The read path accepts every entry that the write path creates, + // in an archive which is just long enough for all of the tiles. + let archive_len = parsed + .iter() + .map(|entry| entry.offset() + u64::from(entry.size())) + .max() + .expect("The entry list is non-empty"); for entry in parsed { - prop_assert!(entry.validated().is_ok()); + prop_assert!(entry.validated(archive_len).is_ok()); } } } From 6055dc4263bbc064d4aa74ea3c49aea5d065f6e5 Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Tue, 13 Oct 2026 20:39:44 -0700 Subject: [PATCH 03/37] Report tile IDs in extract errors and make the byte source generic - ExtractLayout::from_manifest returns the new ExtractLayoutError (EmptyManifest, DuplicateTile, EmptyTile, and InvalidIndexBin). ExtractWriteError wraps it in the Layout variant. A tile with a size of zero is now an error, because a graph tile always has a header. - A tile that is not in the source extract is now ExtractWriteError::MissingTile(GraphId), and an error from the byte source is Source { graph_id, source }. Before this change, both were a Provider error without the tile ID. - TileBytesSource has an associated Error type, so a source that is not a tarball does not have to put its errors in GraphTileProviderError. ExtractWriteError has a default type parameter, so the tarball path keeps the same signature. - The SAFETY comment in with_tile_bytes now refers to the bounds check that the provider does when it opens the tarball. - The round-trip test uses tempfile, so the file is removed even if an assertion fails. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/extract/mod.rs | 3 +- valhalla-graphtile/src/extract/writer.rs | 186 ++++++++++++++--------- 2 files changed, 118 insertions(+), 71 deletions(-) diff --git a/valhalla-graphtile/src/extract/mod.rs b/valhalla-graphtile/src/extract/mod.rs index 013e0f4..3732c4b 100644 --- a/valhalla-graphtile/src/extract/mod.rs +++ b/valhalla-graphtile/src/extract/mod.rs @@ -21,5 +21,6 @@ mod writer; pub use writer::{ - ExtractLayout, ExtractWriteError, TileBytesSource, write_extract, write_extract_from_source, + ExtractLayout, ExtractLayoutError, ExtractWriteError, TileBytesSource, write_extract, + write_extract_from_source, }; diff --git a/valhalla-graphtile/src/extract/writer.rs b/valhalla-graphtile/src/extract/writer.rs index f22f0c6..d896bc3 100644 --- a/valhalla-graphtile/src/extract/writer.rs +++ b/valhalla-graphtile/src/extract/writer.rs @@ -51,9 +51,11 @@ const ZERO_BLOCK: [u8; 512] = [0; 512]; /// The entry name of the tile index (always the first entry in the archive). const INDEX_ENTRY_NAME: &str = "index.bin"; -/// An error while writing a tile extract. -#[derive(Debug, Error)] -pub enum ExtractWriteError { +/// An error in an extract manifest. +/// +/// The layout finds these errors before the writer writes any bytes. +#[derive(Debug, Error, PartialEq, Eq)] +pub enum ExtractLayoutError { /// An extract containing no tiles is meaningless /// (and its `index.bin` would be malformed). #[error("the extract manifest contains no tiles")] @@ -61,6 +63,28 @@ pub enum ExtractWriteError { /// The manifest contains the same tile more than once. #[error("the extract manifest contains tile {0} more than once")] DuplicateTile(GraphId), + /// The manifest gives a tile a size of zero. + /// + /// A graph tile always contains at least a header, + /// so a size of zero is always an error in the manifest. + #[error("the extract manifest gives tile {0} a size of zero")] + EmptyTile(GraphId), + /// The manifest contains an ID which cannot be indexed. + #[error("invalid index.bin: {0}")] + InvalidIndexBin(#[from] InvalidIndexBinError), +} + +/// An error while writing a tile extract. +/// +/// `E` is the error type of the [`TileBytesSource`]. +#[derive(Debug, Error)] +pub enum ExtractWriteError { + /// The manifest is not valid (see [`ExtractLayout::from_manifest`]). + #[error("invalid manifest: {0}")] + Layout(#[from] ExtractLayoutError), + /// A requested tile is not in the source extract. + #[error("tile {0} is not in the source extract")] + MissingTile(GraphId), /// A graph ID in the manifest cannot be converted into a tar entry path. #[error("invalid graph ID: {0}")] InvalidGraphId(#[from] InvalidGraphIdError), @@ -80,13 +104,14 @@ pub enum ExtractWriteError { /// The length of the bytes actually produced by the source. actual: usize, }, - /// The manifest contains an ID which cannot be indexed, - /// or a tile offset that `index.bin` cannot record. - #[error("invalid index.bin: {0}")] - InvalidIndexBin(#[from] InvalidIndexBinError), /// The byte source failed to produce a tile. - #[error("tile provider error: {0}")] - Provider(#[from] GraphTileProviderError), + #[error("the byte source failed to produce tile {graph_id}: {source}")] + Source { + /// The base graph ID of the tile. + graph_id: GraphId, + /// The error from the byte source. + source: E, + }, /// Writing to the destination failed. #[error("I/O error: {0}")] Io(#[from] std::io::Error), @@ -102,35 +127,32 @@ pub enum ExtractWriteError { /// lets implementations hand out bytes borrowed from per-call state, /// such as a tile pointer into a memory map. pub trait TileBytesSource { + /// The error type of this source. + type Error: core::error::Error + 'static; + /// Calls `process` with the raw bytes of the tile containing the given graph ID. /// /// # Errors /// /// This fails if the tile does not exist in the source, /// or if fetching its bytes fails. - fn with_tile_bytes( - &self, - graph_id: GraphId, - process: F, - ) -> Result + fn with_tile_bytes(&self, graph_id: GraphId, process: F) -> Result where F: FnOnce(&[u8]) -> T; } impl TileBytesSource for TarballTileProvider { + type Error = GraphTileProviderError; + #[inline] - fn with_tile_bytes( - &self, - graph_id: GraphId, - process: F, - ) -> Result + fn with_tile_bytes(&self, graph_id: GraphId, process: F) -> Result where F: FnOnce(&[u8]) -> T, { let tile_pointer = self.get_pointer_for_tile_containing(graph_id)?; - // SAFETY: We assume that the tile pointer is live and points to an actual graph tile location in the tarball. - // That is to say, we assume the tile is well-formed (correct index) and that the underlying memory map - // has not changed in a way that would render the tile invalid. + // SAFETY: The provider checks at construction that every index range lies inside the memory map. + // We assume that the underlying memory map has not changed in a way that would render the tile invalid + // (see the safety discussion on TarballTileProvider). let tile_bytes = unsafe { tile_pointer.as_tile_bytes() }; Ok(process(tile_bytes)) } @@ -167,10 +189,11 @@ impl ExtractLayout { /// # Errors /// /// This fails if the manifest is empty, contains duplicate tiles, + /// contains a tile with a size of zero, /// or contains an ID which is not a valid base tile ID. - pub fn from_manifest(manifest: &[(GraphId, u32)]) -> Result { + pub fn from_manifest(manifest: &[(GraphId, u32)]) -> Result { if manifest.is_empty() { - return Err(ExtractWriteError::EmptyManifest); + return Err(ExtractLayoutError::EmptyManifest); } let mut sorted = manifest.to_vec(); @@ -186,7 +209,10 @@ impl ExtractLayout { for (i, (graph_id, size)) in sorted.iter().enumerate() { // Equal (level, tile index) pairs are adjacent after the sort. if i > 0 && sorted[i - 1].0 == *graph_id { - return Err(ExtractWriteError::DuplicateTile(*graph_id)); + return Err(ExtractLayoutError::DuplicateTile(*graph_id)); + } + if *size == 0 { + return Err(ExtractLayoutError::EmptyTile(*graph_id)); } let payload_offset = header_offset + TAR_BLOCK_SIZE; @@ -253,7 +279,7 @@ pub fn write_extract( provider .tile_size(base_id) .map(|size| (base_id, size)) - .ok_or(GraphTileProviderError::TileDoesNotExist) + .ok_or(ExtractWriteError::MissingTile(base_id)) }) .collect::, _>>()?; @@ -290,32 +316,35 @@ pub fn write_extract_from_source( manifest: &[(GraphId, u32)], source: &S, mut dest: W, -) -> Result +) -> Result> where S: TileBytesSource, W: Write, { let layout = ExtractLayout::from_manifest(manifest)?; - let index_bytes = serialize_index_bin(layout.index_entries())?; + let index_bytes = + serialize_index_bin(layout.index_entries()).map_err(ExtractLayoutError::from)?; write_entry(&mut dest, INDEX_ENTRY_NAME, index_bytes)?; for entry in layout.index_entries() { // Infallible in practice: the layout was constructed from valid base tile IDs. - let graph_id = entry.graph_id()?; + let graph_id = entry.graph_id().map_err(ExtractLayoutError::from)?; let path = tar_entry_path(graph_id)?; let expected = entry.size(); - source.with_tile_bytes(graph_id, |tile_bytes| { - if tile_bytes.len() != expected as usize { - return Err(ExtractWriteError::SizeMismatch { - graph_id, - expected, - actual: tile_bytes.len(), - }); - } - write_entry(&mut dest, &path, tile_bytes).map_err(ExtractWriteError::from) - })??; + source + .with_tile_bytes(graph_id, |tile_bytes| { + if tile_bytes.len() != expected as usize { + return Err(ExtractWriteError::SizeMismatch { + graph_id, + expected, + actual: tile_bytes.len(), + }); + } + write_entry(&mut dest, &path, tile_bytes).map_err(ExtractWriteError::from) + }) + .map_err(|source| ExtractWriteError::Source { graph_id, source })??; } // The standard end-of-archive marker: two zero blocks. @@ -379,6 +408,11 @@ mod tests { /// An in-memory byte source for driving the writer without any I/O. struct MemorySource(HashMap>); + /// The error of a [`MemorySource`] when it does not contain a tile. + #[derive(Debug, Error, PartialEq, Eq)] + #[error("the tile is not in memory")] + struct NotInMemory; + impl MemorySource { fn from_manifest(manifest: &[(GraphId, u32)]) -> Self { let tiles = manifest @@ -396,18 +430,16 @@ mod tests { } impl TileBytesSource for MemorySource { - fn with_tile_bytes( - &self, - graph_id: GraphId, - process: F, - ) -> Result + type Error = NotInMemory; + + fn with_tile_bytes(&self, graph_id: GraphId, process: F) -> Result where F: FnOnce(&[u8]) -> T, { self.0 .get(&graph_id.tile_base_id()) .map(|tile_bytes| process(tile_bytes)) - .ok_or(GraphTileProviderError::TileDoesNotExist) + .ok_or(NotInMemory) } } @@ -477,31 +509,40 @@ mod tests { #[test] fn test_layout_rejects_empty_manifest() { - assert!(matches!( - ExtractLayout::from_manifest(&[]), - Err(ExtractWriteError::EmptyManifest) - )); + assert_eq!( + ExtractLayout::from_manifest(&[]).unwrap_err(), + ExtractLayoutError::EmptyManifest + ); } #[test] fn test_layout_rejects_duplicate_tiles() { let manifest = [(tile(1, 47_701), 100), (tile(1, 47_701), 100)]; - assert!(matches!( - ExtractLayout::from_manifest(&manifest), - Err(ExtractWriteError::DuplicateTile(graph_id)) if graph_id == tile(1, 47_701) - )); + assert_eq!( + ExtractLayout::from_manifest(&manifest).unwrap_err(), + ExtractLayoutError::DuplicateTile(tile(1, 47_701)) + ); + } + + #[test] + fn test_layout_rejects_empty_tiles() { + let manifest = [(tile(0, 3015), 512), (tile(1, 47_701), 0)]; + assert_eq!( + ExtractLayout::from_manifest(&manifest).unwrap_err(), + ExtractLayoutError::EmptyTile(tile(1, 47_701)) + ); } #[test] fn test_layout_rejects_non_base_tile_ids() { let with_feature_index = GraphId::try_from_components(1, 47_701, 7).expect("valid graph ID in test"); - assert!(matches!( - ExtractLayout::from_manifest(&[(with_feature_index, 100)]), - Err(ExtractWriteError::InvalidIndexBin( - InvalidIndexBinError::FeatureIndex(_) + assert_eq!( + ExtractLayout::from_manifest(&[(with_feature_index, 100)]).unwrap_err(), + ExtractLayoutError::InvalidIndexBin(InvalidIndexBinError::FeatureIndex( + with_feature_index )) - )); + ); } #[test] @@ -529,9 +570,10 @@ mod tests { let result = write_extract_from_source(&manifest, &source, Vec::new()); assert!(matches!( result, - Err(ExtractWriteError::Provider( - GraphTileProviderError::TileDoesNotExist - )) + Err(ExtractWriteError::Source { + graph_id, + source: NotInMemory + }) if graph_id == tile(0, 3015) )); } @@ -648,6 +690,13 @@ mod tests { let excluded = tile(2, 763_925); assert!(source_provider.tile_size(excluded).is_some()); + // A tile which is not in the fixture at all. + let not_in_fixture = tile(2, 0); + assert!(matches!( + write_extract(&source_provider, &[tile(0, 3015), not_in_fixture], Vec::new()), + Err(ExtractWriteError::MissingTile(graph_id)) if graph_id == not_in_fixture + )); + let mut archive_bytes = Vec::new(); let written = write_extract(&source_provider, &tile_ids, &mut archive_bytes) .expect("the extract writes cleanly"); @@ -660,13 +709,13 @@ mod tests { // Round-trip: Valinor's own reader opens the extract, // and every tile is byte-identical to the source. - let extract_path = std::env::temp_dir().join(format!( - "valinor-extract-round-trip-{}.tar", - std::process::id() - )); - std::fs::write(&extract_path, &archive_bytes).expect("the extract is written to disk"); + // The temporary file is removed on drop, even if an assertion fails. + let mut extract_file = tempfile::NamedTempFile::new().expect("a temporary file is created"); + extract_file + .write_all(&archive_bytes) + .expect("the extract is written to disk"); - let extract_provider = TarballTileProvider::new_readonly(&extract_path) + let extract_provider = TarballTileProvider::new_readonly(extract_file.path()) .expect("the generated extract opens cleanly"); for graph_id in tile_ids { @@ -684,8 +733,5 @@ mod tests { // Tiles outside the subset did not leak into the extract. assert!(extract_provider.tile_size(excluded).is_none()); - - drop(extract_provider); - std::fs::remove_file(&extract_path).expect("the temp extract is removed"); } } From bc2038fd30f9aa98c7a8b8b689174e4e451c1582 Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Tue, 13 Oct 2026 20:40:51 -0700 Subject: [PATCH 04/37] Split valinor-cli into one module for each subcommand main.rs now contains only the clap entry point, the tracing set-up, and the dispatch. Each subcommand is a module in src/commands with its clap arguments and a run function, so a new subcommand does not make main.rs larger. This is a refactor only: get-edge moves into commands/get_edge.rs, and the help output is the same as before. The only code change is that parse_valhalla_data_paths takes a &Path (clippy: ptr_arg). Co-Authored-By: Claude Opus 5.5 --- valinor-cli/src/commands/get_edge.rs | 144 ++++++++++++++++++++++++++ valinor-cli/src/commands/mod.rs | 6 ++ valinor-cli/src/main.rs | 148 +++------------------------ 3 files changed, 166 insertions(+), 132 deletions(-) create mode 100644 valinor-cli/src/commands/get_edge.rs create mode 100644 valinor-cli/src/commands/mod.rs diff --git a/valinor-cli/src/commands/get_edge.rs b/valinor-cli/src/commands/get_edge.rs new file mode 100644 index 0000000..b201ca4 --- /dev/null +++ b/valinor-cli/src/commands/get_edge.rs @@ -0,0 +1,144 @@ +//! The `get-edge` subcommand: pretty-prints information about a directed edge. + +use std::{ + fs, + path::{Path, PathBuf}, + str::FromStr, +}; + +use anyhow::{Context, anyhow}; +use clap::Args; +use serde_json::Value as JsonValue; +use tracing::info; +use valhalla_graphtile::tile_provider::TrafficTileProvider; +use valhalla_graphtile::{ + GraphId, + graph_tile::GraphTile, + tile_provider::{DirectoryGraphTileProvider, GraphTileProvider, TarballTileProvider}, +}; + +/// Pretty-print information about a directed edge (incl. live traffic, if available) +#[derive(Args, Debug)] +pub(crate) struct GetEdgeArgs { + /// Graph ID (u64) or slash-form level/tile/index + graph_id: String, +} + +fn parse_graph_id(input: &str) -> anyhow::Result { + // Try to parse as a pure integer. + if let Ok(id) = input.parse::() { + return GraphId::try_from_id(id).map_err(|e| anyhow!(e)); + } + + // Try slash-separated level/tile/index + let parts: Vec<&str> = input.split('/').collect(); + if parts.len() == 3 { + let level = u8::from_str(parts[0]).context("invalid level in graph id")?; + let tile_id = u64::from_str(parts[1]).context("invalid tile id in graph id")?; + let index = u64::from_str(parts[2]).context("invalid index in graph id")?; + return GraphId::try_from_components(level, tile_id, index).map_err(|e| anyhow!(e)); + } + + Err(anyhow!( + "Unrecognized graph id format. Use a u64 integer or level/tile/index" + )) +} + +#[derive(Debug, Clone)] +struct DataSources { + routing_graph: Option, + traffic_extract: Option, +} + +#[derive(Debug, Clone)] +enum RoutingGraphDataSource { + Tarball(PathBuf), + TileDir(PathBuf), +} + +fn parse_valhalla_data_paths(path: &Path) -> anyhow::Result { + let bytes = + fs::read(path).with_context(|| format!("Failed to read config at {}", path.display()))?; + let json: JsonValue = + serde_json::from_slice(&bytes).context("Invalid JSON in valhalla config")?; + + let get_path_if_exists = |key: &str| -> Option { + if let JsonValue::String(s) = &json["mjolnir"][key] + && !s.is_empty() + && fs::exists(s).unwrap_or_default() + { + Some(PathBuf::from(s)) + } else { + None + } + }; + + let tile_extract = get_path_if_exists("tile_extract"); + let tile_dir = get_path_if_exists("tile_dir"); + let traffic_extract = get_path_if_exists("traffic_extract"); + Ok(DataSources { + routing_graph: match (tile_extract, tile_dir) { + (Some(tarball), _) => Some(RoutingGraphDataSource::Tarball(tarball)), + (_, Some(dir)) => Some(RoutingGraphDataSource::TileDir(dir)), + (None, None) => None, + }, + traffic_extract, + }) +} + +fn pretty_print_edge_info( + provider: &T, + traffic_provider: Option<&TrafficTileProvider>, + gid: GraphId, +) -> anyhow::Result<()> { + let output = provider.with_tile_containing(gid, |tile| { + let edge = tile.get_directed_edge(gid)?; + let edge_info = tile.get_edge_info(edge)?; + let traffic_info = traffic_provider.map(|tp| unsafe { tp.get_speeds_for_edge(gid).ok() }); + + Ok::(serde_json::json!({ + "graph_id": gid, + "directed_edge": edge, + "edge_info": edge_info, + "traffic": traffic_info, + })) + })??; + println!("{}", serde_json::to_string_pretty(&output)?); + Ok(()) +} + +/// Runs the `get-edge` subcommand with the given Valhalla config. +pub(crate) fn run(valhalla_config: &Path, args: &GetEdgeArgs) -> anyhow::Result<()> { + let gid = parse_graph_id(&args.graph_id)?; + let sources = parse_valhalla_data_paths(valhalla_config)?; + let traffic_extract = if let Some(path) = sources.traffic_extract { + info!(path = path.to_str(), "Using traffic extract"); + Some(TrafficTileProvider::new_readonly(path)?) + } else { + info!("No traffic extract could be found"); + None + }; + + // Prefer tarball if set, fall back to directory. + if let Some(graph) = sources.routing_graph { + match graph { + RoutingGraphDataSource::Tarball(path) => { + info!(path = path.to_str(), "Using tarball tile extract"); + + let provider = TarballTileProvider::new_readonly(&path)?; + pretty_print_edge_info(&provider, traffic_extract.as_ref(), gid) + } + RoutingGraphDataSource::TileDir(path) => { + info!(path = path.to_str(), "Using tile directory"); + + let provider = + DirectoryGraphTileProvider::new(path, std::num::NonZeroUsize::new(1).unwrap()); + pretty_print_edge_info(&provider, traffic_extract.as_ref(), gid) + } + } + } else { + Err(anyhow!( + "No routing graph data sources could be loaded. Expected a valid 'tile_extract' (tarball) or 'tile_dir' in the config." + )) + } +} diff --git a/valinor-cli/src/commands/mod.rs b/valinor-cli/src/commands/mod.rs new file mode 100644 index 0000000..24c714d --- /dev/null +++ b/valinor-cli/src/commands/mod.rs @@ -0,0 +1,6 @@ +//! The subcommands of the CLI. +//! +//! Each subcommand has its own module, +//! which contains its clap arguments and a `run` function. + +pub(crate) mod get_edge; diff --git a/valinor-cli/src/main.rs b/valinor-cli/src/main.rs index 17eb3be..02a1b68 100644 --- a/valinor-cli/src/main.rs +++ b/valinor-cli/src/main.rs @@ -1,18 +1,13 @@ -use std::{fs, path::PathBuf, str::FromStr}; +mod commands; + +use std::path::PathBuf; -use anyhow::{Context, anyhow}; use clap::{Parser, Subcommand}; -use serde_json::Value as JsonValue; -use tracing::info; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::{EnvFilter, Layer}; -use valhalla_graphtile::tile_provider::TrafficTileProvider; -use valhalla_graphtile::{ - GraphId, - graph_tile::GraphTile, - tile_provider::{DirectoryGraphTileProvider, GraphTileProvider, TarballTileProvider}, -}; + +use commands::get_edge; #[derive(Parser, Debug)] #[command(name = "valinor-cli", author, version, about, long_about = None)] @@ -28,94 +23,7 @@ struct Cli { #[derive(Subcommand, Debug)] enum Commands { - /// Pretty-print information about a directed edge (incl. live traffic, if available) - GetEdge { - /// Graph ID (u64) or slash-form level/tile/index - graph_id: String, - }, -} - -fn parse_graph_id(input: &str) -> anyhow::Result { - // Try to parse as a pure integer. - if let Ok(id) = input.parse::() { - return GraphId::try_from_id(id).map_err(|e| anyhow!(e)); - } - - // Try slash-separated level/tile/index - let parts: Vec<&str> = input.split('/').collect(); - if parts.len() == 3 { - let level = u8::from_str(parts[0]).context("invalid level in graph id")?; - let tile_id = u64::from_str(parts[1]).context("invalid tile id in graph id")?; - let index = u64::from_str(parts[2]).context("invalid index in graph id")?; - return GraphId::try_from_components(level, tile_id, index).map_err(|e| anyhow!(e)); - } - - Err(anyhow!( - "Unrecognized graph id format. Use a u64 integer or level/tile/index" - )) -} - -#[derive(Debug, Clone)] -struct DataSources { - routing_graph: Option, - traffic_extract: Option, -} - -#[derive(Debug, Clone)] -enum RoutingGraphDataSource { - Tarball(PathBuf), - TileDir(PathBuf), -} - -fn parse_valhalla_data_paths(path: &PathBuf) -> anyhow::Result { - let bytes = - fs::read(path).with_context(|| format!("Failed to read config at {}", path.display()))?; - let json: JsonValue = - serde_json::from_slice(&bytes).context("Invalid JSON in valhalla config")?; - - let get_path_if_exists = |key: &str| -> Option { - if let JsonValue::String(s) = &json["mjolnir"][key] - && !s.is_empty() - && fs::exists(s).unwrap_or_default() - { - Some(PathBuf::from(s)) - } else { - None - } - }; - - let tile_extract = get_path_if_exists("tile_extract"); - let tile_dir = get_path_if_exists("tile_dir"); - let traffic_extract = get_path_if_exists("traffic_extract"); - Ok(DataSources { - routing_graph: match (tile_extract, tile_dir) { - (Some(tarball), _) => Some(RoutingGraphDataSource::Tarball(tarball)), - (_, Some(dir)) => Some(RoutingGraphDataSource::TileDir(dir)), - (None, None) => None, - }, - traffic_extract, - }) -} - -fn pretty_print_edge_info( - provider: &T, - traffic_provider: Option<&TrafficTileProvider>, - gid: GraphId, -) -> anyhow::Result<()> { - let output = provider.with_tile_containing(gid, |tile| { - let edge = tile.get_directed_edge(gid)?; - let edge_info = tile.get_edge_info(edge)?; - let traffic_info = traffic_provider.map(|tp| unsafe { tp.get_speeds_for_edge(gid).ok() }); - - Ok::(serde_json::json!({ - "graph_id": gid, - "directed_edge": edge, - "edge_info": edge_info, - "traffic": traffic_info, - })) - })??; - println!("{}", serde_json::to_string_pretty(&output)?); - Ok(()) + GetEdge(get_edge::GetEdgeArgs), } fn main() -> anyhow::Result<()> { @@ -127,41 +35,17 @@ fn main() -> anyhow::Result<()> { let cli = Cli::parse(); match cli.command { - Commands::GetEdge { graph_id } => { - let gid = parse_graph_id(&graph_id)?; - let sources = parse_valhalla_data_paths(&cli.valhalla_config)?; - let traffic_extract = if let Some(path) = sources.traffic_extract { - info!(path = path.to_str(), "Using traffic extract"); - Some(TrafficTileProvider::new_readonly(path)?) - } else { - info!("No traffic extract could be found"); - None - }; - - // Prefer tarball if set, fall back to directory. - if let Some(graph) = sources.routing_graph { - match graph { - RoutingGraphDataSource::Tarball(path) => { - info!(path = path.to_str(), "Using tarball tile extract"); + Commands::GetEdge(args) => get_edge::run(&cli.valhalla_config, &args), + } +} - let provider = TarballTileProvider::new_readonly(&path)?; - pretty_print_edge_info(&provider, traffic_extract.as_ref(), gid) - } - RoutingGraphDataSource::TileDir(path) => { - info!(path = path.to_str(), "Using tile directory"); +#[cfg(test)] +mod tests { + use super::*; + use clap::CommandFactory; - let provider = DirectoryGraphTileProvider::new( - path, - std::num::NonZeroUsize::new(1).unwrap(), - ); - pretty_print_edge_info(&provider, traffic_extract.as_ref(), gid) - } - } - } else { - Err(anyhow!( - "No routing graph data sources could be loaded. Expected a valid 'tile_extract' (tarball) or 'tile_dir' in the config." - )) - } - } + #[test] + fn clap_derive_is_consistent() { + Cli::command().debug_assert(); } } From 58cb49bbb04675829acf5bc2e6e221e352b52b5b Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Tue, 13 Oct 2026 20:41:50 -0700 Subject: [PATCH 05/37] Add valinor-cli extract for offline sub-extracts valinor-cli extract --tile LEVEL/TILE_INDEX copies the given tiles of a source extract tarball into a new, spec-compliant extract with write_extract. --tile can be repeated or given a comma-separated list. The command prints a one-line summary on stderr, and removes a partial output file if the write fails. The extract subcommand takes a tarball directly, not a Valhalla JSON config. So the config path moves from a global argument into the get-edge arguments (valinor-cli get-edge ). The VALHALLA_CONFIG environment variable still works. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + valinor-cli/Cargo.toml | 3 + valinor-cli/README.md | 20 ++- valinor-cli/src/commands/extract.rs | 221 +++++++++++++++++++++++++++ valinor-cli/src/commands/get_edge.rs | 10 +- valinor-cli/src/commands/mod.rs | 1 + valinor-cli/src/main.rs | 44 +++++- 7 files changed, 285 insertions(+), 15 deletions(-) create mode 100644 valinor-cli/src/commands/extract.rs diff --git a/Cargo.lock b/Cargo.lock index a4c24cb..1c1792d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2286,6 +2286,7 @@ dependencies = [ "anyhow", "clap", "serde_json", + "tempfile", "tracing", "tracing-subscriber", "valhalla-graphtile", diff --git a/valinor-cli/Cargo.toml b/valinor-cli/Cargo.toml index cc0b362..576e8f8 100644 --- a/valinor-cli/Cargo.toml +++ b/valinor-cli/Cargo.toml @@ -16,5 +16,8 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter"] } valhalla-graphtile = { workspace = true, features = ["serde"] } +[dev-dependencies] +tempfile = { workspace = true } + [lints] workspace = true diff --git a/valinor-cli/README.md b/valinor-cli/README.md index f620a82..544ac6b 100644 --- a/valinor-cli/README.md +++ b/valinor-cli/README.md @@ -1,8 +1,8 @@ # Valinor CLI A CLI multi-tool for quickly inspecting Valhalla graphs. -This utility takes a Valhalla JSON configuration as input, -and has subcommands with various functionality. +Most subcommands take a Valhalla JSON configuration as input +(`extract` is the exception; it takes a tile extract tarball directly). This is primarily meant as a low-level debugging tool for Valinor developers. If this looks useful to you, it probably is; otherwise, it's probably not. @@ -12,9 +12,21 @@ If this looks useful to you, it probably is; otherwise, it's probably not. As an example, you can get edge info like this (works with either the integer or slash separated form): ```shell -RUST_LOG=info cargo run --package valinor-cli -- ~/valhalla-docker/valhalla/valhalla.json get-edge 901881911882 +RUST_LOG=info cargo run --package valinor-cli -- get-edge ~/valhalla-docker/valhalla/valhalla.json 901881911882 ``` Then you'll get some JSON output on your terminal with the details. -Run the `help` subcommand for more details (everything is automatically documented via clap). \ No newline at end of file +## Example: writing an offline sub-extract + +You can write a smaller, fully spec-compliant extract +which contains only some tiles of a source extract (each tile is `level/tile_index`): + +```shell +cargo run --package valinor-cli -- extract valhalla-graphtile/fixtures/andorra-tiles.tar andorra-sub.tar --tile 0/3015,1/47701,2/763926 +``` + +A one-line summary (tile count and output size) is printed on stderr. +The output is interchangeable with `valhalla_build_extract` output. + +Run the `help` subcommand for more details (everything is automatically documented via clap). diff --git a/valinor-cli/src/commands/extract.rs b/valinor-cli/src/commands/extract.rs new file mode 100644 index 0000000..04efec5 --- /dev/null +++ b/valinor-cli/src/commands/extract.rs @@ -0,0 +1,221 @@ +//! The `extract` subcommand: writes an offline sub-extract of a tile extract tarball. + +use std::{ + fs, + fs::File, + io::{BufWriter, Write}, + path::PathBuf, +}; + +use anyhow::Context; +use clap::Args; +use valhalla_graphtile::GraphId; +use valhalla_graphtile::extract::write_extract; +use valhalla_graphtile::tile_provider::TarballTileProvider; + +/// Write an offline sub-extract of a tile extract tarball +/// +/// Copies the given graph tiles of the source extract into a new, fully spec-compliant +/// extract tarball. Every tile in the output is byte-identical to its source tile, +/// and the output is interchangeable with `valhalla_build_extract` output, +/// so it is consumable by both stock Valhalla and Valinor. +/// +/// Unlike the other subcommands, this takes the tile extract tarball path +/// directly rather than a Valhalla JSON config: extraction reads an extract +/// file, not a configured tile hierarchy. +#[derive(Args, Debug)] +pub(crate) struct ExtractArgs { + /// Path to the source tile extract tarball (e.g. tiles.tar) + source: PathBuf, + + /// Path to write the output extract tarball to + output: PathBuf, + + /// A tile to copy, as level/tile_index (e.g. `--tile 2/763926`) + /// + /// Give the option once for each tile, + /// or give a comma-separated list (e.g. `--tile 0/3015,1/47701`). + #[arg( + long = "tile", + value_name = "LEVEL/TILE_INDEX", + required = true, + value_delimiter = ',', + value_parser = parse_tile + )] + pub(crate) tiles: Vec, +} + +/// A successful extract, as reported on stderr. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct ExtractSummary { + tile_count: usize, + bytes_written: u64, +} + +/// Parses a `level/tile_index` tile argument into a base tile ID. +fn parse_tile(input: &str) -> Result { + let Some((level, tile_index)) = input.split_once('/') else { + return Err(format!("'{input}' is not in the level/tile_index form")); + }; + let level: u8 = level + .trim() + .parse() + .map_err(|_| format!("the level in '{input}' is not a valid number"))?; + let tile_index: u64 = tile_index + .trim() + .parse() + .map_err(|_| format!("the tile index in '{input}' is not a valid number"))?; + GraphId::try_from_components(level, tile_index, 0) + .map_err(|e| format!("'{input}' is not a valid tile: {e}")) +} + +/// Writes the sub-extract, and removes the partial output if the write fails. +fn write_sub_extract(args: &ExtractArgs) -> anyhow::Result { + let provider = TarballTileProvider::new_readonly(&args.source).with_context(|| { + format!( + "Failed to open source tile extract at {}", + args.source.display() + ) + })?; + + let file = File::create(&args.output).with_context(|| { + format!( + "Failed to create the output extract at {}", + args.output.display() + ) + })?; + + let write_result = (|| { + let mut dest = BufWriter::new(file); + let bytes_written = write_extract(&provider, &args.tiles, &mut dest) + .context("Failed to write the extract")?; + dest.flush().context("Failed to flush the output extract")?; + Ok(bytes_written) + })(); + + match write_result { + Ok(bytes_written) => Ok(ExtractSummary { + tile_count: args.tiles.len(), + bytes_written, + }), + Err(error) => { + // A partial tarball is worse than no tarball + // (it may open, but with tiles silently missing or truncated), + // so clean up on a best-effort basis. + let _ = fs::remove_file(&args.output); + Err(error) + } + } +} + +/// Runs the `extract` subcommand, and prints a one-line summary on stderr. +pub(crate) fn run(args: &ExtractArgs) -> anyhow::Result<()> { + let summary = write_sub_extract(args)?; + let noun = if summary.tile_count == 1 { + "tile" + } else { + "tiles" + }; + eprintln!( + "Extracted {} {noun} ({} bytes) to {}", + summary.tile_count, + summary.bytes_written, + args.output.display() + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Path to the Andorra fixture tarball in the sibling crate. + fn andorra_fixture() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("valhalla-graphtile") + .join("fixtures") + .join("andorra-tiles.tar") + } + + fn tile(level: u8, tile_index: u64) -> GraphId { + GraphId::try_from_components(level, tile_index, 0).expect("valid tile ID in test") + } + + #[test] + fn parse_tile_accepts_level_and_tile_index() { + assert_eq!(parse_tile("2/763926"), Ok(tile(2, 763_926))); + assert_eq!(parse_tile(" 0 / 3015 "), Ok(tile(0, 3015))); + } + + #[test] + fn parse_tile_rejects_malformed_input() { + for (input, expected) in [ + ("2", "is not in the level/tile_index form"), + ("x/763926", "the level in 'x/763926' is not a valid number"), + ("2/abc", "the tile index in '2/abc' is not a valid number"), + // A graph ID with a feature index has three components. + ( + "2/763926/7", + "the tile index in '2/763926/7' is not a valid number", + ), + // Levels have 3 bits, and tile indexes have 22 bits. + ("8/0", "'8/0' is not a valid tile"), + ("0/4194304", "'0/4194304' is not a valid tile"), + ] { + let error = parse_tile(input).expect_err("malformed input is rejected"); + assert!(error.contains(expected), "{input}: {error}"); + } + } + + // Tragically, memory maps are not supported by Miri, + // and the tests below open tarballs with the memory-mapped provider! + #[cfg(not(miri))] + #[test] + fn extract_round_trips_through_the_tarball_provider() { + let temp_dir = tempfile::tempdir().expect("a temporary directory is created"); + let output = temp_dir.path().join("andorra-sub.tar"); + let tiles = vec![tile(0, 3015), tile(1, 47_701), tile(2, 763_926)]; + let args = ExtractArgs { + source: andorra_fixture(), + output: output.clone(), + tiles: tiles.clone(), + }; + + let summary = write_sub_extract(&args).expect("the extract writes cleanly"); + assert_eq!(summary.tile_count, tiles.len()); + + let on_disk = fs::metadata(&output).expect("the output exists").len(); + assert_eq!(summary.bytes_written, on_disk); + + // The produced tarball opens with the crate's own reader + // and contains exactly the requested tiles. + let extract = TarballTileProvider::new_readonly(&output) + .expect("the generated extract opens cleanly"); + let mut extracted: Vec = extract.tile_ids().copied().collect(); + extracted.sort_unstable(); + let mut expected = tiles; + expected.sort_unstable(); + assert_eq!(extracted, expected); + } + + #[cfg(not(miri))] + #[test] + fn failed_extract_removes_the_partial_output() { + let temp_dir = tempfile::tempdir().expect("a temporary directory is created"); + let output = temp_dir.path().join("andorra-sub.tar"); + let args = ExtractArgs { + source: andorra_fixture(), + output: output.clone(), + // The fixture has no tile 2/0. + tiles: vec![tile(0, 3015), tile(2, 0)], + }; + + let error = write_sub_extract(&args).expect_err("a missing tile fails the extract"); + assert!( + format!("{error:#}").contains("is not in the source extract"), + "{error:#}" + ); + assert!(!output.exists(), "the partial output is removed"); + } +} diff --git a/valinor-cli/src/commands/get_edge.rs b/valinor-cli/src/commands/get_edge.rs index b201ca4..95d47e7 100644 --- a/valinor-cli/src/commands/get_edge.rs +++ b/valinor-cli/src/commands/get_edge.rs @@ -20,6 +20,10 @@ use valhalla_graphtile::{ /// Pretty-print information about a directed edge (incl. live traffic, if available) #[derive(Args, Debug)] pub(crate) struct GetEdgeArgs { + /// Path to valhalla.json + #[arg(env)] + valhalla_config: PathBuf, + /// Graph ID (u64) or slash-form level/tile/index graph_id: String, } @@ -107,10 +111,10 @@ fn pretty_print_edge_info( Ok(()) } -/// Runs the `get-edge` subcommand with the given Valhalla config. -pub(crate) fn run(valhalla_config: &Path, args: &GetEdgeArgs) -> anyhow::Result<()> { +/// Runs the `get-edge` subcommand. +pub(crate) fn run(args: &GetEdgeArgs) -> anyhow::Result<()> { let gid = parse_graph_id(&args.graph_id)?; - let sources = parse_valhalla_data_paths(valhalla_config)?; + let sources = parse_valhalla_data_paths(&args.valhalla_config)?; let traffic_extract = if let Some(path) = sources.traffic_extract { info!(path = path.to_str(), "Using traffic extract"); Some(TrafficTileProvider::new_readonly(path)?) diff --git a/valinor-cli/src/commands/mod.rs b/valinor-cli/src/commands/mod.rs index 24c714d..60a069a 100644 --- a/valinor-cli/src/commands/mod.rs +++ b/valinor-cli/src/commands/mod.rs @@ -3,4 +3,5 @@ //! Each subcommand has its own module, //! which contains its clap arguments and a `run` function. +pub(crate) mod extract; pub(crate) mod get_edge; diff --git a/valinor-cli/src/main.rs b/valinor-cli/src/main.rs index 02a1b68..da8b2ad 100644 --- a/valinor-cli/src/main.rs +++ b/valinor-cli/src/main.rs @@ -1,21 +1,15 @@ mod commands; -use std::path::PathBuf; - use clap::{Parser, Subcommand}; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::{EnvFilter, Layer}; -use commands::get_edge; +use commands::{extract, get_edge}; #[derive(Parser, Debug)] #[command(name = "valinor-cli", author, version, about, long_about = None)] struct Cli { - /// Path to valhalla.json - #[arg(env)] - valhalla_config: PathBuf, - /// Subcommand/tool to run #[command(subcommand)] command: Commands, @@ -24,6 +18,7 @@ struct Cli { #[derive(Subcommand, Debug)] enum Commands { GetEdge(get_edge::GetEdgeArgs), + Extract(extract::ExtractArgs), } fn main() -> anyhow::Result<()> { @@ -35,7 +30,8 @@ fn main() -> anyhow::Result<()> { let cli = Cli::parse(); match cli.command { - Commands::GetEdge(args) => get_edge::run(&cli.valhalla_config, &args), + Commands::GetEdge(args) => get_edge::run(&args), + Commands::Extract(args) => extract::run(&args), } } @@ -43,9 +39,41 @@ fn main() -> anyhow::Result<()> { mod tests { use super::*; use clap::CommandFactory; + use valhalla_graphtile::GraphId; + + fn tile(level: u8, tile_index: u64) -> GraphId { + GraphId::try_from_components(level, tile_index, 0).expect("valid tile ID in test") + } #[test] fn clap_derive_is_consistent() { Cli::command().debug_assert(); } + + #[test] + fn extract_tiles_accept_repeated_and_comma_separated_options() { + let cli = Cli::try_parse_from([ + "valinor-cli", + "extract", + "in.tar", + "out.tar", + "--tile", + "0/3015,1/47701", + "--tile", + "2/763926", + ]) + .expect("the arguments parse"); + let Commands::Extract(args) = cli.command else { + panic!("expected the extract subcommand"); + }; + assert_eq!( + args.tiles, + vec![tile(0, 3015), tile(1, 47_701), tile(2, 763_926)] + ); + } + + #[test] + fn extract_tiles_are_required() { + assert!(Cli::try_parse_from(["valinor-cli", "extract", "in.tar", "out.tar"]).is_err()); + } } From 95d76d1700360925fef03f8a4e3041570c7d1531 Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Wed, 21 Oct 2026 10:23:50 -0700 Subject: [PATCH 06/37] Add GraphId::relative_path and string parsers for graph IDs The tar writer and the directory provider both build the path of a tile file from a graph ID. Before this change, the writer turned the PathBuf from GraphId::file_path back into a string with `/` separators. Now GraphId::relative_path builds the `/`-separated string directly, and file_path uses it. relative_path also returns InvalidGraphIdError::Level for levels 4 to 7. Before this change, file_path panicked on an index out of bounds for these levels. GraphId now implements FromStr for a packed u64 value or the level/tile_id/index form, and GraphId::parse_tile parses the level/tile_id form of a whole tile. Both use the same component parser and return the new ParseGraphIdError. A later commit uses them in place of the separate parsers in the get-edge and extract subcommands. InvalidGraphIdError and ParseGraphIdError are now re-exported at the crate root, so callers can name them. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/graph_id.rs | 330 +++++++++++++++++++++++++++-- valhalla-graphtile/src/lib.rs | 2 +- 2 files changed, 309 insertions(+), 23 deletions(-) diff --git a/valhalla-graphtile/src/graph_id.rs b/valhalla-graphtile/src/graph_id.rs index fa765b9..1ddbe25 100644 --- a/valhalla-graphtile/src/graph_id.rs +++ b/valhalla-graphtile/src/graph_id.rs @@ -1,4 +1,5 @@ use crate::tile_hierarchy::{STANDARD_LEVELS, TRANSIT_LEVEL}; +use core::str::FromStr; #[cfg(feature = "serde")] use serde::{Serialize, Serializer}; use std::fmt::{Display, Formatter}; @@ -37,6 +38,31 @@ pub enum InvalidGraphIdError { InvalidGraphId, } +/// The reasons why a string is not a valid graph ID or tile. +#[derive(Debug, Error, PartialEq)] +pub enum ParseGraphIdError { + /// The input does not have the expected number of `/`-separated components. + #[error("'{input}' is not in the {expected} form")] + Format { + input: String, + /// The expected form (e.g. `level/tile_id`). + expected: &'static str, + }, + /// A component of the input is not a non-negative integer. + #[error("the {component} in '{input}' is not a valid number")] + Component { + input: String, + /// The name of the component (e.g. `level`). + component: &'static str, + }, + /// The components are numbers, but they do not make a valid graph ID. + #[error("'{input}' is not a valid graph ID: {source}")] + Invalid { + input: String, + source: InvalidGraphIdError, + }, +} + /// An Identifier of a node or an edge within the tiled, hierarchical graph. /// It packs a hierarchy level, tile ID, and an identifier within /// the tile/level into a 64-bit integer. @@ -209,19 +235,44 @@ impl GraphId { GraphId(U64::::new(self.value() & 0x01ff_ffff)) } - /// Constructs a relative path for the given tile. + /// Parses the base graph ID of a tile from the `level/tile_id` form (e.g. `2/762485`). + /// + /// This is the tile form of [`GraphId::from_str`]: + /// it names a whole tile, so it has no feature index component, + /// and the feature index of the result is zero. + /// Whitespace around each component is ignored. + /// + /// # Errors + /// + /// This fails if the input does not have exactly two `/`-separated components, + /// if a component is not a non-negative integer, + /// or if the level or tile ID is out of range (see [`GraphId::try_from_components`]). + pub fn parse_tile(input: &str) -> Result { + let [level, tile_id] = parse_components(input, "level/tile_id", ["level", "tile ID"])?; + components_to_graph_id(input, level, tile_id, 0) + } + + /// Constructs the relative path of the tile file for this graph ID, + /// with `/` separators on every platform (e.g. `2/000/762/485.gph`). + /// + /// This is the form that Valhalla uses for tile directories, tar entry names, and tile URLs. + /// The first component is the level; + /// the rest is the tile ID, zero-padded to the width of the level + /// and split into groups of three digits. + /// The feature index is ignored. /// /// # Errors /// /// This will fail if the tile is invalid for this level of tiling. /// TODO: It seems like we could do this check at tile creation time? - pub fn file_path(&self, extension: &str) -> Result { - // This is quite ugly and prone to failure; this is a pretty literal C++ translation + pub fn relative_path(&self, extension: &str) -> Result { let level_number = self.level(); let level = if level_number == TRANSIT_LEVEL.level { &*TRANSIT_LEVEL } else { - &STANDARD_LEVELS[level_number as usize] + STANDARD_LEVELS + .get(level_number as usize) + .ok_or(InvalidGraphIdError::Level)? }; let max_id = level.tiling_system.n_cols * level.tiling_system.n_rows - 1; @@ -230,28 +281,107 @@ impl GraphId { return Err(InvalidGraphIdError::GraphTileId); } - let l = max_id.max(1).ilog10() + 1; - let rem = l % 3; - let n_digits = if rem == 0 { l } else { l + (3 - rem) }; - debug_assert!(n_digits % 3 == 0); - - // This may not be as efficient as the Valhalla implementation, but it's more readable. - // Format tile_id with leading zeros to match max_length - let padded_id = format!("{:0>width$}", tile_id, width = n_digits as usize); - let tile_id_chars: Vec<_> = padded_id.chars().collect(); - // Create the final path using groups of threes - let tile_id_component = tile_id_chars - .rchunks(3) - .fold(PathBuf::new(), |acc, chunk| { - PathBuf::from(chunk.iter().collect::()).join(acc) - }) - .with_extension(extension); + // The width of the largest tile ID of the level, rounded up to a whole number of groups. + let n_digits = (max_id.max(1).ilog10() + 1).next_multiple_of(3) as usize; + let padded_id = format!("{tile_id:0>n_digits$}"); + + let mut path = level_number.to_string(); + // The padded ID is ASCII digits, and its length is a multiple of 3, + // so these byte ranges are always whole groups. + for group_start in (0..n_digits).step_by(3) { + path.push('/'); + path.push_str(&padded_id[group_start..group_start + 3]); + } + if !extension.is_empty() { + path.push('.'); + path.push_str(extension); + } + + Ok(path) + } + + /// Constructs a relative path for the given tile. + /// + /// This is [`GraphId::relative_path`] as a [`PathBuf`], + /// with the separators of the platform. + /// + /// # Errors + /// + /// This will fail if the tile is invalid for this level of tiling. + pub fn file_path(&self, extension: &str) -> Result { + Ok(self.relative_path(extension)?.split('/').collect()) + } +} + +impl FromStr for GraphId { + type Err = ParseGraphIdError; + + /// Parses a graph ID from its packed `u64` value (e.g. `16889572344463360`) + /// or from the `level/tile_id/index` form (e.g. `2/762485/42`). + /// + /// To parse a whole tile in the `level/tile_id` form, use [`GraphId::parse_tile`]. + fn from_str(input: &str) -> Result { + if let Ok(id) = input.trim().parse::() { + return GraphId::try_from_id(id).map_err(|source| ParseGraphIdError::Invalid { + input: input.to_string(), + source, + }); + } - // Build and return the final string - Ok(PathBuf::from(self.level().to_string()).join(tile_id_component)) + let [level, tile_id, index] = parse_components( + input, + "level/tile_id/index", + ["level", "tile ID", "feature index"], + )?; + components_to_graph_id(input, level, tile_id, index) } } +/// Splits `input` into exactly `N` `/`-separated integer components. +fn parse_components( + input: &str, + expected: &'static str, + names: [&'static str; N], +) -> Result<[u64; N], ParseGraphIdError> { + let format_error = || ParseGraphIdError::Format { + input: input.to_string(), + expected, + }; + + let mut parts = input.split('/'); + let mut values = [0; N]; + for (value, component) in values.iter_mut().zip(names) { + let part = parts.next().ok_or_else(format_error)?; + *value = part + .trim() + .parse() + .map_err(|_| ParseGraphIdError::Component { + input: input.to_string(), + component, + })?; + } + if parts.next().is_some() { + return Err(format_error()); + } + + Ok(values) +} + +/// Converts parsed components into a graph ID, and reports range errors against `input`. +fn components_to_graph_id( + input: &str, + level: u64, + tile_id: u64, + index: u64, +) -> Result { + let invalid = |source| ParseGraphIdError::Invalid { + input: input.to_string(), + source, + }; + let level = u8::try_from(level).map_err(|_| invalid(InvalidGraphIdError::Level))?; + GraphId::try_from_components(level, tile_id, index).map_err(invalid) +} + impl Ord for GraphId { #[inline] fn cmp(&self, other: &Self) -> std::cmp::Ordering { @@ -423,4 +553,160 @@ mod tests { Ok("3/001/000/000.gph".into()) ); } + + /// Helper to construct a base tile [`GraphId`]. + fn tile(level: u8, tile_id: u64) -> GraphId { + GraphId::try_from_components(level, tile_id, 0).expect("valid tile ID in test") + } + + #[test] + fn test_relative_path() { + // Levels 0 and 1 pad the tile ID to 6 digits (two groups); + // level 2 pads to 9 digits (three groups). + for (graph_id, expected) in [ + (tile(0, 3015), "0/003/015.gph"), + (tile(0, 49), "0/000/049.gph"), + (tile(1, 47_701), "1/047/701.gph"), + (tile(1, 0), "1/000/000.gph"), + (tile(2, 762_485), "2/000/762/485.gph"), + (tile(2, 1_036_799), "2/001/036/799.gph"), + ] { + assert_eq!(graph_id.relative_path("gph").as_deref(), Ok(expected)); + // The file path is the same path with the separators of the platform. + assert_eq!( + graph_id.file_path("gph"), + Ok(expected.split('/').collect::()) + ); + } + + // The feature index is not part of the path. + let with_index = GraphId::try_from_components(2, 762_485, 42).unwrap(); + assert_eq!( + with_index.relative_path("gph").as_deref(), + Ok("2/000/762/485.gph") + ); + + // An empty extension adds no dot. + assert_eq!( + tile(1, 47_701).relative_path("").as_deref(), + Ok("1/047/701") + ); + } + + #[test] + fn test_relative_path_invalid() { + // Level 0 has 45 rows and 90 columns of 4-degree tiles, + // so its largest tile ID is 45 * 90 - 1 = 4049. + assert_eq!( + tile(0, 4049).relative_path("gph").as_deref(), + Ok("0/004/049.gph") + ); + assert_eq!( + tile(0, 4050).relative_path("gph"), + Err(InvalidGraphIdError::GraphTileId) + ); + // Levels 4 to 7 fit in the bit field, but there is no tiling system for them. + assert_eq!( + tile(5, 0).relative_path("gph"), + Err(InvalidGraphIdError::Level) + ); + } + + #[test] + fn test_parse_tile() { + assert_eq!(GraphId::parse_tile("2/763926"), Ok(tile(2, 763_926))); + assert_eq!(GraphId::parse_tile(" 0 / 3015 "), Ok(tile(0, 3015))); + } + + #[test] + fn test_parse_tile_invalid() { + let format = |input: &str| ParseGraphIdError::Format { + input: input.to_string(), + expected: "level/tile_id", + }; + assert_eq!(GraphId::parse_tile("2"), Err(format("2"))); + // A graph ID with a feature index is not a tile. + assert_eq!(GraphId::parse_tile("2/763926/7"), Err(format("2/763926/7"))); + assert_eq!( + GraphId::parse_tile("2/"), + Err(ParseGraphIdError::Component { + input: "2/".to_string(), + component: "tile ID", + }) + ); + assert_eq!( + GraphId::parse_tile("x/763926"), + Err(ParseGraphIdError::Component { + input: "x/763926".to_string(), + component: "level", + }) + ); + assert_eq!( + GraphId::parse_tile("-1/0"), + Err(ParseGraphIdError::Component { + input: "-1/0".to_string(), + component: "level", + }) + ); + // Levels have 3 bits, and tile IDs have 22 bits. + for (input, source) in [ + ("8/0", InvalidGraphIdError::Level), + ("256/0", InvalidGraphIdError::Level), + ("0/4194304", InvalidGraphIdError::GraphTileId), + ] { + assert_eq!( + GraphId::parse_tile(input), + Err(ParseGraphIdError::Invalid { + input: input.to_string(), + source, + }) + ); + } + } + + #[test] + fn test_from_str() { + let expected = GraphId::try_from_components(2, 762_485, 42).unwrap(); + assert_eq!("2/762485/42".parse(), Ok(expected)); + assert_eq!(expected.value().to_string().parse(), Ok(expected)); + // Whitespace around a packed value is ignored. + assert_eq!( + " 16889572344463360 " + .parse::() + .map(|id| id.value()), + Ok(16_889_572_344_463_360) + ); + } + + #[test] + fn test_from_str_invalid() { + assert_eq!( + "2/762485".parse::(), + Err(ParseGraphIdError::Format { + input: "2/762485".to_string(), + expected: "level/tile_id/index", + }) + ); + assert_eq!( + "2/762485/x".parse::(), + Err(ParseGraphIdError::Component { + input: "2/762485/x".to_string(), + component: "feature index", + }) + ); + assert_eq!( + INVALID_GRAPH_ID.to_string().parse::(), + Err(ParseGraphIdError::Invalid { + input: INVALID_GRAPH_ID.to_string(), + source: InvalidGraphIdError::InvalidGraphId, + }) + ); + assert_eq!( + "0/0/2097152".parse::(), + Err(ParseGraphIdError::Invalid { + input: "0/0/2097152".to_string(), + source: InvalidGraphIdError::TileIndex, + }) + ); + } } diff --git a/valhalla-graphtile/src/lib.rs b/valhalla-graphtile/src/lib.rs index 4b5416f..ea748f6 100644 --- a/valhalla-graphtile/src/lib.rs +++ b/valhalla-graphtile/src/lib.rs @@ -20,7 +20,7 @@ use serde::{Deserialize, Serialize}; // The implementations are sufficiently complex that we want to have lots of files, // But many of those only have one or two useful definitions to re-export, // so this flattens things for better ergonomics. -pub use graph_id::GraphId; +pub use graph_id::{GraphId, InvalidGraphIdError, ParseGraphIdError}; /// Road class; broad hierarchies of relative (and sometimes locally specific) importance. /// From dc5a59ccf2e9c75c1ac89ed911c38720afdd1cd5 Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Wed, 21 Oct 2026 10:25:18 -0700 Subject: [PATCH 07/37] Add one helper module for the test fixture paths Before this change, 22 tests built each fixture path with PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("fixtures").join(...). The new test_fixtures module gives each fixture a function with a short description of what the fixture contains, so a test can find a fixture by name. This is a test-only change. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/extract/writer.rs | 7 ++-- valhalla-graphtile/src/graph_tile.rs | 17 +++------- valhalla-graphtile/src/graph_tile/builder.rs | 21 ++++-------- valhalla-graphtile/src/lib.rs | 2 ++ valhalla-graphtile/src/test_fixtures.rs | 33 +++++++++++++++++++ .../src/tile_provider/directory.rs | 22 ++++--------- valhalla-graphtile/src/tile_provider/mod.rs | 14 +++----- .../src/tile_provider/tarball.rs | 14 +++----- .../src/tile_provider/traffic.rs | 9 ++--- 9 files changed, 65 insertions(+), 74 deletions(-) create mode 100644 valhalla-graphtile/src/test_fixtures.rs diff --git a/valhalla-graphtile/src/extract/writer.rs b/valhalla-graphtile/src/extract/writer.rs index d896bc3..4e5bf18 100644 --- a/valhalla-graphtile/src/extract/writer.rs +++ b/valhalla-graphtile/src/extract/writer.rs @@ -395,6 +395,7 @@ fn tar_entry_path(graph_id: GraphId) -> Result { #[cfg(test)] mod tests { use super::*; + use crate::test_fixtures; use crate::tile_provider::parse_index_bin; use std::collections::HashMap; use std::io::Read; @@ -669,11 +670,7 @@ mod tests { #[cfg(not(miri))] #[test] fn test_andorra_round_trip() { - use std::path::PathBuf; - - let fixture_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles.tar"); + let fixture_path = test_fixtures::andorra_tiles_tar(); let source_provider = TarballTileProvider::new_readonly(fixture_path) .expect("Unable to init the source tile provider"); diff --git a/valhalla-graphtile/src/graph_tile.rs b/valhalla-graphtile/src/graph_tile.rs index 567a8ec..b06c3f1 100644 --- a/valhalla-graphtile/src/graph_tile.rs +++ b/valhalla-graphtile/src/graph_tile.rs @@ -12,6 +12,8 @@ use std::sync::Arc; use thiserror::Error; use zerocopy::{FromBytes, I16, LE, U32}; +#[cfg(test)] +use crate::test_fixtures; use enumset::EnumSet; use geo::{CoordFloat, Point, coord}; use num_traits::FromPrimitive; @@ -932,10 +934,7 @@ static TEST_GRAPH_TILE_L0: LazyLock = LazyLock::new(|| { let relative_path = TEST_GRAPH_TILE_ID_L0 .file_path("gph") .expect("Unable to get relative path"); - let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles") - .join(relative_path); + let path = test_fixtures::andorra_tiles_dir().join(relative_path); let bytes = std::fs::read(path).expect("Unable to read file"); OwnedGraphTileHandle::try_from(bytes).expect("Unable to get tile") @@ -946,10 +945,7 @@ static TEST_GRAPH_TILE_L2: LazyLock = LazyLock::new(|| { let relative_path = TEST_GRAPH_TILE_ID_L2 .file_path("gph") .expect("Unable to get relative path"); - let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles") - .join(relative_path); + let path = test_fixtures::andorra_tiles_dir().join(relative_path); let bytes = std::fs::read(path).expect("Unable to read file"); OwnedGraphTileHandle::try_from(bytes).expect("Unable to get tile") @@ -971,10 +967,7 @@ static TEST_GRAPH_TILE_WITH_FLOW: LazyLock = LazyLock::new let relative_path = TEST_GRAPH_TILE_ID_L0 .file_path("gph") .expect("Unable to get relative path"); - let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles-with-traffic") - .join(relative_path); + let path = test_fixtures::andorra_tiles_with_traffic_dir().join(relative_path); let bytes = std::fs::read(path).expect("Unable to read file"); OwnedGraphTileHandle::try_from(bytes).expect("Unable to get tile") diff --git a/valhalla-graphtile/src/graph_tile/builder.rs b/valhalla-graphtile/src/graph_tile/builder.rs index 18504cb..348fd20 100644 --- a/valhalla-graphtile/src/graph_tile/builder.rs +++ b/valhalla-graphtile/src/graph_tile/builder.rs @@ -665,6 +665,7 @@ where mod tests { use crate::graph_tile::predicted_speeds::COEFFICIENT_COUNT; use crate::graph_tile::{GraphTileBuilder, GraphTileHeader, OwnedGraphTileHandle}; + use crate::test_fixtures; use std::path::Path; use walkdir::WalkDir; @@ -709,9 +710,7 @@ mod tests { #[test] fn round_trip_from_builder_without_changes() { - let base_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base_dir = test_fixtures::andorra_tiles_dir(); for entry in WalkDir::new(base_dir) { let entry = entry.expect("Unable to traverse directory"); @@ -733,9 +732,7 @@ mod tests { #[test] fn round_trip_from_builder_without_changes_including_traffic() { - let base_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles-with-traffic"); + let base_dir = test_fixtures::andorra_tiles_with_traffic_dir(); for entry in WalkDir::new(base_dir) { let entry = entry.expect("Unable to traverse directory"); @@ -757,9 +754,7 @@ mod tests { #[test] fn add_predicted_speeds() { - let base_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base_dir = test_fixtures::andorra_tiles_dir(); let in_bytes = std::fs::read(base_dir.join("0").join("003").join("015.gph")) .expect("Unable to read file"); @@ -782,9 +777,7 @@ mod tests { .with_predicted_encoded_speeds(42, "BcUACQAEACEADP/7/9sAGf/fAAwAGwARAAX/+AAAAB0AFAAS//AAF//+ACwACQAqAAAAKP/6AEMABABsAAsBBAAq/rcAAP+bAAz/zv/s/9MABf/Y//X/8//9/+wACf/P//EADv/8//L//P/y//H////7AAwAFf/5//oADgAZAAQAFf/3/+8AB//yAB8ABv/0AAUAEf/8//QAFAAG//b////j//v//QAT//7/+f/kABMABwABAAv/6//8//cAEwAAABT/8v/6//wAAAAQ//cACwAFAAT/1//sAAEADAABAAYAE//9AAn/7gAH/+AAFQAB//4AC//o/+gAE//+AAAAFf/l//kABP/+//kACAAG//cAHv/qAB0AAv/4/+v/+wALAAMABP/3AAT/8wAIAAr/9wAK//j/+wAEAAD/+P/8//v/8f/2//L//AALAAcABgAG//gAAv/5AAoAHv//AAcAFf/zABD/7AAUAAv/7v/8AAgACAAN//0ADP/iABD/9f/3//7/+P/3AAQADP//AAMABw==").unwrap() .into_bytes().unwrap(); - let base_dir_with_traffic = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles-with-traffic"); + let base_dir_with_traffic = test_fixtures::andorra_tiles_with_traffic_dir(); let expected_out_bytes = std::fs::read(base_dir_with_traffic.join("0").join("003").join("015.gph")) .expect("Unable to read file"); @@ -798,9 +791,7 @@ mod tests { #[test] fn predicted_speeds_deduplication() { - let base_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base_dir = test_fixtures::andorra_tiles_dir(); let in_bytes = std::fs::read(base_dir.join("0").join("003").join("015.gph")) .expect("Unable to read file"); diff --git a/valhalla-graphtile/src/lib.rs b/valhalla-graphtile/src/lib.rs index ea748f6..af36f27 100644 --- a/valhalla-graphtile/src/lib.rs +++ b/valhalla-graphtile/src/lib.rs @@ -5,6 +5,8 @@ mod graph_id; pub mod graph_tile; pub mod shape_codec; pub mod spatial; +#[cfg(test)] +mod test_fixtures; pub mod tile_hierarchy; pub mod tile_provider; pub mod traffic_tile; diff --git a/valhalla-graphtile/src/test_fixtures.rs b/valhalla-graphtile/src/test_fixtures.rs new file mode 100644 index 0000000..d620b2a --- /dev/null +++ b/valhalla-graphtile/src/test_fixtures.rs @@ -0,0 +1,33 @@ +//! Paths to the test fixtures in the `fixtures` directory of this crate. +//! +//! All fixtures are made from the same OpenStreetMap extract of Andorra. + +use std::path::PathBuf; + +fn fixture(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("fixtures") + .join(name) +} + +/// A tile directory, as `valhalla_build_tiles` writes it. +pub(crate) fn andorra_tiles_dir() -> PathBuf { + fixture("andorra-tiles") +} + +/// The tiles of [`andorra_tiles_dir`] in an extract tarball, +/// as `valhalla_build_extract` writes it. +pub(crate) fn andorra_tiles_tar() -> PathBuf { + fixture("andorra-tiles.tar") +} + +/// The tiles of [`andorra_tiles_dir`] with predicted traffic speeds +/// from `valhalla_add_predicted_traffic`. +pub(crate) fn andorra_tiles_with_traffic_dir() -> PathBuf { + fixture("andorra-tiles-with-traffic") +} + +/// A live traffic tarball for the Andorra tiles. +pub(crate) fn andorra_traffic_tar() -> PathBuf { + fixture("andorra-traffic.tar") +} diff --git a/valhalla-graphtile/src/tile_provider/directory.rs b/valhalla-graphtile/src/tile_provider/directory.rs index 01212a3..25ad3dc 100644 --- a/valhalla-graphtile/src/tile_provider/directory.rs +++ b/valhalla-graphtile/src/tile_provider/directory.rs @@ -202,6 +202,7 @@ mod test { use super::DirectoryGraphTileProvider; use crate::GraphId; use crate::graph_tile::GraphTile; + use crate::test_fixtures; use crate::tile_hierarchy::STANDARD_LEVELS; use crate::tile_provider::GraphTileProvider; use core::num::NonZeroUsize; @@ -210,13 +211,10 @@ mod test { rng, }; use std::collections::HashSet; - use std::path::PathBuf; #[test] fn test_get_tile() { - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(1).unwrap()); let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); let tile = provider @@ -232,9 +230,7 @@ mod test { fn test_get_opp_edge() { let mut rng = rng(); - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(1).unwrap()); let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); let tile = provider @@ -264,9 +260,7 @@ mod test { #[test] fn test_get_shortcut_fixture() { - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(1).unwrap()); // This test relies on the state of the andorra-tiles fixtures. @@ -310,9 +304,7 @@ mod test { #[cfg(not(miri))] #[test] fn test_level_zero_shortcut_sanity() { - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(1).unwrap()); let base_id = GraphId::try_from_components(0, 3015, 0).unwrap(); @@ -393,9 +385,7 @@ mod test { // how the tile structure works. // Specifically, there is only a single "copy" of a feature in the routing graph, // and all features of a specific road class exist in specific tile levels. - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(1).unwrap()); let graph_ids = vec![ GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"), diff --git a/valhalla-graphtile/src/tile_provider/mod.rs b/valhalla-graphtile/src/tile_provider/mod.rs index d068fd3..ab60959 100644 --- a/valhalla-graphtile/src/tile_provider/mod.rs +++ b/valhalla-graphtile/src/tile_provider/mod.rs @@ -519,13 +519,13 @@ impl LockTable { mod tests { use crate::GraphId; use crate::graph_tile::GraphTile; + use crate::test_fixtures; use crate::tile_hierarchy::STANDARD_LEVELS; use crate::tile_provider::{DirectoryGraphTileProvider, GraphTileProvider}; use geo::{Destination, HasDimensions, Haversine, LineString, Validation, coord, point}; use rand::rngs::ThreadRng; use rand::{Rng, rng}; use std::num::NonZeroUsize; - use std::path::PathBuf; #[test] fn haversine_antimeridian_wraps() { @@ -544,9 +544,7 @@ mod tests { fn test_nodes_within_radius() { let mut rng = rng(); - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(4).unwrap()); let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); provider.with_tile_containing_or_panic(graph_id, |tile_view| { @@ -665,9 +663,7 @@ mod tests { #[test] fn test_edges_within_radius_level0() { - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(4).unwrap()); let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); sanity_check_edges_within_radius(provider, graph_id, rng()); @@ -675,9 +671,7 @@ mod tests { #[test] fn test_edges_within_radius_level2() { - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(4).unwrap()); // Use a known level 2 tile from fixtures to exercise edge bins path let graph_id = diff --git a/valhalla-graphtile/src/tile_provider/tarball.rs b/valhalla-graphtile/src/tile_provider/tarball.rs index 6a4b0ad..f1449cf 100644 --- a/valhalla-graphtile/src/tile_provider/tarball.rs +++ b/valhalla-graphtile/src/tile_provider/tarball.rs @@ -651,11 +651,11 @@ pub fn serialize_index_bin(entries: &[TileIndexBinEntry]) -> Result<&[u8], Inval mod test { use super::*; use crate::graph_tile::{GraphTile, GraphTileView}; + use crate::test_fixtures; use crate::tile_provider::DirectoryGraphTileProvider; use proptest::collection::vec; use proptest::{prop_assert, prop_assert_eq, proptest}; use std::num::NonZeroUsize; - use std::path::PathBuf; /// Bytes taken from the start of a large extract generated by official Valhalla tooling. const INDEX_BIN_FIXTURE: &[u8] = &[ @@ -976,9 +976,7 @@ mod test { #[cfg(not(miri))] #[test] fn test_get_tile() { - let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles.tar"); + let path = test_fixtures::andorra_tiles_tar(); let provider = TarballTileProvider::new_readonly(path).expect("Unable to init tile provider"); let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); @@ -1013,15 +1011,11 @@ mod test { GraphId::try_from_components(2, 763927, 1).expect("Unable to create graph ID"), ]; - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let directory_provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(1).unwrap()); - let tarball_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles.tar"); + let tarball_path = test_fixtures::andorra_tiles_tar(); let tarball_provider = TarballTileProvider::new_readonly(tarball_path.clone()) .expect("Unable to init tile provider"); let immutable_tarball_provider = unsafe { diff --git a/valhalla-graphtile/src/tile_provider/traffic.rs b/valhalla-graphtile/src/tile_provider/traffic.rs index 7e7a5b4..876c818 100644 --- a/valhalla-graphtile/src/tile_provider/traffic.rs +++ b/valhalla-graphtile/src/tile_provider/traffic.rs @@ -231,15 +231,14 @@ impl TrafficTileProvider { #[cfg(all(test, not(miri)))] mod tests { use crate::GraphId; + use crate::test_fixtures; use crate::tile_provider::TrafficTileProvider; use crate::traffic_tile::{SpeedValue, TrafficSpeed}; use std::path::PathBuf; #[test] fn test_get_speed() { - let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-traffic.tar"); + let path = test_fixtures::andorra_traffic_tar(); let provider = TrafficTileProvider::new_readonly(path).expect("Unable to init tile provider"); @@ -269,9 +268,7 @@ mod tests { fn test_set_speed() { const DESIRED_SPEED: u8 = 32; - let fixture_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-traffic.tar"); + let fixture_path = test_fixtures::andorra_traffic_tar(); // let tmp_path = PathBuf::from("/Users/ianthetechie/valhalla-docker/valhalla/traffic.tar"); let tmp_dir = option_env!("RUNNER_TEMP").unwrap_or("/tmp"); From 0bffa5666539607b7f4ca4dd57a12a6811a74295 Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Wed, 21 Oct 2026 10:26:35 -0700 Subject: [PATCH 08/37] Share the tar block size and index entry name between reader and writer The tarball provider and the extract writer each defined the tar block size, and each used the "index.bin" entry name. Both constants are now defined once, next to the index.bin format in the tarball provider, and the writer imports them. The test of TileIndexBinEntry::validated now calculates the end of the first fixture entry from the parsed entry (offset plus size), not from numbers copied into the test. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/extract/writer.rs | 15 +++----- valhalla-graphtile/src/tile_provider/mod.rs | 1 + .../src/tile_provider/tarball.rs | 36 ++++++++++--------- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/valhalla-graphtile/src/extract/writer.rs b/valhalla-graphtile/src/extract/writer.rs index 4e5bf18..d73945a 100644 --- a/valhalla-graphtile/src/extract/writer.rs +++ b/valhalla-graphtile/src/extract/writer.rs @@ -34,22 +34,17 @@ use crate::GraphId; use crate::graph_id::InvalidGraphIdError; use crate::tile_provider::{ - GraphTileProviderError, InvalidIndexBinError, TarballTileProvider, TileIndexBinEntry, - serialize_index_bin, + GraphTileProviderError, INDEX_BIN_ENTRY_NAME, InvalidIndexBinError, TAR_BLOCK_SIZE, + TarballTileProvider, TileIndexBinEntry, serialize_index_bin, }; use std::io::Write; use tar::{EntryType, Header}; use thiserror::Error; -/// The tar block size; every entry is a one-block header plus a payload padded to a block. -const TAR_BLOCK_SIZE: u64 = 512; - /// A zeroed block (of [`TAR_BLOCK_SIZE`] bytes) /// for payload padding and the end-of-archive marker. -const ZERO_BLOCK: [u8; 512] = [0; 512]; - -/// The entry name of the tile index (always the first entry in the archive). -const INDEX_ENTRY_NAME: &str = "index.bin"; +#[expect(clippy::cast_possible_truncation, reason = "512 fits in every usize")] +const ZERO_BLOCK: [u8; TAR_BLOCK_SIZE as usize] = [0; TAR_BLOCK_SIZE as usize]; /// An error in an extract manifest. /// @@ -325,7 +320,7 @@ where let index_bytes = serialize_index_bin(layout.index_entries()).map_err(ExtractLayoutError::from)?; - write_entry(&mut dest, INDEX_ENTRY_NAME, index_bytes)?; + write_entry(&mut dest, INDEX_BIN_ENTRY_NAME, index_bytes)?; for entry in layout.index_entries() { // Infallible in practice: the layout was constructed from valid base tile IDs. diff --git a/valhalla-graphtile/src/tile_provider/mod.rs b/valhalla-graphtile/src/tile_provider/mod.rs index ab60959..06dd56d 100644 --- a/valhalla-graphtile/src/tile_provider/mod.rs +++ b/valhalla-graphtile/src/tile_provider/mod.rs @@ -26,6 +26,7 @@ use crate::graph_tile::{ pub use directory::DirectoryGraphTileProvider; pub use iterators::GraphSearchResult; use iterators::ResultsWithinRadius; +pub(crate) use tarball::{INDEX_BIN_ENTRY_NAME, TAR_BLOCK_SIZE}; pub use tarball::{ InvalidIndexBinError, TarballTileProvider, TileIndexBinEntry, parse_index_bin, serialize_index_bin, diff --git a/valhalla-graphtile/src/tile_provider/tarball.rs b/valhalla-graphtile/src/tile_provider/tarball.rs index f1449cf..586eb00 100644 --- a/valhalla-graphtile/src/tile_provider/tarball.rs +++ b/valhalla-graphtile/src/tile_provider/tarball.rs @@ -116,10 +116,10 @@ impl TarballTileProvider { }; let mut entry = entry?; - if entry.path()?.to_string_lossy() != "index.bin" { - return Err(GraphTileProviderError::InvalidTarball( - "Expected index.bin at the start of the archive".to_string(), - )); + if entry.path()?.to_string_lossy() != INDEX_BIN_ENTRY_NAME { + return Err(GraphTileProviderError::InvalidTarball(format!( + "Expected {INDEX_BIN_ENTRY_NAME} at the start of the archive" + ))); } // Read the index file @@ -459,8 +459,14 @@ pub struct TileIndexBinEntry { const INDEX_ENTRY_SIZE: usize = size_of::(); /// The block size of a tar archive. -/// The data of each tar entry starts on a block boundary. -const TAR_BLOCK_SIZE: u64 = 512; +/// +/// Each tar entry is a one-block header, +/// then the data of the entry, padded with zeros to a whole number of blocks. +/// So the data of each entry starts on a block boundary. +pub(crate) const TAR_BLOCK_SIZE: u64 = 512; + +/// The name of the tile index entry, which is the first entry of an extract tarball. +pub(crate) const INDEX_BIN_ENTRY_NAME: &str = "index.bin"; /// The reasons why an `index.bin` file or one of its entries is not valid. #[derive(Debug, Error, PartialEq, Eq)] @@ -693,24 +699,22 @@ mod test { #[test] fn test_validate_parsed_entry() { - // The first fixture entry has 1,181,852 bytes at offset 23,040, - // so it ends at byte 1,204,892. - const FIRST_ENTRY_END: u64 = 23_040 + 1_181_852; - let graph_id = GraphId::try_from_components(0, 2540, 0).expect("Unable to create graph ID"); let index_entries = parse_index_bin(INDEX_BIN_FIXTURE).expect("Unable to parse fixture"); + // The smallest archive that can contain the first tile ends where that tile ends. + let first_entry_end = index_entries[0].offset() + u64::from(index_entries[0].size()); assert_eq!( - index_entries[0].validated(FIRST_ENTRY_END), + index_entries[0].validated(first_entry_end), Ok((graph_id, 23040)), "A tile may end exactly at the end of the archive" ); assert_eq!( - index_entries[0].validated(FIRST_ENTRY_END - 1), + index_entries[0].validated(first_entry_end - 1), Err(InvalidIndexBinError::OutOfBounds { graph_id, offset: 23040, size: 1_181_852, - archive_len: FIRST_ENTRY_END - 1 + archive_len: first_entry_end - 1 }), "A tile must not end past the end of the archive" ); @@ -720,7 +724,7 @@ mod test { bytes[0] = 0x01; let index_entries = parse_index_bin(&bytes).expect("Unable to parse modified fixture"); assert_eq!( - index_entries[0].validated(FIRST_ENTRY_END), + index_entries[0].validated(first_entry_end), Err(InvalidIndexBinError::Offset { graph_id, offset: 23041 @@ -733,7 +737,7 @@ mod test { bytes[..8].fill(0); let index_entries = parse_index_bin(&bytes).expect("Unable to parse modified fixture"); assert_eq!( - index_entries[0].validated(FIRST_ENTRY_END), + index_entries[0].validated(first_entry_end), Err(InvalidIndexBinError::Offset { graph_id, offset: 0 @@ -746,7 +750,7 @@ mod test { bytes[11] |= 0x02; let index_entries = parse_index_bin(&bytes).expect("Unable to parse modified fixture"); assert_eq!( - index_entries[0].validated(FIRST_ENTRY_END), + index_entries[0].validated(first_entry_end), Err(InvalidIndexBinError::FeatureIndex( graph_id .with_feature_index(1) From 82b06e424c679e2dca042468ab085ad934448cee Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Wed, 21 Oct 2026 10:30:41 -0700 Subject: [PATCH 09/37] Make the extract writer generic over tarball and directory sources write_extract now takes any TileBytesSource and a list of tile IDs. The source gives the size and the bytes of each tile, so a caller never gives tile sizes by hand. TarballTileProvider (for all MUT and CACHE values) and DirectoryGraphTileProvider implement the trait, so an extract can be written from an extract tarball or from a tile directory. - TileBytesSource moves from the extract module to tile_provider, next to GraphTileProvider, and gets a tile_size method. GraphTileProvider cannot be used for this: a GraphTileView does not keep the bytes that it was decoded from. - The directory provider reads the size from the file metadata and the bytes from the file, under the per-tile lock. It does not use its tile cache, because a copy reads each tile once. A tile file larger than u32::MAX bytes is the new GraphTileProviderError::TileTooLarge. - write_extract_from_source and ExtractLayout are no longer public. The inherent TarballTileProvider::tile_size is replaced by the trait method. - The layout code names its cursor next_entry_offset and calculates the length of each entry with entry_len, in place of inline offset arithmetic. The writer counts the bytes that it writes, checks each payload offset against the layout in debug builds, and returns the real count. - The tests express the expected offsets in blocks, and compare the index with the entry offsets that the tar crate finds. A new proptest does the same for random tile sets. A new test makes sure that the directory and tarball fixtures give byte-identical extracts. - write_extract tests now cover an empty tile list, the same tile with two feature indexes, a missing tile (with no bytes written), a size mismatch, and a source error. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/extract/mod.rs | 19 +- valhalla-graphtile/src/extract/writer.rs | 669 ++++++++++-------- .../src/tile_provider/directory.rs | 61 +- valhalla-graphtile/src/tile_provider/mod.rs | 39 + .../src/tile_provider/tarball.rs | 59 +- 5 files changed, 536 insertions(+), 311 deletions(-) diff --git a/valhalla-graphtile/src/extract/mod.rs b/valhalla-graphtile/src/extract/mod.rs index 3732c4b..4433d51 100644 --- a/valhalla-graphtile/src/extract/mod.rs +++ b/valhalla-graphtile/src/extract/mod.rs @@ -1,26 +1,27 @@ //! # Offline tile extract generation //! -//! This module writes *sub-extracts* of a full Valhalla tile extract: -//! smaller, fully spec-compliant tarballs which contain a subset of the source tiles. +//! This module writes *sub-extracts* of an existing set of Valhalla graph tiles: +//! fully spec-compliant tarballs which contain a subset of the source tiles. +//! The source can be any [`TileBytesSource`](crate::tile_provider::TileBytesSource), +//! such as an extract tarball +//! ([`TarballTileProvider`](crate::tile_provider::TarballTileProvider)) +//! or a tile directory +//! ([`DirectoryGraphTileProvider`](crate::tile_provider::DirectoryGraphTileProvider)). //! The output is interchangeable with `valhalla_build_extract` output, //! so it is consumable by both stock Valhalla //! and [`TarballTileProvider`](crate::tile_provider::TarballTileProvider). //! //! # Writing //! -//! The writer streams the archive in a single pass: +//! [`write_extract`] streams the archive in a single pass: //! an `index.bin` entry first //! (see [`serialize_index_bin`](crate::tile_provider::serialize_index_bin)), //! followed by the graph tiles sorted ascending by (level, tile index) //! (note that this differs from the `Ord` impl of [`GraphId`](crate::GraphId), //! which deliberately orders levels in reverse). //! Output is deterministic: -//! the same source extract and tile set always produce byte-identical output. -//! See [`ExtractLayout`] and [`write_extract_from_source`] for the details. +//! the same source tiles and tile set always produce byte-identical output. mod writer; -pub use writer::{ - ExtractLayout, ExtractLayoutError, ExtractWriteError, TileBytesSource, write_extract, - write_extract_from_source, -}; +pub use writer::{ExtractLayoutError, ExtractWriteError, write_extract}; diff --git a/valhalla-graphtile/src/extract/writer.rs b/valhalla-graphtile/src/extract/writer.rs index d73945a..7b8abcf 100644 --- a/valhalla-graphtile/src/extract/writer.rs +++ b/valhalla-graphtile/src/extract/writer.rs @@ -1,15 +1,18 @@ //! Manifest-first, single-pass streaming tar writer for tile extracts. //! //! The tar format makes every output offset computable *before writing a single byte*, -//! given a manifest of `(GraphId, size)` pairs -//! (which we always have, because the source provider's tile index stores sizes): +//! given a manifest of `(GraphId, size)` pairs. +//! [`write_extract`] gets the manifest from the [`TileBytesSource`] +//! (a tarball provider reads the sizes from its `index.bin`, +//! and a directory provider reads them from the file system), +//! so a caller never gives sizes by hand: //! //! - `index.bin` is one [`TileIndexBinEntry`] per tile, //! padded to a 512-byte boundary, preceded by its 512-byte tar header. //! - Each tile contributes a 512-byte tar header plus its payload padded to 512 bytes. //! -//! [`ExtractLayout`] is that pure computation; -//! [`write_extract_from_source`] streams the corresponding archive into any [`Write`], +//! [`ExtractLayout`] is that pure computation. +//! The writer then streams the corresponding archive into any [`Write`], //! with no `Seek`, no temp files, and no buffering of the whole archive. //! //! # Archive layout @@ -21,7 +24,7 @@ //! Writing it first satisfies every consumer //! and keeps HTTP range-request serving possible. //! 2. Graph tiles follow, sorted ascending by (level, tile index), -//! with spec-compliant entry paths (`level/xxx/yyy….gph`) +//! with spec-compliant entry paths (see [`GraphId::relative_path`]) //! so that index-less consumers scanning the tar can still resolve them. //! Note that this canonical order differs from the `Ord` impl of [`GraphId`], //! which deliberately orders levels in reverse. @@ -35,7 +38,7 @@ use crate::GraphId; use crate::graph_id::InvalidGraphIdError; use crate::tile_provider::{ GraphTileProviderError, INDEX_BIN_ENTRY_NAME, InvalidIndexBinError, TAR_BLOCK_SIZE, - TarballTileProvider, TileIndexBinEntry, serialize_index_bin, + TileBytesSource, TileIndexBinEntry, serialize_index_bin, }; use std::io::Write; use tar::{EntryType, Header}; @@ -46,25 +49,33 @@ use thiserror::Error; #[expect(clippy::cast_possible_truncation, reason = "512 fits in every usize")] const ZERO_BLOCK: [u8; TAR_BLOCK_SIZE as usize] = [0; TAR_BLOCK_SIZE as usize]; -/// An error in an extract manifest. +/// The length of the end-of-archive marker: two zero blocks. +const END_OF_ARCHIVE_LEN: u64 = 2 * TAR_BLOCK_SIZE; + +/// The extension of graph tile files. +const TILE_EXTENSION: &str = "gph"; + +/// An error in the set of tiles of an extract. /// -/// The layout finds these errors before the writer writes any bytes. +/// The writer finds these errors before it writes any bytes. #[derive(Debug, Error, PartialEq, Eq)] pub enum ExtractLayoutError { /// An extract containing no tiles is meaningless /// (and its `index.bin` would be malformed). - #[error("the extract manifest contains no tiles")] + #[error("the extract contains no tiles")] EmptyManifest, - /// The manifest contains the same tile more than once. - #[error("the extract manifest contains tile {0} more than once")] + /// The same tile is requested more than once. + /// + /// Two graph IDs which differ only in the feature index are the same tile. + #[error("the extract contains tile {0} more than once")] DuplicateTile(GraphId), - /// The manifest gives a tile a size of zero. + /// The source gives a tile a size of zero. /// /// A graph tile always contains at least a header, - /// so a size of zero is always an error in the manifest. - #[error("the extract manifest gives tile {0} a size of zero")] + /// so a size of zero is always an error in the source. + #[error("the source gives tile {0} a size of zero")] EmptyTile(GraphId), - /// The manifest contains an ID which cannot be indexed. + /// A tile cannot be indexed. #[error("invalid index.bin: {0}")] InvalidIndexBin(#[from] InvalidIndexBinError), } @@ -74,37 +85,39 @@ pub enum ExtractLayoutError { /// `E` is the error type of the [`TileBytesSource`]. #[derive(Debug, Error)] pub enum ExtractWriteError { - /// The manifest is not valid (see [`ExtractLayout::from_manifest`]). - #[error("invalid manifest: {0}")] + /// The requested tiles do not make a valid extract. + #[error("invalid extract: {0}")] Layout(#[from] ExtractLayoutError), - /// A requested tile is not in the source extract. + /// A requested tile is not in the source. #[error("tile {0} is not in the source extract")] MissingTile(GraphId), - /// A graph ID in the manifest cannot be converted into a tar entry path. + /// A requested graph ID cannot be converted into a tar entry path. #[error("invalid graph ID: {0}")] InvalidGraphId(#[from] InvalidGraphIdError), - /// The byte source produced a tile whose length differs from the manifest size. + /// The source produced a tile whose length differs from the size that it reported. /// /// Sizes must match exactly: - /// every downstream offset was computed from the manifest before any bytes were written, + /// every downstream offset was computed from the reported sizes before any bytes were written, /// so a mismatch would silently corrupt the rest of the archive. + /// This can happen if the source changes during the write + /// (for example, if a tile file in a tile directory is replaced). #[error( - "the byte source produced {actual} bytes for tile {graph_id}, but the manifest size is {expected}" + "the source produced {actual} bytes for tile {graph_id}, but it reported a size of {expected} bytes" )] SizeMismatch { /// The base graph ID of the offending tile. graph_id: GraphId, - /// The size recorded in the manifest. + /// The size that the source reported before the write. expected: u32, /// The length of the bytes actually produced by the source. actual: usize, }, - /// The byte source failed to produce a tile. - #[error("the byte source failed to produce tile {graph_id}: {source}")] + /// The source failed to give the size or the bytes of a tile. + #[error("the source failed to produce tile {graph_id}: {source}")] Source { /// The base graph ID of the tile. graph_id: GraphId, - /// The error from the byte source. + /// The error from the source. source: E, }, /// Writing to the destination failed. @@ -112,45 +125,63 @@ pub enum ExtractWriteError { Io(#[from] std::io::Error), } -/// A source of tile bytes for extract writing. +/// Writes a tile extract which contains the tiles of `tile_ids` into the destination. /// -/// The writer core ([`write_extract_from_source`]) is generic over this trait, -/// so any byte lookup can drive it: -/// a memory-mapped tarball in production, or an in-memory map in tests. +/// Each ID is normalized to its base tile ID (see [`GraphId::tile_base_id`]), +/// and every tile is copied verbatim from the source. +/// The source can be any [`TileBytesSource`], +/// such as a [`TarballTileProvider`](crate::tile_provider::TarballTileProvider) +/// or a [`DirectoryGraphTileProvider`](crate::tile_provider::DirectoryGraphTileProvider). /// -/// The callback style (rather than returning a slice) -/// lets implementations hand out bytes borrowed from per-call state, -/// such as a tile pointer into a memory map. -pub trait TileBytesSource { - /// The error type of this source. - type Error: core::error::Error + 'static; - - /// Calls `process` with the raw bytes of the tile containing the given graph ID. - /// - /// # Errors - /// - /// This fails if the tile does not exist in the source, - /// or if fetching its bytes fails. - fn with_tile_bytes(&self, graph_id: GraphId, process: F) -> Result - where - F: FnOnce(&[u8]) -> T; -} - -impl TileBytesSource for TarballTileProvider { - type Error = GraphTileProviderError; +/// The archive is streamed sequentially in a single pass +/// (no `Seek`, no temp files, no buffering of the whole archive): +/// the `index.bin` entry first, +/// then one entry per tile in canonical archive order +/// (ascending by level, then tile index), +/// and finally the two-block end-of-archive marker. +/// The order of `tile_ids` does not matter. +/// Output is byte-deterministic: +/// the same tiles always produce byte-identical output. +/// +/// Returns the total number of bytes written. +/// +/// # Errors +/// +/// This fails before it writes any bytes if `tile_ids` is empty, +/// contains the same tile more than once, +/// or contains a tile which is not in the source. +/// It fails during the write if the source fails to produce a tile, +/// if a tile's length differs from the size that the source reported, +/// or if writing to the destination fails. +/// +/// The destination contents are unspecified after an error; +/// streaming destinations cannot roll back partially written entries. +pub fn write_extract( + source: &S, + tile_ids: &[GraphId], + dest: W, +) -> Result> +where + S: TileBytesSource, + W: Write, +{ + let manifest = tile_ids + .iter() + .map(|graph_id| { + let base_id = graph_id.tile_base_id(); + match source.tile_size(base_id) { + Ok(Some(size)) => Ok((base_id, size)), + Ok(None) => Err(ExtractWriteError::MissingTile(base_id)), + Err(source) => Err(ExtractWriteError::Source { + graph_id: base_id, + source, + }), + } + }) + .collect::, _>>()?; - #[inline] - fn with_tile_bytes(&self, graph_id: GraphId, process: F) -> Result - where - F: FnOnce(&[u8]) -> T, - { - let tile_pointer = self.get_pointer_for_tile_containing(graph_id)?; - // SAFETY: The provider checks at construction that every index range lies inside the memory map. - // We assume that the underlying memory map has not changed in a way that would render the tile invalid - // (see the safety discussion on TarballTileProvider). - let tile_bytes = unsafe { tile_pointer.as_tile_bytes() }; - Ok(process(tile_bytes)) - } + let layout = ExtractLayout::from_manifest(&manifest)?; + write_layout(&layout, source, dest) } /// The precomputed layout of an extract tarball. @@ -166,7 +197,7 @@ impl TileBytesSource for TarballTileProvider { /// so the index entries are in the exact order /// their tiles appear in the archive. #[derive(Debug)] -pub struct ExtractLayout { +struct ExtractLayout { /// One index entry per tile, in canonical archive order. entries: Vec, /// The total length of the archive in bytes, including the end-of-archive marker. @@ -186,7 +217,7 @@ impl ExtractLayout { /// This fails if the manifest is empty, contains duplicate tiles, /// contains a tile with a size of zero, /// or contains an ID which is not a valid base tile ID. - pub fn from_manifest(manifest: &[(GraphId, u32)]) -> Result { + fn from_manifest(manifest: &[(GraphId, u32)]) -> Result { if manifest.is_empty() { return Err(ExtractLayoutError::EmptyManifest); } @@ -194,42 +225,40 @@ impl ExtractLayout { let mut sorted = manifest.to_vec(); sorted.sort_unstable_by_key(|(graph_id, _)| (graph_id.level(), graph_id.tile_id())); - // The index entry for a tile records the offset of its *payload*, - // so the first tile payload lands one header block past the index entry: - // the index header block, the padded index payload, and the tile's own header block. + // The archive is a sequence of entries, and `index.bin` is the first entry (at offset 0). + // `next_entry_offset` is the offset of the header block of the next entry; + // the payload of that entry starts one block later. let index_len = (size_of::() * sorted.len()) as u64; - let mut header_offset = TAR_BLOCK_SIZE + index_len.next_multiple_of(TAR_BLOCK_SIZE); + let mut next_entry_offset = entry_len(index_len); let mut entries = Vec::with_capacity(sorted.len()); - for (i, (graph_id, size)) in sorted.iter().enumerate() { + for (i, &(graph_id, size)) in sorted.iter().enumerate() { // Equal (level, tile index) pairs are adjacent after the sort. - if i > 0 && sorted[i - 1].0 == *graph_id { - return Err(ExtractLayoutError::DuplicateTile(*graph_id)); + if i > 0 && sorted[i - 1].0 == graph_id { + return Err(ExtractLayoutError::DuplicateTile(graph_id)); } - if *size == 0 { - return Err(ExtractLayoutError::EmptyTile(*graph_id)); + if size == 0 { + return Err(ExtractLayoutError::EmptyTile(graph_id)); } - let payload_offset = header_offset + TAR_BLOCK_SIZE; - entries.push(TileIndexBinEntry::new(*graph_id, payload_offset, *size)?); - header_offset = payload_offset + u64::from(*size).next_multiple_of(TAR_BLOCK_SIZE); + let payload_offset = next_entry_offset + TAR_BLOCK_SIZE; + entries.push(TileIndexBinEntry::new(graph_id, payload_offset, size)?); + next_entry_offset += entry_len(u64::from(size)); } Ok(Self { entries, - // The archive ends with two zero blocks (the standard end-of-archive marker). - total_len: header_offset + 2 * TAR_BLOCK_SIZE, + total_len: next_entry_offset + END_OF_ARCHIVE_LEN, }) } /// The index entries of the archive, in canonical archive order /// (ascending by level, then tile index). /// - /// Serializing these with - /// [`serialize_index_bin`](crate::tile_provider::serialize_index_bin) + /// Serializing these with [`serialize_index_bin`] /// yields exactly the `index.bin` payload of the archive. #[inline] - pub fn index_entries(&self) -> &[TileIndexBinEntry] { + fn index_entries(&self) -> &[TileIndexBinEntry] { &self.entries } @@ -237,86 +266,34 @@ impl ExtractLayout { /// including the end-of-archive marker. /// /// This is always a multiple of 512, - /// and is known before a single byte is written, - /// which makes it possible to set a `Content-Length` up front - /// when streaming an extract over HTTP. + /// and is known before a single byte is written. #[inline] - pub fn total_len(&self) -> u64 { + fn total_len(&self) -> u64 { self.total_len } } -/// Writes a tile extract sourced from an existing tarball. -/// -/// This is a convenience wrapper over [`write_extract_from_source`]: -/// the manifest is built by resolving each ID in `tile_ids` -/// (normalized to its base tile ID) -/// against the source provider's tile index, -/// and tile bytes are read from the provider's memory map. -/// See [`write_extract_from_source`] for the full output format contract. -/// -/// Returns the total number of bytes written. -/// -/// # Errors -/// -/// This fails if `tile_ids` is empty, contains duplicate tiles, -/// or references a tile which is not present in the source tarball. -/// It also fails if writing to the destination fails. -pub fn write_extract( - provider: &TarballTileProvider, - tile_ids: &[GraphId], - dest: W, -) -> Result { - let manifest = tile_ids - .iter() - .map(|graph_id| { - let base_id = graph_id.tile_base_id(); - provider - .tile_size(base_id) - .map(|size| (base_id, size)) - .ok_or(ExtractWriteError::MissingTile(base_id)) - }) - .collect::, _>>()?; - - write_extract_from_source(&manifest, provider, dest) +/// The length of a tar entry with a payload of `payload_len` bytes: +/// a header block, then the payload padded to a whole number of blocks. +const fn entry_len(payload_len: u64) -> u64 { + TAR_BLOCK_SIZE + payload_len.next_multiple_of(TAR_BLOCK_SIZE) } -/// Writes a tile extract as a tar archive into the destination. -/// -/// The archive is streamed sequentially in a single pass -/// (no `Seek`, no temp files, no buffering of the whole archive): -/// the `index.bin` entry first, -/// then one entry per manifest tile in canonical archive order -/// (ascending by level, then tile index), -/// and finally the two-block end-of-archive marker. -/// The manifest may be in any order; the writer sorts its own copy. -/// -/// Output is byte-deterministic: -/// the same manifest and tile bytes always produce byte-identical output. -/// -/// Returns the total number of bytes written, -/// which always equals [`ExtractLayout::total_len`] for the same manifest. -/// -/// # Errors -/// -/// This fails if the layout cannot be computed from the manifest -/// (see [`ExtractLayout::from_manifest`]), -/// if the source fails to produce a tile, -/// if a tile's length differs from its manifest size, -/// or if writing to the destination fails. -/// -/// The destination contents are unspecified after an error; -/// streaming destinations cannot roll back partially written entries. -pub fn write_extract_from_source( - manifest: &[(GraphId, u32)], +/// Streams the archive of a layout into the destination, +/// and returns the number of bytes written. +fn write_layout( + layout: &ExtractLayout, source: &S, - mut dest: W, + dest: W, ) -> Result> where S: TileBytesSource, W: Write, { - let layout = ExtractLayout::from_manifest(manifest)?; + let mut dest = CountingWriter { + inner: dest, + position: 0, + }; let index_bytes = serialize_index_bin(layout.index_entries()).map_err(ExtractLayoutError::from)?; @@ -325,7 +302,7 @@ where for entry in layout.index_entries() { // Infallible in practice: the layout was constructed from valid base tile IDs. let graph_id = entry.graph_id().map_err(ExtractLayoutError::from)?; - let path = tar_entry_path(graph_id)?; + let path = graph_id.relative_path(TILE_EXTENSION)?; let expected = entry.size(); source @@ -337,21 +314,33 @@ where actual: tile_bytes.len(), }); } - write_entry(&mut dest, &path, tile_bytes).map_err(ExtractWriteError::from) + let payload_offset = write_entry(&mut dest, &path, tile_bytes)?; + debug_assert_eq!( + payload_offset, + entry.offset(), + "the payload of {graph_id} is not where index.bin says it is" + ); + Ok(()) }) .map_err(|source| ExtractWriteError::Source { graph_id, source })??; } - // The standard end-of-archive marker: two zero blocks. dest.write_all(&ZERO_BLOCK)?; dest.write_all(&ZERO_BLOCK)?; dest.flush()?; - Ok(layout.total_len()) + debug_assert_eq!(dest.position, layout.total_len()); + Ok(dest.position) } /// Writes a single tar entry: a 512-byte header, the payload, and zero padding to a block boundary. -fn write_entry(dest: &mut W, path: &str, payload: &[u8]) -> std::io::Result<()> { +/// +/// Returns the offset of the payload in the archive. +fn write_entry( + dest: &mut CountingWriter, + path: &str, + payload: &[u8], +) -> std::io::Result { // Determinism requires every header field to be fixed: // ustar format, mtime 0, uid/gid 0 (with empty user/group names), mode 644. let mut header = Header::new_ustar(); @@ -365,35 +354,49 @@ fn write_entry(dest: &mut W, path: &str, payload: &[u8]) -> std::io::R header.set_cksum(); dest.write_all(header.as_bytes())?; + let payload_offset = dest.position; dest.write_all(payload)?; let padding = payload.len().next_multiple_of(ZERO_BLOCK.len()) - payload.len(); - dest.write_all(&ZERO_BLOCK[..padding]) + dest.write_all(&ZERO_BLOCK[..padding])?; + + Ok(payload_offset) } -/// Constructs the spec-compliant tar entry path for a tile. +/// A writer which counts the bytes written through it. /// -/// The first component is the bare level digit; -/// the remaining components are the tile index, -/// zero-padded to the level's width and split into 3-digit groups -/// (e.g. `2/000/762/485.gph`). -/// Tar entry names always use `/` separators, regardless of platform. -fn tar_entry_path(graph_id: GraphId) -> Result { - let path = graph_id.file_path("gph")?; - let components: Vec<_> = path - .iter() - .map(|component| component.to_string_lossy()) - .collect(); - Ok(components.join("/")) +/// The writer uses the count to check the precomputed layout, +/// and to return the number of bytes that it actually wrote. +struct CountingWriter { + inner: W, + /// The number of bytes written so far, which is the offset of the next byte in the archive. + position: u64, +} + +impl Write for CountingWriter { + #[inline] + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let written = self.inner.write(buf)?; + self.position += written as u64; + Ok(written) + } + + #[inline] + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } } #[cfg(test)] mod tests { use super::*; use crate::test_fixtures; - use crate::tile_provider::parse_index_bin; - use std::collections::HashMap; + use crate::tile_provider::{DirectoryGraphTileProvider, TarballTileProvider, parse_index_bin}; + use proptest::collection::btree_map; + use proptest::{prop_assert, prop_assert_eq, proptest}; + use std::collections::{HashMap, HashSet}; use std::io::Read; + use std::num::NonZeroUsize; use tar::Archive; /// Helper to construct a base tile [`GraphId`]. @@ -401,15 +404,23 @@ mod tests { GraphId::try_from_components(level, tile_id, 0).expect("valid tile ID in test") } - /// An in-memory byte source for driving the writer without any I/O. - struct MemorySource(HashMap>); + /// An in-memory tile source for driving the writer without any I/O. + #[derive(Default)] + struct MemorySource { + tiles: HashMap>, + /// Sizes which `tile_size` reports in place of the real sizes. + reported_sizes: HashMap, + /// Tiles which the source has, but whose bytes it fails to produce. + unreadable: HashSet, + } - /// The error of a [`MemorySource`] when it does not contain a tile. + /// The error of a [`MemorySource`] when it cannot produce a tile. #[derive(Debug, Error, PartialEq, Eq)] - #[error("the tile is not in memory")] - struct NotInMemory; + #[error("the tile is not readable")] + struct Unreadable; impl MemorySource { + /// A source which contains one tile of the given size for each manifest entry. fn from_manifest(manifest: &[(GraphId, u32)]) -> Self { let tiles = manifest .iter() @@ -421,21 +432,43 @@ mod tests { (*graph_id, vec![fill; *size as usize]) }) .collect(); - Self(tiles) + Self { + tiles, + ..Self::default() + } + } + + fn tile_ids(&self) -> Vec { + self.tiles.keys().copied().collect() } } impl TileBytesSource for MemorySource { - type Error = NotInMemory; + type Error = Unreadable; + + fn tile_size(&self, graph_id: GraphId) -> Result, Self::Error> { + let graph_id = graph_id.tile_base_id(); + if let Some(size) = self.reported_sizes.get(&graph_id) { + return Ok(Some(*size)); + } + Ok(self + .tiles + .get(&graph_id) + .map(|tile_bytes| u32::try_from(tile_bytes.len()).expect("test tiles are small"))) + } fn with_tile_bytes(&self, graph_id: GraphId, process: F) -> Result where F: FnOnce(&[u8]) -> T, { - self.0 - .get(&graph_id.tile_base_id()) + let graph_id = graph_id.tile_base_id(); + if self.unreadable.contains(&graph_id) { + return Err(Unreadable); + } + self.tiles + .get(&graph_id) .map(|tile_bytes| process(tile_bytes)) - .ok_or(NotInMemory) + .ok_or(Unreadable) } } @@ -453,42 +486,56 @@ mod tests { ] } - #[test] - fn test_tar_entry_path() { - // Levels 0 and 1 pad the tile index to 6 digits (two components); - // level 2 pads to 9 digits (three components). - assert_eq!(tar_entry_path(tile(0, 3015)).unwrap(), "0/003/015.gph"); - assert_eq!(tar_entry_path(tile(0, 49)).unwrap(), "0/000/049.gph"); - assert_eq!(tar_entry_path(tile(1, 47_701)).unwrap(), "1/047/701.gph"); - assert_eq!(tar_entry_path(tile(1, 0)).unwrap(), "1/000/000.gph"); - assert_eq!( - tar_entry_path(tile(2, 762_485)).unwrap(), - "2/000/762/485.gph" - ); - assert_eq!(tar_entry_path(tile(2, 2)).unwrap(), "2/000/000/002.gph"); - assert_eq!( - tar_entry_path(tile(2, 1_036_799)).unwrap(), - "2/001/036/799.gph" - ); + /// Reads the archive with the `tar` crate (an independent tar reader), + /// and returns the path and payload offset of each entry, in archive order. + fn read_entries(archive_bytes: &[u8]) -> Vec<(String, u64)> { + Archive::new(archive_bytes) + .entries() + .expect("the archive is readable") + .map(|entry| { + let entry = entry.expect("every entry is readable"); + let path = entry + .path() + .expect("every entry path is valid") + .to_string_lossy() + .into_owned(); + (path, entry.raw_file_position()) + }) + .collect() + } + + /// Reads the `index.bin` payload, which is the first entry of the archive. + fn read_index_bytes(archive_bytes: &[u8]) -> Vec { + let mut index_bytes = Vec::new(); + Archive::new(archive_bytes) + .entries() + .expect("the archive is readable") + .next() + .expect("the archive has entries") + .expect("the index entry is readable") + .read_to_end(&mut index_bytes) + .expect("the index payload is readable"); + index_bytes } #[test] fn test_layout_offsets() { + const B: u64 = TAR_BLOCK_SIZE; + let layout = ExtractLayout::from_manifest(&scrambled_manifest()) .expect("the manifest is well-formed"); - // 5 tiles × 16 bytes = 80 bytes of index, padded to one block; - // with the index header, the first tile header lands at 1024. - // - // All offsets were computed by hand from the layout rules: - // each tile is a 512-byte header plus its payload padded to 512. + // Each entry is a header block, then its payload padded to whole blocks. + // The index has 5 tiles × 16 bytes = 80 bytes, so the index entry is 2 blocks, + // and the first tile header is at block 2. + // The payload of each tile starts one block after its header. let expected = [ // (graph_id, payload offset, size), in canonical (level, tile index) order - (tile(0, 3015), 1536, 512), // exactly one block; no padding - (tile(1, 47_700), 2560, 511), // padded to one block - (tile(1, 47_701), 3584, 513), // padded to two blocks - (tile(2, 762_484), 5120, 1024), // exactly two blocks; no padding - (tile(2, 762_485), 6656, 100), // padded to one block + (tile(0, 3015), 3 * B, 512), // header at block 2; payload is 1 block + (tile(1, 47_700), 5 * B, 511), // header at block 4; payload is padded to 1 block + (tile(1, 47_701), 7 * B, 513), // header at block 6; payload is padded to 2 blocks + (tile(2, 762_484), 10 * B, 1024), // header at block 9; payload is 2 blocks + (tile(2, 762_485), 13 * B, 100), // header at block 12; payload is padded to 1 block ]; assert_eq!(layout.index_entries().len(), expected.len()); @@ -498,9 +545,8 @@ mod tests { assert_eq!(entry.size(), size); } - // Last tile payload ends at 6656 + 512 (padded), plus the 1024-byte end-of-archive marker. - assert_eq!(layout.total_len(), 7168 + 1024); - assert!(layout.total_len().is_multiple_of(512)); + // The last payload ends at block 14, and the end-of-archive marker is 2 blocks. + assert_eq!(layout.total_len(), 16 * B); } #[test] @@ -541,34 +587,65 @@ mod tests { ); } + #[test] + fn test_write_rejects_empty_tile_list() { + let source = MemorySource::default(); + assert!(matches!( + write_extract(&source, &[], Vec::new()), + Err(ExtractWriteError::Layout(ExtractLayoutError::EmptyManifest)) + )); + } + + #[test] + fn test_write_rejects_the_same_tile_twice() { + let source = MemorySource::from_manifest(&[(tile(1, 47_701), 100)]); + // These IDs differ only in the feature index, so they are the same tile. + let with_feature_index = + GraphId::try_from_components(1, 47_701, 7).expect("valid graph ID in test"); + assert!(matches!( + write_extract(&source, &[tile(1, 47_701), with_feature_index], Vec::new()), + Err(ExtractWriteError::Layout(ExtractLayoutError::DuplicateTile(graph_id))) + if graph_id == tile(1, 47_701) + )); + } + + #[test] + fn test_write_rejects_missing_tiles() { + let source = MemorySource::from_manifest(&[(tile(0, 3015), 100)]); + let mut dest = Vec::new(); + assert!(matches!( + write_extract(&source, &[tile(0, 3015), tile(2, 0)], &mut dest), + Err(ExtractWriteError::MissingTile(graph_id)) if graph_id == tile(2, 0) + )); + assert!(dest.is_empty(), "a missing tile is found before any write"); + } + #[test] fn test_write_rejects_size_mismatch() { - let manifest = [(tile(0, 3015), 100)]; - // The source produces 99 bytes where the manifest promises 100. - let source = MemorySource::from_manifest(&[(tile(0, 3015), 99)]); + // The source produces 99 bytes, but it reports a size of 100. + let mut source = MemorySource::from_manifest(&[(tile(0, 3015), 99)]); + source.reported_sizes.insert(tile(0, 3015), 100); - let result = write_extract_from_source(&manifest, &source, Vec::new()); assert!(matches!( - result, + write_extract(&source, &[tile(0, 3015)], Vec::new()), Err(ExtractWriteError::SizeMismatch { + graph_id, expected: 100, actual: 99, - .. - }) + }) if graph_id == tile(0, 3015) )); } #[test] - fn test_write_rejects_missing_tiles() { - let manifest = [(tile(0, 3015), 100)]; - let source = MemorySource(HashMap::new()); + fn test_write_reports_source_errors() { + let mut source = MemorySource::from_manifest(&[(tile(0, 3015), 100)]); + source.unreadable.insert(tile(0, 3015)); - let result = write_extract_from_source(&manifest, &source, Vec::new()); assert!(matches!( - result, + write_extract(&source, &[tile(0, 3015)], Vec::new()), Err(ExtractWriteError::Source { graph_id, - source: NotInMemory + source: Unreadable }) if graph_id == tile(0, 3015) )); } @@ -579,27 +656,16 @@ mod tests { let source = MemorySource::from_manifest(&manifest); let mut archive_bytes = Vec::new(); - let written = write_extract_from_source(&manifest, &source, &mut archive_bytes) - .expect("the manifest and source are well-formed"); + let written = write_extract(&source, &source.tile_ids(), &mut archive_bytes) + .expect("the source is well-formed"); assert_eq!(written, archive_bytes.len() as u64); - assert!(archive_bytes.len().is_multiple_of(512)); + assert!(archive_bytes.len().is_multiple_of(ZERO_BLOCK.len())); // The archive must list index.bin first, // then the tiles in canonical (level, tile index) order under spec-compliant paths. - let mut archive = Archive::new(archive_bytes.as_slice()); - let entry_paths: Vec = archive - .entries() - .expect("the archive is readable") - .map(|entry| { - entry - .expect("every entry is readable") - .path() - .expect("every entry path is valid") - .to_string_lossy() - .into_owned() - }) - .collect(); + let entries = read_entries(&archive_bytes); + let entry_paths: Vec<&str> = entries.iter().map(|(path, _)| path.as_str()).collect(); assert_eq!( entry_paths, vec![ @@ -612,30 +678,18 @@ mod tests { ] ); - // The index payload starts right after the index header - // and describes every tile payload in the archive. - let mut archive = Archive::new(archive_bytes.as_slice()); - let mut index_bytes = Vec::new(); - archive - .entries() - .expect("the archive is readable") - .next() - .expect("the archive has entries") - .expect("the index entry is readable") - .read_to_end(&mut index_bytes) - .expect("the index payload is readable"); - + // The index says where each payload starts, + // and an independent tar reader finds each payload at the same offset. + let index_bytes = read_index_bytes(&archive_bytes); let index_entries = parse_index_bin(&index_bytes).expect("the index parses"); assert_eq!(index_entries.len(), manifest.len()); - for index_entry in index_entries { - let offset = index_entry.offset(); - assert!(offset > 0); - assert!(offset.is_multiple_of(512)); + for (index_entry, (_, payload_offset)) in index_entries.iter().zip(&entries[1..]) { + assert_eq!(index_entry.offset(), *payload_offset); // The payload at the indexed offset is byte-identical to the source tile. let graph_id = index_entry.graph_id().expect("the entry ID is valid"); - let start = usize::try_from(offset).expect("the offset fits in memory"); + let start = usize::try_from(index_entry.offset()).expect("the offset fits in memory"); let payload = &archive_bytes[start..start + index_entry.size() as usize]; source .with_tile_bytes(graph_id, |tile_bytes| assert_eq!(payload, tile_bytes)) @@ -645,28 +699,60 @@ mod tests { #[test] fn test_determinism() { - let manifest = scrambled_manifest(); - let source = MemorySource::from_manifest(&manifest); + let source = MemorySource::from_manifest(&scrambled_manifest()); + let tile_ids = source.tile_ids(); let mut first = Vec::new(); - write_extract_from_source(&manifest, &source, &mut first).expect("first write succeeds"); + write_extract(&source, &tile_ids, &mut first).expect("first write succeeds"); - let mut second = Vec::new(); - // A reordered manifest must not change the output either: + // A reordered tile list must not change the output either: // the writer sorts its own copy. - let mut reversed = manifest.clone(); + let mut reversed = tile_ids; reversed.reverse(); - write_extract_from_source(&reversed, &source, &mut second).expect("second write succeeds"); + let mut second = Vec::new(); + write_extract(&source, &reversed, &mut second).expect("second write succeeds"); assert_eq!(first, second); } + proptest! { + #[test] + fn test_archive_matches_independent_reader( + // Unique tiles (the map keys) on the three standard levels, + // with sizes that cover the padding cases around one and several blocks. + tiles in btree_map((0u8..=2, 0u64..4050), 1u32..=2049, 1..32) + ) { + let manifest: Vec<(GraphId, u32)> = tiles + .into_iter() + .map(|((level, tile_id), size)| (tile(level, tile_id), size)) + .collect(); + let source = MemorySource::from_manifest(&manifest); + + let mut archive_bytes = Vec::new(); + let written = write_extract(&source, &source.tile_ids(), &mut archive_bytes) + .expect("the source is well-formed"); + let layout = ExtractLayout::from_manifest(&manifest).expect("the manifest is well-formed"); + prop_assert_eq!(written, archive_bytes.len() as u64); + prop_assert_eq!(written, layout.total_len()); + + // Every index entry points at the payload that the tar reader finds, + // under the path of the same tile. + let entries = read_entries(&archive_bytes); + prop_assert_eq!(entries.len(), manifest.len() + 1); + for (index_entry, (path, payload_offset)) in layout.index_entries().iter().zip(&entries[1..]) { + let graph_id = index_entry.graph_id().expect("the entry ID is valid"); + prop_assert_eq!(index_entry.offset(), *payload_offset); + prop_assert_eq!(path, &graph_id.relative_path(TILE_EXTENSION).expect("valid tile")); + prop_assert!(index_entry.offset().is_multiple_of(TAR_BLOCK_SIZE)); + } + } + } + // Tragically, memory maps are not supported by Miri! #[cfg(not(miri))] #[test] fn test_andorra_round_trip() { - let fixture_path = test_fixtures::andorra_tiles_tar(); - let source_provider = TarballTileProvider::new_readonly(fixture_path) + let source_provider = TarballTileProvider::new_readonly(test_fixtures::andorra_tiles_tar()) .expect("Unable to init the source tile provider"); // A hand-picked subset of the fixture, spanning all three standard levels, @@ -680,14 +766,7 @@ mod tests { ]; // A fixture tile which is *not* part of the subset. let excluded = tile(2, 763_925); - assert!(source_provider.tile_size(excluded).is_some()); - - // A tile which is not in the fixture at all. - let not_in_fixture = tile(2, 0); - assert!(matches!( - write_extract(&source_provider, &[tile(0, 3015), not_in_fixture], Vec::new()), - Err(ExtractWriteError::MissingTile(graph_id)) if graph_id == not_in_fixture - )); + assert!(source_provider.tile_size(excluded).unwrap().is_some()); let mut archive_bytes = Vec::new(); let written = write_extract(&source_provider, &tile_ids, &mut archive_bytes) @@ -711,11 +790,10 @@ mod tests { .expect("the generated extract opens cleanly"); for graph_id in tile_ids { - let base_id = graph_id.tile_base_id(); extract_provider - .with_tile_bytes(base_id, |extract_bytes| { + .with_tile_bytes(graph_id, |extract_bytes| { source_provider - .with_tile_bytes(base_id, |source_bytes| { + .with_tile_bytes(graph_id, |source_bytes| { assert_eq!(extract_bytes, source_bytes); }) .expect("the tile is in the source"); @@ -724,6 +802,43 @@ mod tests { } // Tiles outside the subset did not leak into the extract. - assert!(extract_provider.tile_size(excluded).is_none()); + assert!(extract_provider.tile_size(excluded).unwrap().is_none()); + } + + // Tragically, memory maps are not supported by Miri! + #[cfg(not(miri))] + #[test] + fn test_directory_and_tarball_sources_write_the_same_extract() { + // The tarball fixture was made from the directory fixture with valhalla_build_extract, + // so both sources contain the same tiles, and the extracts must be byte-identical. + let tarball = TarballTileProvider::new_readonly(test_fixtures::andorra_tiles_tar()) + .expect("Unable to init the tarball tile provider"); + let directory = DirectoryGraphTileProvider::new( + test_fixtures::andorra_tiles_dir(), + NonZeroUsize::new(1).unwrap(), + ); + + // Every tile of the fixture. + let tile_ids: Vec = tarball.tile_ids().copied().collect(); + assert!(tile_ids.len() > 1); + + let mut from_tarball = Vec::new(); + write_extract(&tarball, &tile_ids, &mut from_tarball) + .expect("the extract writes cleanly from the tarball"); + let mut from_directory = Vec::new(); + write_extract(&directory, &tile_ids, &mut from_directory) + .expect("the extract writes cleanly from the directory"); + assert_eq!(from_tarball, from_directory); + + // A tile which is in neither fixture fails the same way for both sources. + for result in [ + write_extract(&tarball, &[tile(0, 3015), tile(2, 0)], Vec::new()), + write_extract(&directory, &[tile(0, 3015), tile(2, 0)], Vec::new()), + ] { + assert!(matches!( + result, + Err(ExtractWriteError::MissingTile(graph_id)) if graph_id == tile(2, 0) + )); + } } } diff --git a/valhalla-graphtile/src/tile_provider/directory.rs b/valhalla-graphtile/src/tile_provider/directory.rs index 25ad3dc..1cee9b1 100644 --- a/valhalla-graphtile/src/tile_provider/directory.rs +++ b/valhalla-graphtile/src/tile_provider/directory.rs @@ -4,7 +4,7 @@ use crate::graph_tile::{GraphTileBuildError, GraphTileBuilder}; use crate::graph_tile::{GraphTileView, handles::OwnedGraphTileHandle}; use crate::spatial::bbox_with_center; use crate::tile_hierarchy::STANDARD_LEVELS; -use crate::tile_provider::{GraphTileProvider, GraphTileProviderError, LockTable}; +use crate::tile_provider::{GraphTileProvider, GraphTileProviderError, LockTable, TileBytesSource}; use geo::{CoordFloat, Point}; use lru::LruCache; use num_traits::FromPrimitive; @@ -126,6 +126,57 @@ impl DirectoryGraphTileProvider { fn path_for_graph_id(&self, graph_id: GraphId) -> Result { Ok(self.base_directory.join(graph_id.file_path("gph")?)) } + + /// Reads all bytes of the tile file for the given base graph ID. + /// + /// The caller must hold the lock for the graph ID, + /// so that the read cannot overlap a call to [`DirectoryGraphTileProvider::overwrite_tile`]. + fn read_tile_bytes(&self, base_graph_id: GraphId) -> Result, GraphTileProviderError> { + let path = self.path_for_graph_id(base_graph_id)?; + // NOTE: Does not handle compressed tiles + std::fs::read(path).map_err(|e| match e.kind() { + ErrorKind::NotFound => GraphTileProviderError::TileDoesNotExist, + _ => GraphTileProviderError::IoError(e), + }) + } +} + +impl TileBytesSource for DirectoryGraphTileProvider { + type Error = GraphTileProviderError; + + /// The size comes from the file system metadata, so this reads no tile data. + fn tile_size(&self, graph_id: GraphId) -> Result, Self::Error> { + let base_graph_id = graph_id.tile_base_id(); + let size = match std::fs::metadata(self.path_for_graph_id(base_graph_id)?) { + Ok(metadata) if metadata.is_file() => metadata.len(), + // Something other than a tile file (such as a directory) has the tile's path. + Ok(_) => return Ok(None), + Err(e) if e.kind() == ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e.into()), + }; + + u32::try_from(size) + .map(Some) + .map_err(|_| GraphTileProviderError::TileTooLarge { + graph_id: base_graph_id, + size, + }) + } + + fn with_tile_bytes(&self, graph_id: GraphId, process: F) -> Result + where + F: FnOnce(&[u8]) -> T, + { + let base_graph_id = graph_id.tile_base_id(); + let lock = self.lock_table.lock_for(base_graph_id); + let _guard = lock.lock(); + + // This reads the file directly, not through the tile cache: + // a caller that copies tiles reads each tile once, + // and caching those tiles would evict the tiles that other readers use. + let tile_bytes = self.read_tile_bytes(base_graph_id)?; + Ok(process(&tile_bytes)) + } } impl GraphTileProvider for DirectoryGraphTileProvider { @@ -160,13 +211,7 @@ impl GraphTileProvider for DirectoryGraphTileProvider { .map_err(|e| GraphTileProviderError::PoisonedCacheLock(e.to_string()))?; let tile = cache .try_get_or_insert(base_graph_id, || { - let path = self.path_for_graph_id(base_graph_id)?; - // Open the file and read all bytes into a buffer - // NOTE: Does not handle compressed tiles - let data = std::fs::read(path).map_err(|e| match e.kind() { - ErrorKind::NotFound => GraphTileProviderError::TileDoesNotExist, - _ => GraphTileProviderError::IoError(e), - })?; + let data = self.read_tile_bytes(base_graph_id)?; let tile = OwnedGraphTileHandle::try_from(data)?; Ok::<_, GraphTileProviderError>(Arc::new(tile)) }) diff --git a/valhalla-graphtile/src/tile_provider/mod.rs b/valhalla-graphtile/src/tile_provider/mod.rs index 06dd56d..c7b41ea 100644 --- a/valhalla-graphtile/src/tile_provider/mod.rs +++ b/valhalla-graphtile/src/tile_provider/mod.rs @@ -55,6 +55,45 @@ pub enum GraphTileProviderError { InvalidIndexBin(#[from] InvalidIndexBinError), #[error("Unsupported tile version; this may or may not be compatible.")] UnsupportedTileVersion, + #[error("the tile file for {graph_id} has {size} bytes, which is more than a tile can have")] + TileTooLarge { graph_id: GraphId, size: u64 }, +} + +/// Access to the raw bytes of graph tiles, without decoding them. +/// +/// This is for code which copies tiles verbatim, +/// such as the extract writer ([`write_extract`](crate::extract::write_extract)). +/// [`GraphTileProvider`] gives decoded views of tiles, +/// and a view does not keep the bytes it was decoded from. +/// +/// The tarball and directory providers implement this trait, +/// so tiles can be copied from either an extract tarball or a tile directory. +pub trait TileBytesSource { + /// The error type of this source. + type Error: core::error::Error + 'static; + + /// The size in bytes of the tile containing the given graph ID, + /// or [`None`] if the source does not contain the tile. + /// + /// # Errors + /// + /// This fails if the source cannot find out whether it contains the tile + /// (for example, because of an I/O error). + fn tile_size(&self, graph_id: GraphId) -> Result, Self::Error>; + + /// Calls `process` with the raw bytes of the tile containing the given graph ID. + /// + /// The callback style (rather than returning a slice) + /// lets implementations hand out bytes borrowed from per-call state, + /// such as a tile pointer into a memory map. + /// + /// # Errors + /// + /// This fails if the tile does not exist in the source, + /// or if fetching its bytes fails. + fn with_tile_bytes(&self, graph_id: GraphId, process: F) -> Result + where + F: FnOnce(&[u8]) -> T; } /// A generic trait for all graph tile providers. diff --git a/valhalla-graphtile/src/tile_provider/tarball.rs b/valhalla-graphtile/src/tile_provider/tarball.rs index 586eb00..1bf6007 100644 --- a/valhalla-graphtile/src/tile_provider/tarball.rs +++ b/valhalla-graphtile/src/tile_provider/tarball.rs @@ -1,4 +1,4 @@ -use super::{GraphTileProvider, GraphTileProviderError}; +use super::{GraphTileProvider, GraphTileProviderError, TileBytesSource}; use crate::GraphId; use crate::graph_tile::handles::MmapGraphTileHandle; use crate::graph_tile::{ @@ -190,16 +190,49 @@ impl TarballTileProvider { self.tile_index.keys() } - /// The size in bytes of the tile containing this graph ID, - /// or [`None`] if the tile is not part of the tarball archive. + /// Gets a pointer for the tile containing this graph ID. + /// + /// # Errors /// - /// The size comes straight from the tarball's `index.bin`, - /// so no tile data is accessed. + /// If the tile is not part of the tarball archive, + /// this returns [`GraphTileProviderError::TileDoesNotExist`]. + #[inline] + fn tile_pointer(&self, graph_id: GraphId) -> Result { + let base_graph_id = graph_id.tile_base_id(); + let Some(offsets) = self.tile_index.get(&base_graph_id) else { + return Err(GraphTileProviderError::TileDoesNotExist); + }; + + Ok(MmapTilePointer { + mmap: self.mmap.clone(), + offsets: *offsets, + }) + } +} + +impl TileBytesSource for TarballTileProvider { + type Error = GraphTileProviderError; + + /// The size comes from the `index.bin` of the tarball, so this reads no tile data. #[inline] - pub fn tile_size(&self, graph_id: GraphId) -> Option { - self.tile_index + fn tile_size(&self, graph_id: GraphId) -> Result, Self::Error> { + Ok(self + .tile_index .get(&graph_id.tile_base_id()) - .map(|tile_offset| tile_offset.size) + .map(|tile_offset| tile_offset.size)) + } + + #[inline] + fn with_tile_bytes(&self, graph_id: GraphId, process: F) -> Result + where + F: FnOnce(&[u8]) -> T, + { + let tile_pointer = self.tile_pointer(graph_id)?; + // SAFETY: The provider checks at construction that every index range lies inside the memory map. + // We assume that the underlying memory map has not changed in a way that would render the tile invalid + // (see the safety discussion on TarballTileProvider). + let tile_bytes = unsafe { tile_pointer.as_tile_bytes() }; + Ok(process(tile_bytes)) } } @@ -322,15 +355,7 @@ impl TarballTileProvider { &self, graph_id: GraphId, ) -> Result { - let base_graph_id = graph_id.tile_base_id(); - let Some(offsets) = self.tile_index.get(&base_graph_id) else { - return Err(GraphTileProviderError::TileDoesNotExist); - }; - - Ok(MmapTilePointer { - mmap: self.mmap.clone(), - offsets: *offsets, - }) + self.tile_pointer(graph_id) } } From 708b60b3a484420b710328f80d708ef27e3e0a72 Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Wed, 21 Oct 2026 10:33:15 -0700 Subject: [PATCH 10/37] Write valinor-cli extracts atomically, and from tarballs or directories - valinor-cli extract now accepts a tile directory as the source, as well as an extract tarball, because write_extract takes any tile source. - The command writes the extract to a temporary file in the output directory, syncs it, and then renames it to the output path. So the output path is either the complete extract or what it was before the command. Before this change, the command truncated the output first, and only tried to remove a partial output after an error. The temporary file gets the permissions that File::create gives (0o666 minus the umask), not the 0o600 default of tempfile. tempfile moves from the dev-dependencies to the dependencies. - --tile uses GraphId::parse_tile, and get-edge parses its graph ID with the FromStr implementation of GraphId. The two local parsers are removed. - ExtractWriteError::MissingTile now says "not in the source", because the source can be a directory. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/extract/writer.rs | 2 +- valinor-cli/Cargo.toml | 4 +- valinor-cli/README.md | 9 +- valinor-cli/src/commands/extract.rs | 289 ++++++++++++++--------- valinor-cli/src/commands/get_edge.rs | 27 +-- valinor-cli/src/main.rs | 30 +++ 6 files changed, 217 insertions(+), 144 deletions(-) diff --git a/valhalla-graphtile/src/extract/writer.rs b/valhalla-graphtile/src/extract/writer.rs index 7b8abcf..8e7aadd 100644 --- a/valhalla-graphtile/src/extract/writer.rs +++ b/valhalla-graphtile/src/extract/writer.rs @@ -89,7 +89,7 @@ pub enum ExtractWriteError { #[error("invalid extract: {0}")] Layout(#[from] ExtractLayoutError), /// A requested tile is not in the source. - #[error("tile {0} is not in the source extract")] + #[error("tile {0} is not in the source")] MissingTile(GraphId), /// A requested graph ID cannot be converted into a tar entry path. #[error("invalid graph ID: {0}")] diff --git a/valinor-cli/Cargo.toml b/valinor-cli/Cargo.toml index 576e8f8..1961b76 100644 --- a/valinor-cli/Cargo.toml +++ b/valinor-cli/Cargo.toml @@ -12,12 +12,10 @@ rust-version.workspace = true anyhow = { workspace = true } clap = { workspace = true, features = ["derive", "env"] } serde_json = { workspace = true } +tempfile = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter"] } valhalla-graphtile = { workspace = true, features = ["serde"] } -[dev-dependencies] -tempfile = { workspace = true } - [lints] workspace = true diff --git a/valinor-cli/README.md b/valinor-cli/README.md index 544ac6b..df299af 100644 --- a/valinor-cli/README.md +++ b/valinor-cli/README.md @@ -2,7 +2,7 @@ A CLI multi-tool for quickly inspecting Valhalla graphs. Most subcommands take a Valhalla JSON configuration as input -(`extract` is the exception; it takes a tile extract tarball directly). +(`extract` is the exception; it takes a tile extract tarball or a tile directory directly). This is primarily meant as a low-level debugging tool for Valinor developers. If this looks useful to you, it probably is; otherwise, it's probably not. @@ -20,13 +20,18 @@ Then you'll get some JSON output on your terminal with the details. ## Example: writing an offline sub-extract You can write a smaller, fully spec-compliant extract -which contains only some tiles of a source extract (each tile is `level/tile_index`): +which contains only some tiles of a source (each tile is `level/tile_id`). +The source can be a tile extract tarball or a tile directory: ```shell cargo run --package valinor-cli -- extract valhalla-graphtile/fixtures/andorra-tiles.tar andorra-sub.tar --tile 0/3015,1/47701,2/763926 +cargo run --package valinor-cli -- extract valhalla-graphtile/fixtures/andorra-tiles andorra-sub.tar --tile 0/3015,1/47701,2/763926 ``` A one-line summary (tile count and output size) is printed on stderr. The output is interchangeable with `valhalla_build_extract` output. +The command writes to a temporary file in the output directory, +and renames it to the output path only when the extract is complete. +So the output path never contains a partial extract. Run the `help` subcommand for more details (everything is automatically documented via clap). diff --git a/valinor-cli/src/commands/extract.rs b/valinor-cli/src/commands/extract.rs index 04efec5..39e023e 100644 --- a/valinor-cli/src/commands/extract.rs +++ b/valinor-cli/src/commands/extract.rs @@ -1,46 +1,52 @@ -//! The `extract` subcommand: writes an offline sub-extract of a tile extract tarball. +//! The `extract` subcommand: writes an offline sub-extract of a tile extract or tile directory. use std::{ - fs, - fs::File, - io::{BufWriter, Write}, - path::PathBuf, + io::BufWriter, + num::NonZeroUsize, + path::{Path, PathBuf}, }; +#[cfg(unix)] +use std::{fs::Permissions, os::unix::fs::PermissionsExt}; + use anyhow::Context; use clap::Args; +use tempfile::NamedTempFile; use valhalla_graphtile::GraphId; use valhalla_graphtile::extract::write_extract; -use valhalla_graphtile::tile_provider::TarballTileProvider; +use valhalla_graphtile::tile_provider::{ + DirectoryGraphTileProvider, TarballTileProvider, TileBytesSource, +}; -/// Write an offline sub-extract of a tile extract tarball +/// Write an offline sub-extract of a tile extract tarball or a tile directory /// -/// Copies the given graph tiles of the source extract into a new, fully spec-compliant +/// Copies the given graph tiles of the source into a new, fully spec-compliant /// extract tarball. Every tile in the output is byte-identical to its source tile, /// and the output is interchangeable with `valhalla_build_extract` output, /// so it is consumable by both stock Valhalla and Valinor. /// -/// Unlike the other subcommands, this takes the tile extract tarball path -/// directly rather than a Valhalla JSON config: extraction reads an extract -/// file, not a configured tile hierarchy. +/// Unlike the other subcommands, this takes the source path directly +/// rather than a Valhalla JSON config: extraction reads a set of tiles, +/// not a configured tile hierarchy. #[derive(Args, Debug)] pub(crate) struct ExtractArgs { - /// Path to the source tile extract tarball (e.g. tiles.tar) + /// Path to the source: a tile extract tarball (e.g. tiles.tar) + /// or a tile directory (e.g. the `tile_dir` of a Valhalla config) source: PathBuf, /// Path to write the output extract tarball to output: PathBuf, - /// A tile to copy, as level/tile_index (e.g. `--tile 2/763926`) + /// A tile to copy, as level/tile_id (e.g. `--tile 2/763926`) /// /// Give the option once for each tile, /// or give a comma-separated list (e.g. `--tile 0/3015,1/47701`). #[arg( long = "tile", - value_name = "LEVEL/TILE_INDEX", + value_name = "LEVEL/TILE_ID", required = true, value_delimiter = ',', - value_parser = parse_tile + value_parser = GraphId::parse_tile )] pub(crate) tiles: Vec, } @@ -52,60 +58,81 @@ struct ExtractSummary { bytes_written: u64, } -/// Parses a `level/tile_index` tile argument into a base tile ID. -fn parse_tile(input: &str) -> Result { - let Some((level, tile_index)) = input.split_once('/') else { - return Err(format!("'{input}' is not in the level/tile_index form")); +/// Writes the sub-extract from a tile directory or a tarball, as the source path requires. +fn write_sub_extract(args: &ExtractArgs) -> anyhow::Result { + let bytes_written = if args.source.is_dir() { + // The writer reads each tile once, so a cache is of no use. + let provider = DirectoryGraphTileProvider::new(args.source.clone(), NonZeroUsize::MIN); + write_atomically(&provider, &args.tiles, &args.output)? + } else { + let provider = TarballTileProvider::new_readonly(&args.source).with_context(|| { + format!( + "Failed to open source tile extract at {}", + args.source.display() + ) + })?; + write_atomically(&provider, &args.tiles, &args.output)? }; - let level: u8 = level - .trim() - .parse() - .map_err(|_| format!("the level in '{input}' is not a valid number"))?; - let tile_index: u64 = tile_index - .trim() - .parse() - .map_err(|_| format!("the tile index in '{input}' is not a valid number"))?; - GraphId::try_from_components(level, tile_index, 0) - .map_err(|e| format!("'{input}' is not a valid tile: {e}")) + + Ok(ExtractSummary { + tile_count: args.tiles.len(), + bytes_written, + }) } -/// Writes the sub-extract, and removes the partial output if the write fails. -fn write_sub_extract(args: &ExtractArgs) -> anyhow::Result { - let provider = TarballTileProvider::new_readonly(&args.source).with_context(|| { +/// Writes the extract to a temporary file next to `output`, +/// then renames the temporary file to `output`. +/// +/// A partial tarball is worse than no tarball +/// (it may open, but with tiles silently missing or truncated). +/// The rename is atomic, so `output` is never a partial tarball: +/// it is either the complete extract, or what it was before the call. +/// If the write fails, the temporary file is removed. +fn write_atomically(source: &S, tiles: &[GraphId], output: &Path) -> anyhow::Result +where + S: TileBytesSource, + S::Error: Send + Sync, +{ + // The temporary file must be on the same file system as the output, + // or the rename cannot be atomic. + let output_dir = output + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + + let mut builder = tempfile::Builder::new(); + builder.prefix(".valinor-extract-").suffix(".tar.partial"); + // Temporary files are only readable by the owner by default. + // Give the output the same permissions as a file from `File::create`. + #[cfg(unix)] + builder.permissions(Permissions::from_mode(0o666)); + let temp_file: NamedTempFile = builder.tempfile_in(output_dir).with_context(|| { format!( - "Failed to open source tile extract at {}", - args.source.display() + "Failed to create a temporary file in {}", + output_dir.display() ) })?; - let file = File::create(&args.output).with_context(|| { + let mut dest = BufWriter::new(temp_file); + let bytes_written = + write_extract(source, tiles, &mut dest).context("Failed to write the extract")?; + let temp_file = dest + .into_inner() + .map_err(|e| e.into_error()) + .context("Failed to flush the output extract")?; + // Make sure that the data is on disk before the rename makes the file visible. + temp_file + .as_file() + .sync_all() + .context("Failed to sync the output extract to disk")?; + temp_file.persist(output).with_context(|| { format!( - "Failed to create the output extract at {}", - args.output.display() + "Failed to move the output extract into place at {}", + output.display() ) })?; - let write_result = (|| { - let mut dest = BufWriter::new(file); - let bytes_written = write_extract(&provider, &args.tiles, &mut dest) - .context("Failed to write the extract")?; - dest.flush().context("Failed to flush the output extract")?; - Ok(bytes_written) - })(); - - match write_result { - Ok(bytes_written) => Ok(ExtractSummary { - tile_count: args.tiles.len(), - bytes_written, - }), - Err(error) => { - // A partial tarball is worse than no tarball - // (it may open, but with tiles silently missing or truncated), - // so clean up on a best-effort basis. - let _ = fs::remove_file(&args.output); - Err(error) - } - } + Ok(bytes_written) } /// Runs the `extract` subcommand, and prints a one-line summary on stderr. @@ -128,94 +155,128 @@ pub(crate) fn run(args: &ExtractArgs) -> anyhow::Result<()> { #[cfg(test)] mod tests { use super::*; + use std::fs; - /// Path to the Andorra fixture tarball in the sibling crate. - fn andorra_fixture() -> PathBuf { + /// Path to a fixture in the sibling `valhalla-graphtile` crate. + fn fixture(name: &str) -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("..") .join("valhalla-graphtile") .join("fixtures") - .join("andorra-tiles.tar") + .join(name) } - fn tile(level: u8, tile_index: u64) -> GraphId { - GraphId::try_from_components(level, tile_index, 0).expect("valid tile ID in test") + fn tile(level: u8, tile_id: u64) -> GraphId { + GraphId::try_from_components(level, tile_id, 0).expect("valid tile ID in test") } + // Tragically, memory maps are not supported by Miri, + // and the tests below open tarballs with the memory-mapped provider! + #[cfg(not(miri))] #[test] - fn parse_tile_accepts_level_and_tile_index() { - assert_eq!(parse_tile("2/763926"), Ok(tile(2, 763_926))); - assert_eq!(parse_tile(" 0 / 3015 "), Ok(tile(0, 3015))); - } + fn extract_round_trips_from_a_tarball_and_a_directory() { + let temp_dir = tempfile::tempdir().expect("a temporary directory is created"); + let tiles = vec![tile(0, 3015), tile(1, 47_701), tile(2, 763_926)]; - #[test] - fn parse_tile_rejects_malformed_input() { - for (input, expected) in [ - ("2", "is not in the level/tile_index form"), - ("x/763926", "the level in 'x/763926' is not a valid number"), - ("2/abc", "the tile index in '2/abc' is not a valid number"), - // A graph ID with a feature index has three components. - ( - "2/763926/7", - "the tile index in '2/763926/7' is not a valid number", - ), - // Levels have 3 bits, and tile indexes have 22 bits. - ("8/0", "'8/0' is not a valid tile"), - ("0/4194304", "'0/4194304' is not a valid tile"), + let mut outputs = Vec::new(); + for (source, name) in [ + (fixture("andorra-tiles.tar"), "from-tarball.tar"), + (fixture("andorra-tiles"), "from-directory.tar"), ] { - let error = parse_tile(input).expect_err("malformed input is rejected"); - assert!(error.contains(expected), "{input}: {error}"); + let output = temp_dir.path().join(name); + let args = ExtractArgs { + source, + output: output.clone(), + tiles: tiles.clone(), + }; + + let summary = write_sub_extract(&args).expect("the extract writes cleanly"); + assert_eq!(summary.tile_count, tiles.len()); + + let on_disk = fs::metadata(&output).expect("the output exists").len(); + assert_eq!(summary.bytes_written, on_disk); + + // The produced tarball opens with the crate's own reader + // and contains exactly the requested tiles. + let extract = TarballTileProvider::new_readonly(&output) + .expect("the generated extract opens cleanly"); + let mut extracted: Vec = extract.tile_ids().copied().collect(); + extracted.sort_unstable(); + let mut expected = tiles.clone(); + expected.sort_unstable(); + assert_eq!(extracted, expected); + + outputs.push(fs::read(&output).expect("the output is readable")); } + + assert_eq!( + outputs[0], outputs[1], + "the tarball and directory fixtures give the same extract" + ); + + // Only the outputs are in the directory: no temporary files are left. + let mut names: Vec<_> = fs::read_dir(temp_dir.path()) + .expect("the directory is readable") + .map(|entry| entry.expect("the entry is readable").file_name()) + .collect(); + names.sort_unstable(); + assert_eq!(names, ["from-directory.tar", "from-tarball.tar"]); } - // Tragically, memory maps are not supported by Miri, - // and the tests below open tarballs with the memory-mapped provider! #[cfg(not(miri))] #[test] - fn extract_round_trips_through_the_tarball_provider() { + fn failed_extract_keeps_the_previous_output() { let temp_dir = tempfile::tempdir().expect("a temporary directory is created"); let output = temp_dir.path().join("andorra-sub.tar"); - let tiles = vec![tile(0, 3015), tile(1, 47_701), tile(2, 763_926)]; + fs::write(&output, b"previous contents").expect("the previous output is written"); + let args = ExtractArgs { - source: andorra_fixture(), + source: fixture("andorra-tiles.tar"), output: output.clone(), - tiles: tiles.clone(), + // The fixture has no tile 2/0. + tiles: vec![tile(0, 3015), tile(2, 0)], }; - let summary = write_sub_extract(&args).expect("the extract writes cleanly"); - assert_eq!(summary.tile_count, tiles.len()); - - let on_disk = fs::metadata(&output).expect("the output exists").len(); - assert_eq!(summary.bytes_written, on_disk); - - // The produced tarball opens with the crate's own reader - // and contains exactly the requested tiles. - let extract = TarballTileProvider::new_readonly(&output) - .expect("the generated extract opens cleanly"); - let mut extracted: Vec = extract.tile_ids().copied().collect(); - extracted.sort_unstable(); - let mut expected = tiles; - expected.sort_unstable(); - assert_eq!(extracted, expected); + let error = write_sub_extract(&args).expect_err("a missing tile fails the extract"); + assert!( + format!("{error:#}").contains("is not in the source"), + "{error:#}" + ); + assert_eq!( + fs::read(&output).expect("the previous output still exists"), + b"previous contents" + ); + assert_eq!( + fs::read_dir(temp_dir.path()) + .expect("the directory is readable") + .count(), + 1, + "the temporary file is removed" + ); } + #[cfg(unix)] #[cfg(not(miri))] #[test] - fn failed_extract_removes_the_partial_output() { + fn output_has_the_permissions_of_a_new_file() { let temp_dir = tempfile::tempdir().expect("a temporary directory is created"); let output = temp_dir.path().join("andorra-sub.tar"); + let reference = temp_dir.path().join("reference"); + fs::File::create(&reference).expect("the reference file is created"); + let args = ExtractArgs { - source: andorra_fixture(), + source: fixture("andorra-tiles.tar"), output: output.clone(), - // The fixture has no tile 2/0. - tiles: vec![tile(0, 3015), tile(2, 0)], + tiles: vec![tile(0, 3015)], }; + write_sub_extract(&args).expect("the extract writes cleanly"); - let error = write_sub_extract(&args).expect_err("a missing tile fails the extract"); - assert!( - format!("{error:#}").contains("is not in the source extract"), - "{error:#}" - ); - assert!(!output.exists(), "the partial output is removed"); + let mode = |path: &Path| { + fs::metadata(path) + .expect("the file exists") + .permissions() + .mode() + }; + assert_eq!(mode(&output), mode(&reference)); } } diff --git a/valinor-cli/src/commands/get_edge.rs b/valinor-cli/src/commands/get_edge.rs index 95d47e7..211cef9 100644 --- a/valinor-cli/src/commands/get_edge.rs +++ b/valinor-cli/src/commands/get_edge.rs @@ -3,7 +3,6 @@ use std::{ fs, path::{Path, PathBuf}, - str::FromStr, }; use anyhow::{Context, anyhow}; @@ -24,28 +23,8 @@ pub(crate) struct GetEdgeArgs { #[arg(env)] valhalla_config: PathBuf, - /// Graph ID (u64) or slash-form level/tile/index - graph_id: String, -} - -fn parse_graph_id(input: &str) -> anyhow::Result { - // Try to parse as a pure integer. - if let Ok(id) = input.parse::() { - return GraphId::try_from_id(id).map_err(|e| anyhow!(e)); - } - - // Try slash-separated level/tile/index - let parts: Vec<&str> = input.split('/').collect(); - if parts.len() == 3 { - let level = u8::from_str(parts[0]).context("invalid level in graph id")?; - let tile_id = u64::from_str(parts[1]).context("invalid tile id in graph id")?; - let index = u64::from_str(parts[2]).context("invalid index in graph id")?; - return GraphId::try_from_components(level, tile_id, index).map_err(|e| anyhow!(e)); - } - - Err(anyhow!( - "Unrecognized graph id format. Use a u64 integer or level/tile/index" - )) + /// Graph ID (u64) or slash-form level/tile_id/index + pub(crate) graph_id: GraphId, } #[derive(Debug, Clone)] @@ -113,7 +92,7 @@ fn pretty_print_edge_info( /// Runs the `get-edge` subcommand. pub(crate) fn run(args: &GetEdgeArgs) -> anyhow::Result<()> { - let gid = parse_graph_id(&args.graph_id)?; + let gid = args.graph_id; let sources = parse_valhalla_data_paths(&args.valhalla_config)?; let traffic_extract = if let Some(path) = sources.traffic_extract { info!(path = path.to_str(), "Using traffic extract"); diff --git a/valinor-cli/src/main.rs b/valinor-cli/src/main.rs index da8b2ad..5bd1fe4 100644 --- a/valinor-cli/src/main.rs +++ b/valinor-cli/src/main.rs @@ -72,6 +72,36 @@ mod tests { ); } + #[test] + fn extract_tiles_reject_malformed_values() { + for input in ["2", "2/763926/7", "2/abc", "8/0"] { + let error = Cli::try_parse_from([ + "valinor-cli", + "extract", + "in.tar", + "out.tar", + "--tile", + input, + ]) + .expect_err("a malformed tile is rejected"); + // The error from GraphId::parse_tile names the bad value. + assert!(error.to_string().contains(input), "{input}: {error}"); + } + } + + #[test] + fn get_edge_accepts_both_graph_id_forms() { + let expected = GraphId::try_from_components(2, 762_485, 42).expect("valid graph ID"); + for input in [expected.value().to_string(), "2/762485/42".to_string()] { + let cli = Cli::try_parse_from(["valinor-cli", "get-edge", "valhalla.json", &input]) + .expect("the arguments parse"); + let Commands::GetEdge(args) = cli.command else { + panic!("expected the get-edge subcommand"); + }; + assert_eq!(args.graph_id, expected); + } + } + #[test] fn extract_tiles_are_required() { assert!(Cli::try_parse_from(["valinor-cli", "extract", "in.tar", "out.tar"]).is_err()); From 241ba8702b0be507e6f74f02d658bf17da31643b Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Wed, 21 Oct 2026 10:36:17 -0700 Subject: [PATCH 11/37] Share the tile file extension and index entry size constants A search for other duplicates in the files of this PR found two more: - The "gph" extension was a literal in the directory provider and a private constant in the extract writer. It is now TILE_FILE_EXTENSION in tile_provider. - The writer calculated size_of::() again, although tarball.rs already defines INDEX_ENTRY_SIZE. That constant is now pub(crate), and the writer uses it. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/extract/writer.rs | 13 +++++-------- valhalla-graphtile/src/tile_provider/directory.rs | 8 ++++++-- valhalla-graphtile/src/tile_provider/mod.rs | 5 ++++- valhalla-graphtile/src/tile_provider/tarball.rs | 2 +- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/valhalla-graphtile/src/extract/writer.rs b/valhalla-graphtile/src/extract/writer.rs index 8e7aadd..5ca66d0 100644 --- a/valhalla-graphtile/src/extract/writer.rs +++ b/valhalla-graphtile/src/extract/writer.rs @@ -37,8 +37,8 @@ use crate::GraphId; use crate::graph_id::InvalidGraphIdError; use crate::tile_provider::{ - GraphTileProviderError, INDEX_BIN_ENTRY_NAME, InvalidIndexBinError, TAR_BLOCK_SIZE, - TileBytesSource, TileIndexBinEntry, serialize_index_bin, + GraphTileProviderError, INDEX_BIN_ENTRY_NAME, INDEX_ENTRY_SIZE, InvalidIndexBinError, + TAR_BLOCK_SIZE, TILE_FILE_EXTENSION, TileBytesSource, TileIndexBinEntry, serialize_index_bin, }; use std::io::Write; use tar::{EntryType, Header}; @@ -52,9 +52,6 @@ const ZERO_BLOCK: [u8; TAR_BLOCK_SIZE as usize] = [0; TAR_BLOCK_SIZE as usize]; /// The length of the end-of-archive marker: two zero blocks. const END_OF_ARCHIVE_LEN: u64 = 2 * TAR_BLOCK_SIZE; -/// The extension of graph tile files. -const TILE_EXTENSION: &str = "gph"; - /// An error in the set of tiles of an extract. /// /// The writer finds these errors before it writes any bytes. @@ -228,7 +225,7 @@ impl ExtractLayout { // The archive is a sequence of entries, and `index.bin` is the first entry (at offset 0). // `next_entry_offset` is the offset of the header block of the next entry; // the payload of that entry starts one block later. - let index_len = (size_of::() * sorted.len()) as u64; + let index_len = (INDEX_ENTRY_SIZE * sorted.len()) as u64; let mut next_entry_offset = entry_len(index_len); let mut entries = Vec::with_capacity(sorted.len()); @@ -302,7 +299,7 @@ where for entry in layout.index_entries() { // Infallible in practice: the layout was constructed from valid base tile IDs. let graph_id = entry.graph_id().map_err(ExtractLayoutError::from)?; - let path = graph_id.relative_path(TILE_EXTENSION)?; + let path = graph_id.relative_path(TILE_FILE_EXTENSION)?; let expected = entry.size(); source @@ -742,7 +739,7 @@ mod tests { for (index_entry, (path, payload_offset)) in layout.index_entries().iter().zip(&entries[1..]) { let graph_id = index_entry.graph_id().expect("the entry ID is valid"); prop_assert_eq!(index_entry.offset(), *payload_offset); - prop_assert_eq!(path, &graph_id.relative_path(TILE_EXTENSION).expect("valid tile")); + prop_assert_eq!(path, &graph_id.relative_path(TILE_FILE_EXTENSION).expect("valid tile")); prop_assert!(index_entry.offset().is_multiple_of(TAR_BLOCK_SIZE)); } } diff --git a/valhalla-graphtile/src/tile_provider/directory.rs b/valhalla-graphtile/src/tile_provider/directory.rs index 1cee9b1..ae9570e 100644 --- a/valhalla-graphtile/src/tile_provider/directory.rs +++ b/valhalla-graphtile/src/tile_provider/directory.rs @@ -4,7 +4,9 @@ use crate::graph_tile::{GraphTileBuildError, GraphTileBuilder}; use crate::graph_tile::{GraphTileView, handles::OwnedGraphTileHandle}; use crate::spatial::bbox_with_center; use crate::tile_hierarchy::STANDARD_LEVELS; -use crate::tile_provider::{GraphTileProvider, GraphTileProviderError, LockTable, TileBytesSource}; +use crate::tile_provider::{ + GraphTileProvider, GraphTileProviderError, LockTable, TILE_FILE_EXTENSION, TileBytesSource, +}; use geo::{CoordFloat, Point}; use lru::LruCache; use num_traits::FromPrimitive; @@ -124,7 +126,9 @@ impl DirectoryGraphTileProvider { /// /// NOTE: This function assumes that the graph ID is already a base ID. fn path_for_graph_id(&self, graph_id: GraphId) -> Result { - Ok(self.base_directory.join(graph_id.file_path("gph")?)) + Ok(self + .base_directory + .join(graph_id.file_path(TILE_FILE_EXTENSION)?)) } /// Reads all bytes of the tile file for the given base graph ID. diff --git a/valhalla-graphtile/src/tile_provider/mod.rs b/valhalla-graphtile/src/tile_provider/mod.rs index c7b41ea..6c73a12 100644 --- a/valhalla-graphtile/src/tile_provider/mod.rs +++ b/valhalla-graphtile/src/tile_provider/mod.rs @@ -26,7 +26,10 @@ use crate::graph_tile::{ pub use directory::DirectoryGraphTileProvider; pub use iterators::GraphSearchResult; use iterators::ResultsWithinRadius; -pub(crate) use tarball::{INDEX_BIN_ENTRY_NAME, TAR_BLOCK_SIZE}; +pub(crate) use tarball::{INDEX_BIN_ENTRY_NAME, INDEX_ENTRY_SIZE, TAR_BLOCK_SIZE}; + +/// The extension of graph tile files, in tile directories and in extract tarballs. +pub(crate) const TILE_FILE_EXTENSION: &str = "gph"; pub use tarball::{ InvalidIndexBinError, TarballTileProvider, TileIndexBinEntry, parse_index_bin, serialize_index_bin, diff --git a/valhalla-graphtile/src/tile_provider/tarball.rs b/valhalla-graphtile/src/tile_provider/tarball.rs index 1bf6007..04b3248 100644 --- a/valhalla-graphtile/src/tile_provider/tarball.rs +++ b/valhalla-graphtile/src/tile_provider/tarball.rs @@ -481,7 +481,7 @@ pub struct TileIndexBinEntry { } /// The size of one [`TileIndexBinEntry`] in bytes. -const INDEX_ENTRY_SIZE: usize = size_of::(); +pub(crate) const INDEX_ENTRY_SIZE: usize = size_of::(); /// The block size of a tar archive. /// From 280c9e334b130c49f0f16373f6ee821a5fc240cc Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Wed, 21 Oct 2026 16:45:04 -0700 Subject: [PATCH 12/37] Cut the code comments and docstrings to the contract Remove the comments that restate the code, tell the design story, or explain a decision that belongs in the PR. Keep the contracts, the `# Errors` sections, the guards, and the facts that the code does not show (for example, the block offsets in test_layout_offsets). The help text of `valinor-cli extract` is shorter. It no longer says why the command takes a path in place of a Valhalla config. This commit changes only comments and docstrings. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/extract/mod.rs | 27 +-- valhalla-graphtile/src/extract/writer.rs | 211 ++++-------------- valhalla-graphtile/src/graph_id.rs | 40 +--- .../src/tile_provider/directory.rs | 10 +- valhalla-graphtile/src/tile_provider/mod.rs | 25 +-- .../src/tile_provider/tarball.rs | 30 +-- valinor-cli/src/commands/extract.rs | 33 +-- valinor-cli/src/commands/mod.rs | 5 +- valinor-cli/src/main.rs | 1 - 9 files changed, 89 insertions(+), 293 deletions(-) diff --git a/valhalla-graphtile/src/extract/mod.rs b/valhalla-graphtile/src/extract/mod.rs index 4433d51..4168f49 100644 --- a/valhalla-graphtile/src/extract/mod.rs +++ b/valhalla-graphtile/src/extract/mod.rs @@ -1,26 +1,9 @@ -//! # Offline tile extract generation +//! Offline tile extracts: tarballs with a subset of the tiles of a +//! [`TileBytesSource`](crate::tile_provider::TileBytesSource). //! -//! This module writes *sub-extracts* of an existing set of Valhalla graph tiles: -//! fully spec-compliant tarballs which contain a subset of the source tiles. -//! The source can be any [`TileBytesSource`](crate::tile_provider::TileBytesSource), -//! such as an extract tarball -//! ([`TarballTileProvider`](crate::tile_provider::TarballTileProvider)) -//! or a tile directory -//! ([`DirectoryGraphTileProvider`](crate::tile_provider::DirectoryGraphTileProvider)). -//! The output is interchangeable with `valhalla_build_extract` output, -//! so it is consumable by both stock Valhalla -//! and [`TarballTileProvider`](crate::tile_provider::TarballTileProvider). -//! -//! # Writing -//! -//! [`write_extract`] streams the archive in a single pass: -//! an `index.bin` entry first -//! (see [`serialize_index_bin`](crate::tile_provider::serialize_index_bin)), -//! followed by the graph tiles sorted ascending by (level, tile index) -//! (note that this differs from the `Ord` impl of [`GraphId`](crate::GraphId), -//! which deliberately orders levels in reverse). -//! Output is deterministic: -//! the same source tiles and tile set always produce byte-identical output. +//! [`write_extract`] writes an extract in the format of `valhalla_build_extract`, +//! so stock Valhalla and [`TarballTileProvider`](crate::tile_provider::TarballTileProvider) +//! can read it. mod writer; diff --git a/valhalla-graphtile/src/extract/writer.rs b/valhalla-graphtile/src/extract/writer.rs index 5ca66d0..a6d1fbf 100644 --- a/valhalla-graphtile/src/extract/writer.rs +++ b/valhalla-graphtile/src/extract/writer.rs @@ -1,38 +1,13 @@ -//! Manifest-first, single-pass streaming tar writer for tile extracts. +//! A streaming tar writer for tile extracts. //! -//! The tar format makes every output offset computable *before writing a single byte*, -//! given a manifest of `(GraphId, size)` pairs. -//! [`write_extract`] gets the manifest from the [`TileBytesSource`] -//! (a tarball provider reads the sizes from its `index.bin`, -//! and a directory provider reads them from the file system), -//! so a caller never gives sizes by hand: +//! The writer computes every offset from the tile sizes before it writes, +//! so it writes the archive in one pass. The archive contains: //! -//! - `index.bin` is one [`TileIndexBinEntry`] per tile, -//! padded to a 512-byte boundary, preceded by its 512-byte tar header. -//! - Each tile contributes a 512-byte tar header plus its payload padded to 512 bytes. -//! -//! [`ExtractLayout`] is that pure computation. -//! The writer then streams the corresponding archive into any [`Write`], -//! with no `Seek`, no temp files, and no buffering of the whole archive. -//! -//! # Archive layout -//! -//! 1. `index.bin` is always the first entry. -//! Valhalla C++ treats it as optional for local tarballs -//! but requires it as the first entry for remotely-served tarballs, -//! and Valinor's own reader requires it unconditionally. -//! Writing it first satisfies every consumer -//! and keeps HTTP range-request serving possible. -//! 2. Graph tiles follow, sorted ascending by (level, tile index), -//! with spec-compliant entry paths (see [`GraphId::relative_path`]) -//! so that index-less consumers scanning the tar can still resolve them. -//! Note that this canonical order differs from the `Ord` impl of [`GraphId`], -//! which deliberately orders levels in reverse. -//! 3. The archive ends with the standard two-block end-of-archive marker. -//! -//! Output is byte-deterministic: -//! tar header metadata is fixed (mtime 0, uid/gid 0, mode 644, ustar format), -//! and the writer sorts its own manifest, so caller order never leaks into the output. +//! 1. `index.bin`, which Valhalla requires as the first entry of a remote tarball. +//! 2. The tiles at the paths of [`GraphId::relative_path`], +//! in ascending (level, tile index) order. +//! This is not the `Ord` order of [`GraphId`]. +//! 3. The two-block end-of-archive marker. use crate::GraphId; use crate::graph_id::InvalidGraphIdError; @@ -44,8 +19,7 @@ use std::io::Write; use tar::{EntryType, Header}; use thiserror::Error; -/// A zeroed block (of [`TAR_BLOCK_SIZE`] bytes) -/// for payload padding and the end-of-archive marker. +/// Zeros for the payload padding and the end-of-archive marker. #[expect(clippy::cast_possible_truncation, reason = "512 fits in every usize")] const ZERO_BLOCK: [u8; TAR_BLOCK_SIZE as usize] = [0; TAR_BLOCK_SIZE as usize]; @@ -57,19 +31,15 @@ const END_OF_ARCHIVE_LEN: u64 = 2 * TAR_BLOCK_SIZE; /// The writer finds these errors before it writes any bytes. #[derive(Debug, Error, PartialEq, Eq)] pub enum ExtractLayoutError { - /// An extract containing no tiles is meaningless - /// (and its `index.bin` would be malformed). + /// The tile list is empty. #[error("the extract contains no tiles")] EmptyManifest, - /// The same tile is requested more than once. + /// The tile list contains a tile more than once. /// - /// Two graph IDs which differ only in the feature index are the same tile. + /// Graph IDs that differ only in the feature index are the same tile. #[error("the extract contains tile {0} more than once")] DuplicateTile(GraphId), /// The source gives a tile a size of zero. - /// - /// A graph tile always contains at least a header, - /// so a size of zero is always an error in the source. #[error("the source gives tile {0} a size of zero")] EmptyTile(GraphId), /// A tile cannot be indexed. @@ -91,13 +61,8 @@ pub enum ExtractWriteError { /// A requested graph ID cannot be converted into a tar entry path. #[error("invalid graph ID: {0}")] InvalidGraphId(#[from] InvalidGraphIdError), - /// The source produced a tile whose length differs from the size that it reported. - /// - /// Sizes must match exactly: - /// every downstream offset was computed from the reported sizes before any bytes were written, - /// so a mismatch would silently corrupt the rest of the archive. - /// This can happen if the source changes during the write - /// (for example, if a tile file in a tile directory is replaced). + /// The source produced a tile whose length differs from the size that it reported, + /// for example because a tile file changed during the write. #[error( "the source produced {actual} bytes for tile {graph_id}, but it reported a size of {expected} bytes" )] @@ -122,37 +87,20 @@ pub enum ExtractWriteError { Io(#[from] std::io::Error), } -/// Writes a tile extract which contains the tiles of `tile_ids` into the destination. -/// -/// Each ID is normalized to its base tile ID (see [`GraphId::tile_base_id`]), -/// and every tile is copied verbatim from the source. -/// The source can be any [`TileBytesSource`], -/// such as a [`TarballTileProvider`](crate::tile_provider::TarballTileProvider) -/// or a [`DirectoryGraphTileProvider`](crate::tile_provider::DirectoryGraphTileProvider). -/// -/// The archive is streamed sequentially in a single pass -/// (no `Seek`, no temp files, no buffering of the whole archive): -/// the `index.bin` entry first, -/// then one entry per tile in canonical archive order -/// (ascending by level, then tile index), -/// and finally the two-block end-of-archive marker. -/// The order of `tile_ids` does not matter. -/// Output is byte-deterministic: -/// the same tiles always produce byte-identical output. +/// Writes an extract with the tiles of `tile_ids` from `source` into `dest`, +/// and returns the number of bytes written. /// -/// Returns the total number of bytes written. +/// The writer copies each tile verbatim, and it does not seek or buffer the archive. +/// The order of `tile_ids` and their feature indexes do not change the output, +/// so the same set of tiles always gives the same bytes. /// /// # Errors /// -/// This fails before it writes any bytes if `tile_ids` is empty, -/// contains the same tile more than once, -/// or contains a tile which is not in the source. -/// It fails during the write if the source fails to produce a tile, -/// if a tile's length differs from the size that the source reported, -/// or if writing to the destination fails. -/// -/// The destination contents are unspecified after an error; -/// streaming destinations cannot roll back partially written entries. +/// Returns an error before any write if `tile_ids` is empty, +/// names a tile more than once, or names a tile that is not in `source`. +/// Returns an error during the write if `source` fails, +/// if the length of a tile differs from its reported size, or if `dest` fails. +/// After an error during the write, `dest` contains a partial archive. pub fn write_extract( source: &S, tile_ids: &[GraphId], @@ -181,39 +129,22 @@ where write_layout(&layout, source, dest) } -/// The precomputed layout of an extract tarball. -/// -/// This is the pure "manifest → offsets" half of the writer: -/// given `(GraphId, size)` pairs, -/// it computes the byte offset of every tile payload in the output archive -/// and the exact total archive length, -/// without performing any I/O. -/// -/// The layout sorts the manifest into the canonical archive order -/// (ascending by level, then tile index), -/// so the index entries are in the exact order -/// their tiles appear in the archive. +/// The payload offset of each tile and the total length of an extract. #[derive(Debug)] struct ExtractLayout { - /// One index entry per tile, in canonical archive order. + /// One index entry for each tile, in archive order. entries: Vec, /// The total length of the archive in bytes, including the end-of-archive marker. total_len: u64, } impl ExtractLayout { - /// Computes the archive layout for a manifest of `(GraphId, size)` pairs. - /// - /// The manifest may be in any order (the layout sorts a copy into canonical order), - /// but every graph ID must be a *base* tile ID - /// (see [`GraphId::tile_base_id`]), - /// and each tile may appear at most once. + /// Computes the layout for `(GraphId, size)` pairs in any order. /// /// # Errors /// - /// This fails if the manifest is empty, contains duplicate tiles, - /// contains a tile with a size of zero, - /// or contains an ID which is not a valid base tile ID. + /// Returns an error if `manifest` is empty, contains a tile more than once, + /// contains a tile with a size of zero, or contains an ID that is not a base tile ID. fn from_manifest(manifest: &[(GraphId, u32)]) -> Result { if manifest.is_empty() { return Err(ExtractLayoutError::EmptyManifest); @@ -222,15 +153,11 @@ impl ExtractLayout { let mut sorted = manifest.to_vec(); sorted.sort_unstable_by_key(|(graph_id, _)| (graph_id.level(), graph_id.tile_id())); - // The archive is a sequence of entries, and `index.bin` is the first entry (at offset 0). - // `next_entry_offset` is the offset of the header block of the next entry; - // the payload of that entry starts one block later. let index_len = (INDEX_ENTRY_SIZE * sorted.len()) as u64; let mut next_entry_offset = entry_len(index_len); let mut entries = Vec::with_capacity(sorted.len()); for (i, &(graph_id, size)) in sorted.iter().enumerate() { - // Equal (level, tile index) pairs are adjacent after the sort. if i > 0 && sorted[i - 1].0 == graph_id { return Err(ExtractLayoutError::DuplicateTile(graph_id)); } @@ -249,21 +176,13 @@ impl ExtractLayout { }) } - /// The index entries of the archive, in canonical archive order - /// (ascending by level, then tile index). - /// - /// Serializing these with [`serialize_index_bin`] - /// yields exactly the `index.bin` payload of the archive. + /// The index entries, in archive order. #[inline] fn index_entries(&self) -> &[TileIndexBinEntry] { &self.entries } - /// The exact total length of the archive in bytes, - /// including the end-of-archive marker. - /// - /// This is always a multiple of 512, - /// and is known before a single byte is written. + /// The total length of the archive in bytes. #[inline] fn total_len(&self) -> u64 { self.total_len @@ -297,7 +216,7 @@ where write_entry(&mut dest, INDEX_BIN_ENTRY_NAME, index_bytes)?; for entry in layout.index_entries() { - // Infallible in practice: the layout was constructed from valid base tile IDs. + // The layout holds only valid base tile IDs, so this does not fail. let graph_id = entry.graph_id().map_err(ExtractLayoutError::from)?; let path = graph_id.relative_path(TILE_FILE_EXTENSION)?; let expected = entry.size(); @@ -330,16 +249,13 @@ where Ok(dest.position) } -/// Writes a single tar entry: a 512-byte header, the payload, and zero padding to a block boundary. -/// -/// Returns the offset of the payload in the archive. +/// Writes a tar entry for `payload` at `path`, and returns the offset of the payload. fn write_entry( dest: &mut CountingWriter, path: &str, payload: &[u8], ) -> std::io::Result { - // Determinism requires every header field to be fixed: - // ustar format, mtime 0, uid/gid 0 (with empty user/group names), mode 644. + // Keep all metadata fixed, so that the same tiles always give the same bytes. let mut header = Header::new_ustar(); header.set_entry_type(EntryType::Regular); header.set_path(path)?; @@ -360,13 +276,10 @@ fn write_entry( Ok(payload_offset) } -/// A writer which counts the bytes written through it. -/// -/// The writer uses the count to check the precomputed layout, -/// and to return the number of bytes that it actually wrote. +/// A writer that counts the bytes written through it. struct CountingWriter { inner: W, - /// The number of bytes written so far, which is the offset of the next byte in the archive. + /// The offset of the next byte in the archive. position: u64, } @@ -396,12 +309,10 @@ mod tests { use std::num::NonZeroUsize; use tar::Archive; - /// Helper to construct a base tile [`GraphId`]. fn tile(level: u8, tile_id: u64) -> GraphId { GraphId::try_from_components(level, tile_id, 0).expect("valid tile ID in test") } - /// An in-memory tile source for driving the writer without any I/O. #[derive(Default)] struct MemorySource { tiles: HashMap>, @@ -417,13 +328,11 @@ mod tests { struct Unreadable; impl MemorySource { - /// A source which contains one tile of the given size for each manifest entry. fn from_manifest(manifest: &[(GraphId, u32)]) -> Self { let tiles = manifest .iter() .map(|(graph_id, size)| { - // Distinctive fill bytes so tiles cannot be confused with each other - // (or with zero padding). + // A nonzero fill for each tile, so that tiles and padding differ. let fill = 0x11 * (1 + u8::try_from(graph_id.tile_id() % 13).expect("13 fits in u8")); (*graph_id, vec![fill; *size as usize]) @@ -469,10 +378,7 @@ mod tests { } } - /// A manifest which is deliberately *not* in canonical order - /// (and not in `GraphId` order either), - /// with sizes exercising all padding cases: - /// an exact block multiple, a one-byte remainder, and a one-byte shortfall. + /// Tiles out of archive order, with sizes at, above, and below a block boundary. fn scrambled_manifest() -> Vec<(GraphId, u32)> { vec![ (tile(2, 762_485), 100), @@ -483,8 +389,7 @@ mod tests { ] } - /// Reads the archive with the `tar` crate (an independent tar reader), - /// and returns the path and payload offset of each entry, in archive order. + /// Returns the path and payload offset of each entry, as the `tar` crate reads them. fn read_entries(archive_bytes: &[u8]) -> Vec<(String, u64)> { Archive::new(archive_bytes) .entries() @@ -501,7 +406,6 @@ mod tests { .collect() } - /// Reads the `index.bin` payload, which is the first entry of the archive. fn read_index_bytes(archive_bytes: &[u8]) -> Vec { let mut index_bytes = Vec::new(); Archive::new(archive_bytes) @@ -522,12 +426,10 @@ mod tests { let layout = ExtractLayout::from_manifest(&scrambled_manifest()) .expect("the manifest is well-formed"); - // Each entry is a header block, then its payload padded to whole blocks. - // The index has 5 tiles × 16 bytes = 80 bytes, so the index entry is 2 blocks, - // and the first tile header is at block 2. - // The payload of each tile starts one block after its header. + // The index entry (a header and 5 × 16 bytes) fills blocks 0 and 1. + // Each payload starts one block after its header. let expected = [ - // (graph_id, payload offset, size), in canonical (level, tile index) order + // (graph_id, payload offset, size) (tile(0, 3015), 3 * B, 512), // header at block 2; payload is 1 block (tile(1, 47_700), 5 * B, 511), // header at block 4; payload is padded to 1 block (tile(1, 47_701), 7 * B, 513), // header at block 6; payload is padded to 2 blocks @@ -596,7 +498,6 @@ mod tests { #[test] fn test_write_rejects_the_same_tile_twice() { let source = MemorySource::from_manifest(&[(tile(1, 47_701), 100)]); - // These IDs differ only in the feature index, so they are the same tile. let with_feature_index = GraphId::try_from_components(1, 47_701, 7).expect("valid graph ID in test"); assert!(matches!( @@ -619,7 +520,6 @@ mod tests { #[test] fn test_write_rejects_size_mismatch() { - // The source produces 99 bytes, but it reports a size of 100. let mut source = MemorySource::from_manifest(&[(tile(0, 3015), 99)]); source.reported_sizes.insert(tile(0, 3015), 100); @@ -659,8 +559,6 @@ mod tests { assert_eq!(written, archive_bytes.len() as u64); assert!(archive_bytes.len().is_multiple_of(ZERO_BLOCK.len())); - // The archive must list index.bin first, - // then the tiles in canonical (level, tile index) order under spec-compliant paths. let entries = read_entries(&archive_bytes); let entry_paths: Vec<&str> = entries.iter().map(|(path, _)| path.as_str()).collect(); assert_eq!( @@ -675,8 +573,6 @@ mod tests { ] ); - // The index says where each payload starts, - // and an independent tar reader finds each payload at the same offset. let index_bytes = read_index_bytes(&archive_bytes); let index_entries = parse_index_bin(&index_bytes).expect("the index parses"); assert_eq!(index_entries.len(), manifest.len()); @@ -684,7 +580,6 @@ mod tests { for (index_entry, (_, payload_offset)) in index_entries.iter().zip(&entries[1..]) { assert_eq!(index_entry.offset(), *payload_offset); - // The payload at the indexed offset is byte-identical to the source tile. let graph_id = index_entry.graph_id().expect("the entry ID is valid"); let start = usize::try_from(index_entry.offset()).expect("the offset fits in memory"); let payload = &archive_bytes[start..start + index_entry.size() as usize]; @@ -702,8 +597,6 @@ mod tests { let mut first = Vec::new(); write_extract(&source, &tile_ids, &mut first).expect("first write succeeds"); - // A reordered tile list must not change the output either: - // the writer sorts its own copy. let mut reversed = tile_ids; reversed.reverse(); let mut second = Vec::new(); @@ -715,8 +608,7 @@ mod tests { proptest! { #[test] fn test_archive_matches_independent_reader( - // Unique tiles (the map keys) on the three standard levels, - // with sizes that cover the padding cases around one and several blocks. + // Unique tiles on levels 0 to 2, with sizes of one to just over four blocks. tiles in btree_map((0u8..=2, 0u64..4050), 1u32..=2049, 1..32) ) { let manifest: Vec<(GraphId, u32)> = tiles @@ -732,8 +624,6 @@ mod tests { prop_assert_eq!(written, archive_bytes.len() as u64); prop_assert_eq!(written, layout.total_len()); - // Every index entry points at the payload that the tar reader finds, - // under the path of the same tile. let entries = read_entries(&archive_bytes); prop_assert_eq!(entries.len(), manifest.len() + 1); for (index_entry, (path, payload_offset)) in layout.index_entries().iter().zip(&entries[1..]) { @@ -752,16 +642,13 @@ mod tests { let source_provider = TarballTileProvider::new_readonly(test_fixtures::andorra_tiles_tar()) .expect("Unable to init the source tile provider"); - // A hand-picked subset of the fixture, spanning all three standard levels, - // deliberately not in canonical order, - // and with one non-zero feature index to test base ID normalization. + // Out of archive order, and with one nonzero feature index. let tile_ids = [ tile(2, 763_926), tile(0, 3015), GraphId::try_from_components(1, 47_701, 42).expect("valid graph ID in test"), tile(2, 762_485), ]; - // A fixture tile which is *not* part of the subset. let excluded = tile(2, 763_925); assert!(source_provider.tile_size(excluded).unwrap().is_some()); @@ -770,14 +657,10 @@ mod tests { .expect("the extract writes cleanly"); assert_eq!(written, archive_bytes.len() as u64); - // Determinism on real data: a second run is byte-identical. let mut second = Vec::new(); write_extract(&source_provider, &tile_ids, &mut second).expect("second write succeeds"); assert_eq!(archive_bytes, second); - // Round-trip: Valinor's own reader opens the extract, - // and every tile is byte-identical to the source. - // The temporary file is removed on drop, even if an assertion fails. let mut extract_file = tempfile::NamedTempFile::new().expect("a temporary file is created"); extract_file .write_all(&archive_bytes) @@ -798,7 +681,6 @@ mod tests { .expect("the tile is in the extract"); } - // Tiles outside the subset did not leak into the extract. assert!(extract_provider.tile_size(excluded).unwrap().is_none()); } @@ -806,8 +688,7 @@ mod tests { #[cfg(not(miri))] #[test] fn test_directory_and_tarball_sources_write_the_same_extract() { - // The tarball fixture was made from the directory fixture with valhalla_build_extract, - // so both sources contain the same tiles, and the extracts must be byte-identical. + // The two fixtures contain the same tiles. let tarball = TarballTileProvider::new_readonly(test_fixtures::andorra_tiles_tar()) .expect("Unable to init the tarball tile provider"); let directory = DirectoryGraphTileProvider::new( @@ -815,7 +696,6 @@ mod tests { NonZeroUsize::new(1).unwrap(), ); - // Every tile of the fixture. let tile_ids: Vec = tarball.tile_ids().copied().collect(); assert!(tile_ids.len() > 1); @@ -827,7 +707,6 @@ mod tests { .expect("the extract writes cleanly from the directory"); assert_eq!(from_tarball, from_directory); - // A tile which is in neither fixture fails the same way for both sources. for result in [ write_extract(&tarball, &[tile(0, 3015), tile(2, 0)], Vec::new()), write_extract(&directory, &[tile(0, 3015), tile(2, 0)], Vec::new()), diff --git a/valhalla-graphtile/src/graph_id.rs b/valhalla-graphtile/src/graph_id.rs index 1ddbe25..5147c74 100644 --- a/valhalla-graphtile/src/graph_id.rs +++ b/valhalla-graphtile/src/graph_id.rs @@ -235,31 +235,25 @@ impl GraphId { GraphId(U64::::new(self.value() & 0x01ff_ffff)) } - /// Parses the base graph ID of a tile from the `level/tile_id` form (e.g. `2/762485`). + /// Parses the base graph ID of a tile from `level/tile_id` (e.g. `2/762485`). /// - /// This is the tile form of [`GraphId::from_str`]: - /// it names a whole tile, so it has no feature index component, - /// and the feature index of the result is zero. /// Whitespace around each component is ignored. /// /// # Errors /// - /// This fails if the input does not have exactly two `/`-separated components, + /// Returns an error if `input` does not have two `/`-separated components, /// if a component is not a non-negative integer, - /// or if the level or tile ID is out of range (see [`GraphId::try_from_components`]). + /// or if the level or the tile ID is out of range. pub fn parse_tile(input: &str) -> Result { let [level, tile_id] = parse_components(input, "level/tile_id", ["level", "tile ID"])?; components_to_graph_id(input, level, tile_id, 0) } - /// Constructs the relative path of the tile file for this graph ID, + /// Returns the relative path of the tile file for this graph ID, /// with `/` separators on every platform (e.g. `2/000/762/485.gph`). /// - /// This is the form that Valhalla uses for tile directories, tar entry names, and tile URLs. - /// The first component is the level; - /// the rest is the tile ID, zero-padded to the width of the level - /// and split into groups of three digits. - /// The feature index is ignored. + /// Valhalla uses this path in tile directories and extract tarballs. + /// The feature index is not part of the path. /// /// # Errors /// @@ -286,8 +280,7 @@ impl GraphId { let padded_id = format!("{tile_id:0>n_digits$}"); let mut path = level_number.to_string(); - // The padded ID is ASCII digits, and its length is a multiple of 3, - // so these byte ranges are always whole groups. + // The padded ID is ASCII, and its length is a multiple of 3. for group_start in (0..n_digits).step_by(3) { path.push('/'); path.push_str(&padded_id[group_start..group_start + 3]); @@ -300,10 +293,7 @@ impl GraphId { Ok(path) } - /// Constructs a relative path for the given tile. - /// - /// This is [`GraphId::relative_path`] as a [`PathBuf`], - /// with the separators of the platform. + /// Returns [`GraphId::relative_path`] as a [`PathBuf`], with the separators of the platform. /// /// # Errors /// @@ -554,15 +544,13 @@ mod tests { ); } - /// Helper to construct a base tile [`GraphId`]. fn tile(level: u8, tile_id: u64) -> GraphId { GraphId::try_from_components(level, tile_id, 0).expect("valid tile ID in test") } #[test] fn test_relative_path() { - // Levels 0 and 1 pad the tile ID to 6 digits (two groups); - // level 2 pads to 9 digits (three groups). + // Levels 0 and 1 pad the tile ID to 6 digits, and level 2 pads it to 9. for (graph_id, expected) in [ (tile(0, 3015), "0/003/015.gph"), (tile(0, 49), "0/000/049.gph"), @@ -572,21 +560,18 @@ mod tests { (tile(2, 1_036_799), "2/001/036/799.gph"), ] { assert_eq!(graph_id.relative_path("gph").as_deref(), Ok(expected)); - // The file path is the same path with the separators of the platform. assert_eq!( graph_id.file_path("gph"), Ok(expected.split('/').collect::()) ); } - // The feature index is not part of the path. let with_index = GraphId::try_from_components(2, 762_485, 42).unwrap(); assert_eq!( with_index.relative_path("gph").as_deref(), Ok("2/000/762/485.gph") ); - // An empty extension adds no dot. assert_eq!( tile(1, 47_701).relative_path("").as_deref(), Ok("1/047/701") @@ -595,8 +580,7 @@ mod tests { #[test] fn test_relative_path_invalid() { - // Level 0 has 45 rows and 90 columns of 4-degree tiles, - // so its largest tile ID is 45 * 90 - 1 = 4049. + // Level 0 has 45 × 90 tiles, so its largest tile ID is 4049. assert_eq!( tile(0, 4049).relative_path("gph").as_deref(), Ok("0/004/049.gph") @@ -605,7 +589,7 @@ mod tests { tile(0, 4050).relative_path("gph"), Err(InvalidGraphIdError::GraphTileId) ); - // Levels 4 to 7 fit in the bit field, but there is no tiling system for them. + // Levels 4 to 7 fit in the bit field, but they have no tiling system. assert_eq!( tile(5, 0).relative_path("gph"), Err(InvalidGraphIdError::Level) @@ -625,7 +609,6 @@ mod tests { expected: "level/tile_id", }; assert_eq!(GraphId::parse_tile("2"), Err(format("2"))); - // A graph ID with a feature index is not a tile. assert_eq!(GraphId::parse_tile("2/763926/7"), Err(format("2/763926/7"))); assert_eq!( GraphId::parse_tile("2/"), @@ -669,7 +652,6 @@ mod tests { let expected = GraphId::try_from_components(2, 762_485, 42).unwrap(); assert_eq!("2/762485/42".parse(), Ok(expected)); assert_eq!(expected.value().to_string().parse(), Ok(expected)); - // Whitespace around a packed value is ignored. assert_eq!( " 16889572344463360 " .parse::() diff --git a/valhalla-graphtile/src/tile_provider/directory.rs b/valhalla-graphtile/src/tile_provider/directory.rs index ae9570e..3f86d2c 100644 --- a/valhalla-graphtile/src/tile_provider/directory.rs +++ b/valhalla-graphtile/src/tile_provider/directory.rs @@ -131,10 +131,9 @@ impl DirectoryGraphTileProvider { .join(graph_id.file_path(TILE_FILE_EXTENSION)?)) } - /// Reads all bytes of the tile file for the given base graph ID. + /// Reads the tile file for `base_graph_id`. /// - /// The caller must hold the lock for the graph ID, - /// so that the read cannot overlap a call to [`DirectoryGraphTileProvider::overwrite_tile`]. + /// The caller must hold the lock for `base_graph_id`. fn read_tile_bytes(&self, base_graph_id: GraphId) -> Result, GraphTileProviderError> { let path = self.path_for_graph_id(base_graph_id)?; // NOTE: Does not handle compressed tiles @@ -153,7 +152,6 @@ impl TileBytesSource for DirectoryGraphTileProvider { let base_graph_id = graph_id.tile_base_id(); let size = match std::fs::metadata(self.path_for_graph_id(base_graph_id)?) { Ok(metadata) if metadata.is_file() => metadata.len(), - // Something other than a tile file (such as a directory) has the tile's path. Ok(_) => return Ok(None), Err(e) if e.kind() == ErrorKind::NotFound => return Ok(None), Err(e) => return Err(e.into()), @@ -175,9 +173,7 @@ impl TileBytesSource for DirectoryGraphTileProvider { let lock = self.lock_table.lock_for(base_graph_id); let _guard = lock.lock(); - // This reads the file directly, not through the tile cache: - // a caller that copies tiles reads each tile once, - // and caching those tiles would evict the tiles that other readers use. + // Bypass the tile cache, so that a copy of many tiles does not evict hot tiles. let tile_bytes = self.read_tile_bytes(base_graph_id)?; Ok(process(&tile_bytes)) } diff --git a/valhalla-graphtile/src/tile_provider/mod.rs b/valhalla-graphtile/src/tile_provider/mod.rs index 6c73a12..6f9d72e 100644 --- a/valhalla-graphtile/src/tile_provider/mod.rs +++ b/valhalla-graphtile/src/tile_provider/mod.rs @@ -62,38 +62,25 @@ pub enum GraphTileProviderError { TileTooLarge { graph_id: GraphId, size: u64 }, } -/// Access to the raw bytes of graph tiles, without decoding them. -/// -/// This is for code which copies tiles verbatim, -/// such as the extract writer ([`write_extract`](crate::extract::write_extract)). -/// [`GraphTileProvider`] gives decoded views of tiles, -/// and a view does not keep the bytes it was decoded from. -/// -/// The tarball and directory providers implement this trait, -/// so tiles can be copied from either an extract tarball or a tile directory. +/// Access to the raw bytes of graph tiles, for code that copies tiles verbatim. pub trait TileBytesSource { /// The error type of this source. type Error: core::error::Error + 'static; - /// The size in bytes of the tile containing the given graph ID, + /// Returns the size in bytes of the tile that contains `graph_id`, /// or [`None`] if the source does not contain the tile. /// /// # Errors /// - /// This fails if the source cannot find out whether it contains the tile - /// (for example, because of an I/O error). + /// Returns an error if the source cannot find out whether it contains the tile. fn tile_size(&self, graph_id: GraphId) -> Result, Self::Error>; - /// Calls `process` with the raw bytes of the tile containing the given graph ID. - /// - /// The callback style (rather than returning a slice) - /// lets implementations hand out bytes borrowed from per-call state, - /// such as a tile pointer into a memory map. + /// Calls `process` with the raw bytes of the tile that contains `graph_id`, + /// and returns its result. /// /// # Errors /// - /// This fails if the tile does not exist in the source, - /// or if fetching its bytes fails. + /// Returns an error if the source does not contain the tile, or if it cannot read the tile. fn with_tile_bytes(&self, graph_id: GraphId, process: F) -> Result where F: FnOnce(&[u8]) -> T; diff --git a/valhalla-graphtile/src/tile_provider/tarball.rs b/valhalla-graphtile/src/tile_provider/tarball.rs index 04b3248..be17343 100644 --- a/valhalla-graphtile/src/tile_provider/tarball.rs +++ b/valhalla-graphtile/src/tile_provider/tarball.rs @@ -103,8 +103,6 @@ impl TarballTileProvider { } let archive_file = File::open(&path)?; - // Get the length before the index is read, - // so that every index entry can be checked against it in a single pass. let archive_len = archive_file.metadata()?.len(); let mut archive = Archive::new(archive_file); let mut entries = archive.entries()?; @@ -153,9 +151,8 @@ impl TarballTileProvider { Arc::new(MmapOptions::new().map_raw_read_only(&file)?) }; - // Tile pointers turn index ranges into slices without a bounds check. - // The index ranges were checked against `archive_len`, - // so the memory map must have exactly that length. + // Tile pointers slice the map without a bounds check, + // so the map must have the length that the index ranges were checked against. if mmap.len() as u64 != archive_len { return Err(GraphTileProviderError::InvalidTarball(format!( "the archive changed size while it was opened ({archive_len} bytes, then {} bytes)", @@ -485,9 +482,7 @@ pub(crate) const INDEX_ENTRY_SIZE: usize = size_of::(); /// The block size of a tar archive. /// -/// Each tar entry is a one-block header, -/// then the data of the entry, padded with zeros to a whole number of blocks. -/// So the data of each entry starts on a block boundary. +/// Each entry is a header block, then its data, padded with zeros to whole blocks. pub(crate) const TAR_BLOCK_SIZE: u64 = 512; /// The name of the tile index entry, which is the first entry of an extract tarball. @@ -541,12 +536,10 @@ impl TileIndexBinEntry { }) } - /// Validates a parsed entry, and returns its graph ID and offset. + /// Checks the entry against an archive of `archive_len` bytes, + /// and returns its graph ID and offset. /// - /// [`parse_index_bin`] does not examine the entry values, - /// so the read path must call this before it uses an entry. - /// The checks are the same as in [`TileIndexBinEntry::new`], - /// and the tile must also lie inside an archive of `archive_len` bytes. + /// Call this before you use an entry from [`parse_index_bin`], which does not check the values. fn validated(&self, archive_len: u64) -> Result<(GraphId, u64), InvalidIndexBinError> { let graph_id = self.graph_id()?; let offset = self.offset.get(); @@ -560,8 +553,7 @@ impl TileIndexBinEntry { /// /// # Errors /// - /// Entries parsed from a foreign `index.bin` may contain arbitrary bit patterns, - /// so this fails if the stored tile ID has non-zero feature index bits. + /// Returns an error if the stored tile ID has nonzero feature index bits. pub fn graph_id(&self) -> Result { // SAFETY: We know that the bit field cannot contain a value // larger than the max allowed value (it's limited to 46 bits). @@ -847,8 +839,7 @@ mod test { #[test] fn test_reject_index_past_end_of_archive() { let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); - // The index says that the tile has 10,000 bytes, - // but the archive only contains 100 bytes of tile data. + // The index gives 10,000 bytes, but the archive has only 100 bytes of tile data. let index_entry = TileIndexBinEntry::new(graph_id, 1536, 10_000).expect("Unable to create index entry"); let index_bytes = serialize_index_bin(core::slice::from_ref(&index_entry)) @@ -968,7 +959,7 @@ mod test { #[test] fn test_index_bin_roundtrip( // Levels are 3 bits, tile IDs are 22 bits, and offsets are counted in 512-byte blocks. - // The offsets are small enough that the end of every tile fits in a u64. + // The offset bound keeps the end of every tile inside a u64. raw_entries in vec( (0u8..=7, 0u64..1 << 22, 1u64..=(u64::MAX - u64::from(u32::MAX)) / 512, 0u32..=u32::MAX), 1..64 @@ -988,8 +979,7 @@ mod test { let parsed = parse_index_bin(bytes).expect("Serialized bytes always parse"); prop_assert_eq!(parsed, entries.as_slice()); - // The read path accepts every entry that the write path creates, - // in an archive which is just long enough for all of the tiles. + // Every entry is valid in an archive that ends where the last tile ends. let archive_len = parsed .iter() .map(|entry| entry.offset() + u64::from(entry.size())) diff --git a/valinor-cli/src/commands/extract.rs b/valinor-cli/src/commands/extract.rs index 39e023e..206e678 100644 --- a/valinor-cli/src/commands/extract.rs +++ b/valinor-cli/src/commands/extract.rs @@ -20,14 +20,8 @@ use valhalla_graphtile::tile_provider::{ /// Write an offline sub-extract of a tile extract tarball or a tile directory /// -/// Copies the given graph tiles of the source into a new, fully spec-compliant -/// extract tarball. Every tile in the output is byte-identical to its source tile, -/// and the output is interchangeable with `valhalla_build_extract` output, -/// so it is consumable by both stock Valhalla and Valinor. -/// -/// Unlike the other subcommands, this takes the source path directly -/// rather than a Valhalla JSON config: extraction reads a set of tiles, -/// not a configured tile hierarchy. +/// The output is an extract tarball in the format of `valhalla_build_extract`, +/// with the given tiles copied byte for byte. #[derive(Args, Debug)] pub(crate) struct ExtractArgs { /// Path to the source: a tile extract tarball (e.g. tiles.tar) @@ -58,7 +52,7 @@ struct ExtractSummary { bytes_written: u64, } -/// Writes the sub-extract from a tile directory or a tarball, as the source path requires. +/// Writes the sub-extract from the tile directory or the tarball at the source path. fn write_sub_extract(args: &ExtractArgs) -> anyhow::Result { let bytes_written = if args.source.is_dir() { // The writer reads each tile once, so a cache is of no use. @@ -80,21 +74,15 @@ fn write_sub_extract(args: &ExtractArgs) -> anyhow::Result { }) } -/// Writes the extract to a temporary file next to `output`, -/// then renames the temporary file to `output`. +/// Writes the extract to a temporary file next to `output`, and then renames it to `output`. /// -/// A partial tarball is worse than no tarball -/// (it may open, but with tiles silently missing or truncated). -/// The rename is atomic, so `output` is never a partial tarball: -/// it is either the complete extract, or what it was before the call. -/// If the write fails, the temporary file is removed. +/// If the write fails, `output` keeps its previous contents. fn write_atomically(source: &S, tiles: &[GraphId], output: &Path) -> anyhow::Result where S: TileBytesSource, S::Error: Send + Sync, { - // The temporary file must be on the same file system as the output, - // or the rename cannot be atomic. + // The rename is atomic only within one file system. let output_dir = output .parent() .filter(|parent| !parent.as_os_str().is_empty()) @@ -102,8 +90,7 @@ where let mut builder = tempfile::Builder::new(); builder.prefix(".valinor-extract-").suffix(".tar.partial"); - // Temporary files are only readable by the owner by default. - // Give the output the same permissions as a file from `File::create`. + // Use the permissions of `File::create`, not the owner-only default of `tempfile`. #[cfg(unix)] builder.permissions(Permissions::from_mode(0o666)); let temp_file: NamedTempFile = builder.tempfile_in(output_dir).with_context(|| { @@ -170,8 +157,7 @@ mod tests { GraphId::try_from_components(level, tile_id, 0).expect("valid tile ID in test") } - // Tragically, memory maps are not supported by Miri, - // and the tests below open tarballs with the memory-mapped provider! + // Tragically, memory maps are not supported by Miri! #[cfg(not(miri))] #[test] fn extract_round_trips_from_a_tarball_and_a_directory() { @@ -196,8 +182,6 @@ mod tests { let on_disk = fs::metadata(&output).expect("the output exists").len(); assert_eq!(summary.bytes_written, on_disk); - // The produced tarball opens with the crate's own reader - // and contains exactly the requested tiles. let extract = TarballTileProvider::new_readonly(&output) .expect("the generated extract opens cleanly"); let mut extracted: Vec = extract.tile_ids().copied().collect(); @@ -214,7 +198,6 @@ mod tests { "the tarball and directory fixtures give the same extract" ); - // Only the outputs are in the directory: no temporary files are left. let mut names: Vec<_> = fs::read_dir(temp_dir.path()) .expect("the directory is readable") .map(|entry| entry.expect("the entry is readable").file_name()) diff --git a/valinor-cli/src/commands/mod.rs b/valinor-cli/src/commands/mod.rs index 60a069a..f23f125 100644 --- a/valinor-cli/src/commands/mod.rs +++ b/valinor-cli/src/commands/mod.rs @@ -1,7 +1,4 @@ -//! The subcommands of the CLI. -//! -//! Each subcommand has its own module, -//! which contains its clap arguments and a `run` function. +//! The subcommands of the CLI, with one module for each subcommand. pub(crate) mod extract; pub(crate) mod get_edge; diff --git a/valinor-cli/src/main.rs b/valinor-cli/src/main.rs index 5bd1fe4..7ed9e99 100644 --- a/valinor-cli/src/main.rs +++ b/valinor-cli/src/main.rs @@ -84,7 +84,6 @@ mod tests { input, ]) .expect_err("a malformed tile is rejected"); - // The error from GraphId::parse_tile names the bad value. assert!(error.to_string().contains(input), "{input}: {error}"); } } From 1fd952d183de9d156327519e21833d9a3e3fe04f Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Wed, 21 Oct 2026 10:25:18 -0700 Subject: [PATCH 13/37] Add one helper module for the test fixture paths Before this change, 22 tests built each fixture path with PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("fixtures").join(...). The new test_fixtures module gives each fixture a function. Each function has a short description of its fixture, so a test can find a fixture by name. This is a test-only change. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/graph_tile.rs | 17 +++------- valhalla-graphtile/src/graph_tile/builder.rs | 21 ++++-------- valhalla-graphtile/src/lib.rs | 2 ++ valhalla-graphtile/src/test_fixtures.rs | 33 +++++++++++++++++++ .../src/tile_provider/directory.rs | 22 ++++--------- valhalla-graphtile/src/tile_provider/mod.rs | 14 +++----- .../src/tile_provider/tarball.rs | 14 +++----- .../src/tile_provider/traffic.rs | 9 ++--- 8 files changed, 63 insertions(+), 69 deletions(-) create mode 100644 valhalla-graphtile/src/test_fixtures.rs diff --git a/valhalla-graphtile/src/graph_tile.rs b/valhalla-graphtile/src/graph_tile.rs index 567a8ec..b06c3f1 100644 --- a/valhalla-graphtile/src/graph_tile.rs +++ b/valhalla-graphtile/src/graph_tile.rs @@ -12,6 +12,8 @@ use std::sync::Arc; use thiserror::Error; use zerocopy::{FromBytes, I16, LE, U32}; +#[cfg(test)] +use crate::test_fixtures; use enumset::EnumSet; use geo::{CoordFloat, Point, coord}; use num_traits::FromPrimitive; @@ -932,10 +934,7 @@ static TEST_GRAPH_TILE_L0: LazyLock = LazyLock::new(|| { let relative_path = TEST_GRAPH_TILE_ID_L0 .file_path("gph") .expect("Unable to get relative path"); - let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles") - .join(relative_path); + let path = test_fixtures::andorra_tiles_dir().join(relative_path); let bytes = std::fs::read(path).expect("Unable to read file"); OwnedGraphTileHandle::try_from(bytes).expect("Unable to get tile") @@ -946,10 +945,7 @@ static TEST_GRAPH_TILE_L2: LazyLock = LazyLock::new(|| { let relative_path = TEST_GRAPH_TILE_ID_L2 .file_path("gph") .expect("Unable to get relative path"); - let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles") - .join(relative_path); + let path = test_fixtures::andorra_tiles_dir().join(relative_path); let bytes = std::fs::read(path).expect("Unable to read file"); OwnedGraphTileHandle::try_from(bytes).expect("Unable to get tile") @@ -971,10 +967,7 @@ static TEST_GRAPH_TILE_WITH_FLOW: LazyLock = LazyLock::new let relative_path = TEST_GRAPH_TILE_ID_L0 .file_path("gph") .expect("Unable to get relative path"); - let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles-with-traffic") - .join(relative_path); + let path = test_fixtures::andorra_tiles_with_traffic_dir().join(relative_path); let bytes = std::fs::read(path).expect("Unable to read file"); OwnedGraphTileHandle::try_from(bytes).expect("Unable to get tile") diff --git a/valhalla-graphtile/src/graph_tile/builder.rs b/valhalla-graphtile/src/graph_tile/builder.rs index 18504cb..348fd20 100644 --- a/valhalla-graphtile/src/graph_tile/builder.rs +++ b/valhalla-graphtile/src/graph_tile/builder.rs @@ -665,6 +665,7 @@ where mod tests { use crate::graph_tile::predicted_speeds::COEFFICIENT_COUNT; use crate::graph_tile::{GraphTileBuilder, GraphTileHeader, OwnedGraphTileHandle}; + use crate::test_fixtures; use std::path::Path; use walkdir::WalkDir; @@ -709,9 +710,7 @@ mod tests { #[test] fn round_trip_from_builder_without_changes() { - let base_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base_dir = test_fixtures::andorra_tiles_dir(); for entry in WalkDir::new(base_dir) { let entry = entry.expect("Unable to traverse directory"); @@ -733,9 +732,7 @@ mod tests { #[test] fn round_trip_from_builder_without_changes_including_traffic() { - let base_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles-with-traffic"); + let base_dir = test_fixtures::andorra_tiles_with_traffic_dir(); for entry in WalkDir::new(base_dir) { let entry = entry.expect("Unable to traverse directory"); @@ -757,9 +754,7 @@ mod tests { #[test] fn add_predicted_speeds() { - let base_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base_dir = test_fixtures::andorra_tiles_dir(); let in_bytes = std::fs::read(base_dir.join("0").join("003").join("015.gph")) .expect("Unable to read file"); @@ -782,9 +777,7 @@ mod tests { .with_predicted_encoded_speeds(42, "BcUACQAEACEADP/7/9sAGf/fAAwAGwARAAX/+AAAAB0AFAAS//AAF//+ACwACQAqAAAAKP/6AEMABABsAAsBBAAq/rcAAP+bAAz/zv/s/9MABf/Y//X/8//9/+wACf/P//EADv/8//L//P/y//H////7AAwAFf/5//oADgAZAAQAFf/3/+8AB//yAB8ABv/0AAUAEf/8//QAFAAG//b////j//v//QAT//7/+f/kABMABwABAAv/6//8//cAEwAAABT/8v/6//wAAAAQ//cACwAFAAT/1//sAAEADAABAAYAE//9AAn/7gAH/+AAFQAB//4AC//o/+gAE//+AAAAFf/l//kABP/+//kACAAG//cAHv/qAB0AAv/4/+v/+wALAAMABP/3AAT/8wAIAAr/9wAK//j/+wAEAAD/+P/8//v/8f/2//L//AALAAcABgAG//gAAv/5AAoAHv//AAcAFf/zABD/7AAUAAv/7v/8AAgACAAN//0ADP/iABD/9f/3//7/+P/3AAQADP//AAMABw==").unwrap() .into_bytes().unwrap(); - let base_dir_with_traffic = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles-with-traffic"); + let base_dir_with_traffic = test_fixtures::andorra_tiles_with_traffic_dir(); let expected_out_bytes = std::fs::read(base_dir_with_traffic.join("0").join("003").join("015.gph")) .expect("Unable to read file"); @@ -798,9 +791,7 @@ mod tests { #[test] fn predicted_speeds_deduplication() { - let base_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base_dir = test_fixtures::andorra_tiles_dir(); let in_bytes = std::fs::read(base_dir.join("0").join("003").join("015.gph")) .expect("Unable to read file"); diff --git a/valhalla-graphtile/src/lib.rs b/valhalla-graphtile/src/lib.rs index 179c3c5..857d10c 100644 --- a/valhalla-graphtile/src/lib.rs +++ b/valhalla-graphtile/src/lib.rs @@ -4,6 +4,8 @@ mod graph_id; pub mod graph_tile; pub mod shape_codec; pub mod spatial; +#[cfg(test)] +mod test_fixtures; pub mod tile_hierarchy; pub mod tile_provider; pub mod traffic_tile; diff --git a/valhalla-graphtile/src/test_fixtures.rs b/valhalla-graphtile/src/test_fixtures.rs new file mode 100644 index 0000000..d620b2a --- /dev/null +++ b/valhalla-graphtile/src/test_fixtures.rs @@ -0,0 +1,33 @@ +//! Paths to the test fixtures in the `fixtures` directory of this crate. +//! +//! All fixtures are made from the same OpenStreetMap extract of Andorra. + +use std::path::PathBuf; + +fn fixture(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("fixtures") + .join(name) +} + +/// A tile directory, as `valhalla_build_tiles` writes it. +pub(crate) fn andorra_tiles_dir() -> PathBuf { + fixture("andorra-tiles") +} + +/// The tiles of [`andorra_tiles_dir`] in an extract tarball, +/// as `valhalla_build_extract` writes it. +pub(crate) fn andorra_tiles_tar() -> PathBuf { + fixture("andorra-tiles.tar") +} + +/// The tiles of [`andorra_tiles_dir`] with predicted traffic speeds +/// from `valhalla_add_predicted_traffic`. +pub(crate) fn andorra_tiles_with_traffic_dir() -> PathBuf { + fixture("andorra-tiles-with-traffic") +} + +/// A live traffic tarball for the Andorra tiles. +pub(crate) fn andorra_traffic_tar() -> PathBuf { + fixture("andorra-traffic.tar") +} diff --git a/valhalla-graphtile/src/tile_provider/directory.rs b/valhalla-graphtile/src/tile_provider/directory.rs index 01212a3..25ad3dc 100644 --- a/valhalla-graphtile/src/tile_provider/directory.rs +++ b/valhalla-graphtile/src/tile_provider/directory.rs @@ -202,6 +202,7 @@ mod test { use super::DirectoryGraphTileProvider; use crate::GraphId; use crate::graph_tile::GraphTile; + use crate::test_fixtures; use crate::tile_hierarchy::STANDARD_LEVELS; use crate::tile_provider::GraphTileProvider; use core::num::NonZeroUsize; @@ -210,13 +211,10 @@ mod test { rng, }; use std::collections::HashSet; - use std::path::PathBuf; #[test] fn test_get_tile() { - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(1).unwrap()); let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); let tile = provider @@ -232,9 +230,7 @@ mod test { fn test_get_opp_edge() { let mut rng = rng(); - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(1).unwrap()); let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); let tile = provider @@ -264,9 +260,7 @@ mod test { #[test] fn test_get_shortcut_fixture() { - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(1).unwrap()); // This test relies on the state of the andorra-tiles fixtures. @@ -310,9 +304,7 @@ mod test { #[cfg(not(miri))] #[test] fn test_level_zero_shortcut_sanity() { - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(1).unwrap()); let base_id = GraphId::try_from_components(0, 3015, 0).unwrap(); @@ -393,9 +385,7 @@ mod test { // how the tile structure works. // Specifically, there is only a single "copy" of a feature in the routing graph, // and all features of a specific road class exist in specific tile levels. - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(1).unwrap()); let graph_ids = vec![ GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"), diff --git a/valhalla-graphtile/src/tile_provider/mod.rs b/valhalla-graphtile/src/tile_provider/mod.rs index d068fd3..ab60959 100644 --- a/valhalla-graphtile/src/tile_provider/mod.rs +++ b/valhalla-graphtile/src/tile_provider/mod.rs @@ -519,13 +519,13 @@ impl LockTable { mod tests { use crate::GraphId; use crate::graph_tile::GraphTile; + use crate::test_fixtures; use crate::tile_hierarchy::STANDARD_LEVELS; use crate::tile_provider::{DirectoryGraphTileProvider, GraphTileProvider}; use geo::{Destination, HasDimensions, Haversine, LineString, Validation, coord, point}; use rand::rngs::ThreadRng; use rand::{Rng, rng}; use std::num::NonZeroUsize; - use std::path::PathBuf; #[test] fn haversine_antimeridian_wraps() { @@ -544,9 +544,7 @@ mod tests { fn test_nodes_within_radius() { let mut rng = rng(); - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(4).unwrap()); let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); provider.with_tile_containing_or_panic(graph_id, |tile_view| { @@ -665,9 +663,7 @@ mod tests { #[test] fn test_edges_within_radius_level0() { - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(4).unwrap()); let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); sanity_check_edges_within_radius(provider, graph_id, rng()); @@ -675,9 +671,7 @@ mod tests { #[test] fn test_edges_within_radius_level2() { - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(4).unwrap()); // Use a known level 2 tile from fixtures to exercise edge bins path let graph_id = diff --git a/valhalla-graphtile/src/tile_provider/tarball.rs b/valhalla-graphtile/src/tile_provider/tarball.rs index 3866fd7..39afc93 100644 --- a/valhalla-graphtile/src/tile_provider/tarball.rs +++ b/valhalla-graphtile/src/tile_provider/tarball.rs @@ -574,11 +574,11 @@ pub fn serialize_index_bin(entries: &[TileIndexBinEntry]) -> Result<&[u8], Inval mod test { use super::*; use crate::graph_tile::{GraphTile, GraphTileView}; + use crate::test_fixtures; use crate::tile_provider::DirectoryGraphTileProvider; use proptest::collection::vec; use proptest::{prop_assert, prop_assert_eq, proptest}; use std::num::NonZeroUsize; - use std::path::PathBuf; /// Bytes taken from the start of a large extract generated by official Valhalla tooling. const INDEX_BIN_FIXTURE: &[u8] = &[ @@ -804,9 +804,7 @@ mod test { #[cfg(not(miri))] #[test] fn test_get_tile() { - let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles.tar"); + let path = test_fixtures::andorra_tiles_tar(); let provider = TarballTileProvider::new_readonly(path).expect("Unable to init tile provider"); let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); @@ -841,15 +839,11 @@ mod test { GraphId::try_from_components(2, 763927, 1).expect("Unable to create graph ID"), ]; - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let directory_provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(1).unwrap()); - let tarball_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles.tar"); + let tarball_path = test_fixtures::andorra_tiles_tar(); let tarball_provider = TarballTileProvider::new_readonly(tarball_path.clone()) .expect("Unable to init tile provider"); let immutable_tarball_provider = unsafe { diff --git a/valhalla-graphtile/src/tile_provider/traffic.rs b/valhalla-graphtile/src/tile_provider/traffic.rs index 7e7a5b4..876c818 100644 --- a/valhalla-graphtile/src/tile_provider/traffic.rs +++ b/valhalla-graphtile/src/tile_provider/traffic.rs @@ -231,15 +231,14 @@ impl TrafficTileProvider { #[cfg(all(test, not(miri)))] mod tests { use crate::GraphId; + use crate::test_fixtures; use crate::tile_provider::TrafficTileProvider; use crate::traffic_tile::{SpeedValue, TrafficSpeed}; use std::path::PathBuf; #[test] fn test_get_speed() { - let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-traffic.tar"); + let path = test_fixtures::andorra_traffic_tar(); let provider = TrafficTileProvider::new_readonly(path).expect("Unable to init tile provider"); @@ -269,9 +268,7 @@ mod tests { fn test_set_speed() { const DESIRED_SPEED: u8 = 32; - let fixture_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-traffic.tar"); + let fixture_path = test_fixtures::andorra_traffic_tar(); // let tmp_path = PathBuf::from("/Users/ianthetechie/valhalla-docker/valhalla/traffic.tar"); let tmp_dir = option_env!("RUNNER_TEMP").unwrap_or("/tmp"); From 1ae638dffecae28dc3306274a2d6af2ffe5293b9 Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Tue, 13 Oct 2026 20:38:58 -0700 Subject: [PATCH 14/37] Reject index.bin entries that point past the end of the archive MmapTilePointer::as_tile_bytes makes a slice from an index offset and size without a bounds check. Before this change, TarballTileProvider did not check that these ranges were inside the file. So a corrupt or truncated index.bin could make safe calls read outside the memory map. - TileIndexBinEntry::validated now also checks that offset + size fits in the archive. It uses checked_add, so a huge offset cannot wrap. A failure is the new InvalidIndexBinError::OutOfBounds. - The provider reads the archive length before it reads the index. So all entry checks run in one pass, and an invalid index fails before the provider maps the file. After it maps the file, the provider makes sure that the map length is the length that it checked. - New tests examine check_range at the exact edge and on overflow, and validated() at the edge. A test opens a real tarball whose index points past the end of the file. A proptest makes sure that the read path accepts every entry that the write path creates. - TileIndexBinEntry::graph_id, offset, and size are now public, so that tests and other readers can examine a parsed entry. - tempfile is a new dev-dependency for the test tarball. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 17 +- Cargo.toml | 1 + valhalla-graphtile/Cargo.toml | 1 + .../src/tile_provider/tarball.rs | 182 ++++++++++++++++-- 4 files changed, 177 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 26b58f2..a4c24cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1048,9 +1048,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.180" +version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" [[package]] name = "libm" @@ -1071,9 +1071,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "lock_api" @@ -1665,9 +1665,9 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.3" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" dependencies = [ "bitflags 2.10.0", "errno", @@ -1926,9 +1926,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.24.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", "getrandom 0.3.4", @@ -2232,6 +2232,7 @@ dependencies = [ "self_cell", "serde", "tar", + "tempfile", "thiserror 2.0.18", "tracing", "trig-const", diff --git a/Cargo.toml b/Cargo.toml index 03469f4..d1b0706 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,6 +54,7 @@ serde = { version = "1.0.210", features = ["derive"] } serde_with = "3.14.0" serde_json = "1.0.132" tar = "0.4.44" +tempfile = "3.27.0" thiserror = "2.0.11" tokio = { version = "1.48.0", features = ["macros", "rt-multi-thread", "fs", "io-util", "sync"] } tracing = "0.1.41" diff --git a/valhalla-graphtile/Cargo.toml b/valhalla-graphtile/Cargo.toml index 62370c8..88ba8a8 100644 --- a/valhalla-graphtile/Cargo.toml +++ b/valhalla-graphtile/Cargo.toml @@ -37,6 +37,7 @@ zerocopy-derive = { workspace = true } insta = { workspace = true } proptest = { workspace = true } rand = { workspace = true } +tempfile = { workspace = true } walkdir = { workspace = true } [lints] diff --git a/valhalla-graphtile/src/tile_provider/tarball.rs b/valhalla-graphtile/src/tile_provider/tarball.rs index 39afc93..16dcf37 100644 --- a/valhalla-graphtile/src/tile_provider/tarball.rs +++ b/valhalla-graphtile/src/tile_provider/tarball.rs @@ -102,7 +102,9 @@ impl TarballTileProvider { ); } - let mut archive = Archive::new(File::open(&path)?); + let archive_file = File::open(&path)?; + let archive_len = archive_file.metadata()?.len(); + let mut archive = Archive::new(archive_file); let mut entries = archive.entries()?; let Some(entry) = entries.next() else { @@ -128,7 +130,7 @@ impl TarballTileProvider { let mut tile_index: HashMap = HashMap::with_capacity(index_entries.len()); for entry in index_entries { - let (graph_id, offset) = entry.validated()?; + let (graph_id, offset) = entry.validated(archive_len)?; tile_index.insert( graph_id, TileOffset { @@ -149,6 +151,15 @@ impl TarballTileProvider { Arc::new(MmapOptions::new().map_raw_read_only(&file)?) }; + // Tile pointers slice the map without a bounds check, + // so the map must have the length that the index ranges were checked against. + if mmap.len() as u64 != archive_len { + return Err(GraphTileProviderError::InvalidTarball(format!( + "the archive changed size while it was opened ({archive_len} bytes, then {} bytes)", + mmap.len() + ))); + } + let mut tile_cache = HashMap::new(); if CACHE { @@ -453,6 +464,15 @@ pub enum InvalidIndexBinError { "the entry for {graph_id} has offset {offset}; expected a non-zero multiple of {TAR_BLOCK_SIZE}" )] Offset { graph_id: GraphId, offset: u64 }, + #[error( + "the entry for {graph_id} has {size} bytes at offset {offset}, which is past the end of the {archive_len}-byte archive" + )] + OutOfBounds { + graph_id: GraphId, + offset: u64, + size: u32, + archive_len: u64, + }, } impl TileIndexBinEntry { @@ -475,20 +495,25 @@ impl TileIndexBinEntry { }) } - /// Validates a parsed entry, and returns its graph ID and offset. + /// Checks the entry against an archive of `archive_len` bytes, + /// and returns its graph ID and offset. /// - /// [`parse_index_bin`] does not examine the entry values, - /// so the read path must call this before it uses an entry. - /// The checks are the same as in [`TileIndexBinEntry::new`]. - fn validated(&self) -> Result<(GraphId, u64), InvalidIndexBinError> { + /// Call this before you use an entry from [`parse_index_bin`], which does not check the values. + fn validated(&self, archive_len: u64) -> Result<(GraphId, u64), InvalidIndexBinError> { let graph_id = self.graph_id()?; let offset = self.offset.get(); check_offset(graph_id, offset)?; + check_range(graph_id, offset, self.size.get(), archive_len)?; Ok((graph_id, offset)) } - fn graph_id(&self) -> Result { + /// The graph ID of the indexed tile. + /// + /// # Errors + /// + /// Returns an error if the stored tile ID has nonzero feature index bits. + pub fn graph_id(&self) -> Result { // SAFETY: We know that the bit field cannot contain a value // larger than the max allowed value (it's limited to 46 bits). // Therefore, this is guaranteed to be a valid Graph ID bit pattern. @@ -497,6 +522,18 @@ impl TileIndexBinEntry { Ok(graph_id) } + + /// The byte offset of the tile payload from the beginning of the tarball. + #[inline] + pub fn offset(&self) -> u64 { + self.offset.get() + } + + /// The size of the tile in bytes. + #[inline] + pub fn size(&self) -> u32 { + self.size.get() + } } /// Makes sure that `graph_id` identifies a full tile, @@ -524,6 +561,28 @@ fn check_offset(graph_id: GraphId, offset: u64) -> Result<(), InvalidIndexBinErr Ok(()) } +/// Makes sure that the tile at `offset` with `size` bytes lies inside an archive of `archive_len` bytes. +fn check_range( + graph_id: GraphId, + offset: u64, + size: u32, + archive_len: u64, +) -> Result<(), InvalidIndexBinError> { + if offset + .checked_add(u64::from(size)) + .is_none_or(|end| end > archive_len) + { + return Err(InvalidIndexBinError::OutOfBounds { + graph_id, + offset, + size, + archive_len, + }); + } + + Ok(()) +} + /// Parses an `index.bin` file to enable random access. /// /// See [`TileIndexBinEntry`] for a description of the tile format. @@ -618,14 +677,30 @@ mod test { fn test_validate_parsed_entry() { let graph_id = GraphId::try_from_components(0, 2540, 0).expect("Unable to create graph ID"); let index_entries = parse_index_bin(INDEX_BIN_FIXTURE).expect("Unable to parse fixture"); - assert_eq!(index_entries[0].validated(), Ok((graph_id, 23040))); + // The smallest archive that can contain the first tile ends where that tile ends. + let first_entry_end = index_entries[0].offset() + u64::from(index_entries[0].size()); + assert_eq!( + index_entries[0].validated(first_entry_end), + Ok((graph_id, 23040)), + "A tile may end exactly at the end of the archive" + ); + assert_eq!( + index_entries[0].validated(first_entry_end - 1), + Err(InvalidIndexBinError::OutOfBounds { + graph_id, + offset: 23040, + size: 1_181_852, + archive_len: first_entry_end - 1 + }), + "A tile must not end past the end of the archive" + ); // Change the low byte of the first offset (23040 = 0x5a00) to make it unaligned let mut bytes = INDEX_BIN_FIXTURE.to_vec(); bytes[0] = 0x01; let index_entries = parse_index_bin(&bytes).expect("Unable to parse modified fixture"); assert_eq!( - index_entries[0].validated(), + index_entries[0].validated(first_entry_end), Err(InvalidIndexBinError::Offset { graph_id, offset: 23041 @@ -638,7 +713,7 @@ mod test { bytes[..8].fill(0); let index_entries = parse_index_bin(&bytes).expect("Unable to parse modified fixture"); assert_eq!( - index_entries[0].validated(), + index_entries[0].validated(first_entry_end), Err(InvalidIndexBinError::Offset { graph_id, offset: 0 @@ -651,7 +726,7 @@ mod test { bytes[11] |= 0x02; let index_entries = parse_index_bin(&bytes).expect("Unable to parse modified fixture"); assert_eq!( - index_entries[0].validated(), + index_entries[0].validated(first_entry_end), Err(InvalidIndexBinError::FeatureIndex( graph_id .with_feature_index(1) @@ -691,6 +766,72 @@ mod test { assert_eq!(index_entries[2].size.get(), 1_809_704); } + #[test] + fn test_check_range() { + let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); + + assert_eq!(check_range(graph_id, 512, 512, 1024), Ok(())); + assert_eq!( + check_range(graph_id, 512, 513, 1024), + Err(InvalidIndexBinError::OutOfBounds { + graph_id, + offset: 512, + size: 513, + archive_len: 1024 + }), + "A tile must end at or before the end of the archive" + ); + assert_eq!( + check_range(graph_id, u64::MAX - 511, 1024, u64::MAX), + Err(InvalidIndexBinError::OutOfBounds { + graph_id, + offset: u64::MAX - 511, + size: 1024, + archive_len: u64::MAX + }), + "The end of the range must not overflow" + ); + } + + // Tragically, memory maps are not supported by Miri! + #[cfg(not(miri))] + #[test] + fn test_reject_index_past_end_of_archive() { + let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); + // The index gives 10,000 bytes, but the archive has only 100 bytes of tile data. + let index_entry = + TileIndexBinEntry::new(graph_id, 1536, 10_000).expect("Unable to create index entry"); + let index_bytes = serialize_index_bin(core::slice::from_ref(&index_entry)) + .expect("Unable to serialize the index"); + + let file = tempfile::NamedTempFile::new().expect("Unable to create a temporary file"); + let mut builder = tar::Builder::new(file.as_file()); + for (path, payload) in [("index.bin", index_bytes), ("0/003/015.gph", &[0x5a; 100])] { + let mut header = tar::Header::new_ustar(); + header.set_size(payload.len() as u64); + header.set_cksum(); + builder + .append_data(&mut header, path, payload) + .expect("Unable to append a tar entry"); + } + builder.finish().expect("Unable to finish the archive"); + + let result = TarballTileProvider::new_readonly(file.path()); + assert!( + matches!( + result, + Err(GraphTileProviderError::InvalidIndexBin( + InvalidIndexBinError::OutOfBounds { + offset: 1536, + size: 10_000, + .. + } + )) + ), + "An index entry past the end of the archive must be rejected" + ); + } + #[test] fn test_tile_index_bin_entry_new() { let graph_id = GraphId::try_from_components(0, 2540, 0).expect("Unable to create graph ID"); @@ -776,8 +917,12 @@ mod test { proptest! { #[test] fn test_index_bin_roundtrip( - // Levels are 3 bits, tile IDs are 22 bits, and offsets are counted in 512-byte blocks - raw_entries in vec((0u8..=7, 0u64..1 << 22, 1u64..=u64::MAX / 512, 0u32..=u32::MAX), 1..64) + // Levels are 3 bits, tile IDs are 22 bits, and offsets are counted in 512-byte blocks. + // The offset bound keeps the end of every tile inside a u64. + raw_entries in vec( + (0u8..=7, 0u64..1 << 22, 1u64..=(u64::MAX - u64::from(u32::MAX)) / 512, 0u32..=u32::MAX), + 1..64 + ) ) { let entries: Vec = raw_entries .into_iter() @@ -793,9 +938,14 @@ mod test { let parsed = parse_index_bin(bytes).expect("Serialized bytes always parse"); prop_assert_eq!(parsed, entries.as_slice()); - // The read path accepts every entry that the write path creates + // Every entry is valid in an archive that ends where the last tile ends. + let archive_len = parsed + .iter() + .map(|entry| entry.offset() + u64::from(entry.size())) + .max() + .expect("The entry list is non-empty"); for entry in parsed { - prop_assert!(entry.validated().is_ok()); + prop_assert!(entry.validated(archive_len).is_ok()); } } } From 76c26ebe45c3fb5fa4785c4b17dba3a3c2d3e91e Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Mon, 2 Nov 2026 09:27:55 -0800 Subject: [PATCH 15/37] Get tile bytes and tile sizes through GraphTileProvider Code that copies tiles verbatim, such as an extract writer, must know the size of each tile before it reads any tile data. It must also get the raw bytes of each tile. Before this change, GraphTileProvider gave only decoded tiles. - GraphTileHandle now has tile_bytes, which returns the bytes that the handle decoded. OwnedGraphTileHandle, MmapGraphTileHandle, and Arc implement the trait. The handles do not copy the bytes. - GraphTileProvider::TileHandle must now implement GraphTileHandle. - GraphTileProvider has a new tile_size method. TarballTileProvider gets the size from its index.bin. DirectoryGraphTileProvider gets the size from the file metadata, so it does not read the file or change its tile cache. - GraphTileProviderError has a new TileTooLarge variant for a tile file that has more than u32::MAX bytes. - The tarball test that compares tiles with the directory provider now also compares the handle bytes and the sizes of all three providers. New tests examine a missing tile, a directory at a tile path, and a sparse file that is too large. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/graph_tile/handles.rs | 50 ++++++++ .../src/tile_provider/directory.rs | 113 +++++++++++++++++- valhalla-graphtile/src/tile_provider/mod.rs | 23 +++- .../src/tile_provider/tarball.rs | 83 +++++++++++++ 4 files changed, 267 insertions(+), 2 deletions(-) diff --git a/valhalla-graphtile/src/graph_tile/handles.rs b/valhalla-graphtile/src/graph_tile/handles.rs index 54855f2..cb6b488 100644 --- a/valhalla-graphtile/src/graph_tile/handles.rs +++ b/valhalla-graphtile/src/graph_tile/handles.rs @@ -15,8 +15,31 @@ use memmap2::MmapRaw; use self_cell::self_cell; use std::sync::Arc; +/// An owned handle to a decoded graph tile, which keeps the bytes that it decoded. pub trait GraphTileHandle { + /// Returns a view of the decoded tile. fn tile_view(&'_ self) -> GraphTileView<'_>; + + /// Returns the bytes of the tile, exactly as the handle got them. + /// + /// The view from [`GraphTileHandle::tile_view`] was decoded from these bytes, + /// so they are a valid graph tile. + fn tile_bytes(&self) -> &[u8]; +} + +impl GraphTileHandle for Arc +where + T: GraphTileHandle, +{ + #[inline] + fn tile_view(&'_ self) -> GraphTileView<'_> { + (**self).tile_view() + } + + #[inline] + fn tile_bytes(&self) -> &[u8] { + (**self).tile_bytes() + } } self_cell! { @@ -38,6 +61,18 @@ impl TryFrom> for OwnedGraphTileHandle { } } +impl GraphTileHandle for OwnedGraphTileHandle { + #[inline] + fn tile_view(&'_ self) -> GraphTileView<'_> { + self.borrow_dependent().clone() + } + + #[inline] + fn tile_bytes(&self) -> &[u8] { + self.borrow_owner() + } +} + impl GraphTile for OwnedGraphTileHandle { #[inline] fn graph_id(&self) -> GraphId { @@ -274,6 +309,21 @@ impl MmapGraphTileHandle { } } +impl GraphTileHandle for MmapGraphTileHandle { + #[inline] + fn tile_view(&'_ self) -> GraphTileView<'_> { + self.borrow_dependent().clone() + } + + #[inline] + fn tile_bytes(&self) -> &[u8] { + // SAFETY: The decoded view borrows this same range of the memory map, + // so the caller of `MmapGraphTileHandle::try_from` already accepted + // the conditions of `MmapTilePointer::as_tile_bytes` for the life of the handle. + unsafe { self.borrow_owner().as_tile_bytes() } + } +} + /// An encapsulating type that can be used to get the actual bytes for a tile. /// /// This provides a single type that can "own" the backing bytes for tile memory, diff --git a/valhalla-graphtile/src/tile_provider/directory.rs b/valhalla-graphtile/src/tile_provider/directory.rs index 25ad3dc..9ea4b3b 100644 --- a/valhalla-graphtile/src/tile_provider/directory.rs +++ b/valhalla-graphtile/src/tile_provider/directory.rs @@ -176,6 +176,28 @@ impl GraphTileProvider for DirectoryGraphTileProvider { Ok(tile) } + /// The size comes from the metadata of the tile file, so this reads no tile data. + /// + /// The size is the size of the file at the time of the call. + /// If the file changes after the provider caches the tile, + /// the size and the cached handle do not agree. + fn tile_size(&self, graph_id: GraphId) -> Result { + let base_graph_id = graph_id.tile_base_id(); + let size = match std::fs::metadata(self.path_for_graph_id(base_graph_id)?) { + Ok(metadata) if metadata.is_file() => metadata.len(), + Ok(_) => return Err(GraphTileProviderError::TileDoesNotExist), + Err(e) if e.kind() == ErrorKind::NotFound => { + return Err(GraphTileProviderError::TileDoesNotExist); + } + Err(e) => return Err(e.into()), + }; + + u32::try_from(size).map_err(|_| GraphTileProviderError::TileTooLarge { + graph_id: base_graph_id, + size, + }) + } + fn tiles_within_radius( &self, center: Point, @@ -202,9 +224,10 @@ mod test { use super::DirectoryGraphTileProvider; use crate::GraphId; use crate::graph_tile::GraphTile; + use crate::graph_tile::handles::GraphTileHandle; use crate::test_fixtures; use crate::tile_hierarchy::STANDARD_LEVELS; - use crate::tile_provider::GraphTileProvider; + use crate::tile_provider::{GraphTileProvider, GraphTileProviderError}; use core::num::NonZeroUsize; use rand::{ distr::{Distribution, Uniform}, @@ -431,4 +454,92 @@ mod test { } } } + + #[test] + fn test_tile_size_matches_handle_bytes() { + let provider = + DirectoryGraphTileProvider::new(test_fixtures::andorra_tiles_dir(), NonZeroUsize::MIN); + // The feature index is not zero, so the provider must use the base ID of the tile. + let graph_id = GraphId::try_from_components(0, 3015, 1).expect("Unable to create graph ID"); + + let size = provider + .tile_size(graph_id) + .expect("Unable to get the tile size"); + let tile = provider + .get_handle_for_tile_containing(graph_id) + .expect("Unable to get tile"); + assert_eq!( + usize::try_from(size).expect("The size fits in a usize"), + tile.tile_bytes().len(), + "The tile size must be the length of the tile bytes" + ); + } + + #[test] + fn test_tile_size_of_missing_tile() { + let provider = + DirectoryGraphTileProvider::new(test_fixtures::andorra_tiles_dir(), NonZeroUsize::MIN); + // This tile is not in the Andorra extract. + let graph_id = GraphId::try_from_components(0, 0, 0).expect("Unable to create graph ID"); + + assert!( + matches!( + provider.tile_size(graph_id), + Err(GraphTileProviderError::TileDoesNotExist) + ), + "A tile without a file must not exist" + ); + } + + #[test] + fn test_tile_size_of_directory_at_tile_path() { + let base = tempfile::tempdir().expect("Unable to create a temporary directory"); + let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); + let tile_path = base.path().join( + graph_id + .file_path("gph") + .expect("Unable to get the tile path"), + ); + std::fs::create_dir_all(tile_path).expect("Unable to create a directory at the tile path"); + let provider = + DirectoryGraphTileProvider::new(base.path().to_path_buf(), NonZeroUsize::MIN); + + assert!( + matches!( + provider.tile_size(graph_id), + Err(GraphTileProviderError::TileDoesNotExist) + ), + "A directory at the tile path must not be a tile" + ); + } + + // On Unix, a sparse file of this size uses almost no disk space. + #[cfg(all(unix, not(miri)))] + #[test] + fn test_tile_size_too_large() { + let base = tempfile::tempdir().expect("Unable to create a temporary directory"); + let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); + let tile_path = base.path().join( + graph_id + .file_path("gph") + .expect("Unable to get the tile path"), + ); + let parent = tile_path.parent().expect("The tile path has a parent"); + std::fs::create_dir_all(parent).expect("Unable to create the tile directory"); + let file_len = u64::from(u32::MAX) + 1; + std::fs::File::create(&tile_path) + .and_then(|file| file.set_len(file_len)) + .expect("Unable to create a large sparse file"); + let provider = + DirectoryGraphTileProvider::new(base.path().to_path_buf(), NonZeroUsize::MIN); + + assert!( + matches!( + provider.tile_size(graph_id), + Err(GraphTileProviderError::TileTooLarge { graph_id: id, size }) + if id == graph_id && size == file_len + ), + "A tile file larger than u32::MAX bytes must be an error" + ); + } } diff --git a/valhalla-graphtile/src/tile_provider/mod.rs b/valhalla-graphtile/src/tile_provider/mod.rs index ab60959..c9d403b 100644 --- a/valhalla-graphtile/src/tile_provider/mod.rs +++ b/valhalla-graphtile/src/tile_provider/mod.rs @@ -19,6 +19,7 @@ mod tarball; mod traffic; use crate::graph_id::InvalidGraphIdError; +use crate::graph_tile::handles::GraphTileHandle; use crate::graph_tile::{ DirectedEdge, GraphTile, GraphTileDecodingError, GraphTileView, LookupError, NodeInfo, OpposingEdgeIndex, @@ -54,6 +55,8 @@ pub enum GraphTileProviderError { InvalidIndexBin(#[from] InvalidIndexBinError), #[error("Unsupported tile version; this may or may not be compatible.")] UnsupportedTileVersion, + #[error("the tile file for {graph_id} has {size} bytes, which is more than a tile can have")] + TileTooLarge { graph_id: GraphId, size: u64 }, } /// A generic trait for all graph tile providers. @@ -66,7 +69,7 @@ pub trait GraphTileProvider { /// The tile handle type for owned handles. /// /// Code using the `GraphTileProvider` may rely on the handles being cheaply cloneable. - type TileHandle: GraphTile + Clone; + type TileHandle: GraphTile + GraphTileHandle + Clone; /// Gets the tile containing the given graph ID, /// and does some work in a closure which takes the reference as a parameter. @@ -110,6 +113,24 @@ pub trait GraphTileProvider { graph_id: GraphId, ) -> Result; + /// Gets the size in bytes of the tile containing the given graph ID, + /// without reading or decoding the tile. + /// + /// While the tile data does not change, the size is the length of + /// [`GraphTileHandle::tile_bytes`] for the handle of the same tile. + /// + /// # Performance + /// + /// Implementations SHOULD get the size from an index or from file metadata, + /// so that a caller can get the sizes of many tiles before it reads any of them. + /// + /// # Errors + /// + /// If the provider does not have the tile, + /// this returns [`GraphTileProviderError::TileDoesNotExist`]. + /// Other errors are possible, such as I/O errors. + fn tile_size(&self, graph_id: GraphId) -> Result; + /// Enumerate base tile Graph IDs across all hierarchy levels that intersect a circle around /// `center` with radius `radius`. /// diff --git a/valhalla-graphtile/src/tile_provider/tarball.rs b/valhalla-graphtile/src/tile_provider/tarball.rs index 16dcf37..aa5ac3a 100644 --- a/valhalla-graphtile/src/tile_provider/tarball.rs +++ b/valhalla-graphtile/src/tile_provider/tarball.rs @@ -222,6 +222,11 @@ impl GraphTileProvider for TarballTileProvider { Ok(Arc::new(unsafe { MmapGraphTileHandle::try_from(pointer)? })) } + #[inline] + fn tile_size(&self, graph_id: GraphId) -> Result { + self.tile_size(graph_id) + } + #[inline] fn tiles_within_radius( &self, @@ -263,6 +268,11 @@ impl GraphTileProvider for TarballTileProvider { Ok(tile.clone()) } + #[inline] + fn tile_size(&self, graph_id: GraphId) -> Result { + self.tile_size(graph_id) + } + #[inline] fn tiles_within_radius( &self, @@ -274,6 +284,15 @@ impl GraphTileProvider for TarballTileProvider { } impl TarballTileProvider { + /// The size comes from the `index.bin` of the tarball, so this reads no tile data. + #[inline] + fn tile_size(&self, graph_id: GraphId) -> Result { + self.tile_index + .get(&graph_id.tile_base_id()) + .map(|tile_offset| tile_offset.size) + .ok_or(GraphTileProviderError::TileDoesNotExist) + } + #[inline] fn tiles_within_radius( &self, @@ -632,6 +651,7 @@ pub fn serialize_index_bin(entries: &[TileIndexBinEntry]) -> Result<&[u8], Inval #[cfg(test)] mod test { use super::*; + use crate::graph_tile::handles::GraphTileHandle; use crate::graph_tile::{GraphTile, GraphTileView}; use crate::test_fixtures; use crate::tile_provider::DirectoryGraphTileProvider; @@ -1020,6 +1040,69 @@ mod test { .as_tile_bytes() }; assert_eq!(directory_tile.borrow_owner(), immutable_tarball_tile_bytes); + + let tarball_handle = tarball_provider + .get_handle_for_tile_containing(*graph_id) + .expect("Unable to get tile"); + for (provider_name, tile_bytes, tile_size) in [ + ( + "directory", + directory_tile.tile_bytes(), + directory_provider.tile_size(*graph_id), + ), + ( + "tarball", + tarball_handle.tile_bytes(), + tarball_provider.tile_size(*graph_id), + ), + ( + "immutable tarball", + immutable_tarball_tile_pointer.tile_bytes(), + immutable_tarball_provider.tile_size(*graph_id), + ), + ] { + assert_eq!( + tile_bytes, + directory_tile.borrow_owner().as_slice(), + "The {provider_name} handle must give the bytes of the tile file for {graph_id}" + ); + let tile_size = tile_size.expect("Unable to get the tile size"); + assert_eq!( + usize::try_from(tile_size).expect("The size fits in a usize"), + tile_bytes.len(), + "The {provider_name} size of {graph_id} must be the length of its bytes" + ); + } } } + + // Tragically, memory maps are not supported by Miri! + #[cfg(not(miri))] + #[test] + fn test_tile_size_of_missing_tile() { + let tarball_path = test_fixtures::andorra_tiles_tar(); + let tarball_provider = TarballTileProvider::new_readonly(tarball_path.clone()) + .expect("Unable to init tile provider"); + // SAFETY: Nothing writes to the fixture tarball while the tests run. + let immutable_tarball_provider = unsafe { + TarballTileProvider::new_immutable(tarball_path).expect("Unable to init tile provider") + }; + // This tile is not in the Andorra extract. + let graph_id = GraphId::try_from_components(0, 0, 0).expect("Unable to create graph ID"); + + assert!( + matches!( + tarball_provider.tile_size(graph_id), + Err(GraphTileProviderError::TileDoesNotExist) + ), + "A tile that is not in the index must not exist" + ); + assert!( + matches!( + immutable_tarball_provider.tile_size(graph_id), + Err(GraphTileProviderError::TileDoesNotExist) + ), + "A tile that is not in the index must not exist" + ); + } } From 7bdbdf88fde7efb5d1749580b0566da5c8874fc8 Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Wed, 21 Oct 2026 10:23:50 -0700 Subject: [PATCH 16/37] Add GraphId::relative_path and string parsers for graph IDs Extract tarballs and tile directories use the same relative path for a tile file, with `/` separators. GraphId::file_path returns a PathBuf with the separators of the platform, so it does not give the tar entry name on Windows. GraphId::relative_path builds the `/`-separated string directly, and file_path now uses it. relative_path also returns InvalidGraphIdError::Level for levels 4 to 7. Before this change, file_path panicked on an index out of bounds for these levels. GraphId now implements FromStr for a packed u64 value or the level/tile_id/index form. GraphId::parse_tile parses the level/tile_id form of a whole tile. Both use the same component parser and return the new ParseGraphIdError. InvalidGraphIdError and ParseGraphIdError are now re-exported at the crate root, so callers can name them. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/graph_id.rs | 312 +++++++++++++++++++++++++++-- valhalla-graphtile/src/lib.rs | 2 +- 2 files changed, 291 insertions(+), 23 deletions(-) diff --git a/valhalla-graphtile/src/graph_id.rs b/valhalla-graphtile/src/graph_id.rs index fa765b9..5147c74 100644 --- a/valhalla-graphtile/src/graph_id.rs +++ b/valhalla-graphtile/src/graph_id.rs @@ -1,4 +1,5 @@ use crate::tile_hierarchy::{STANDARD_LEVELS, TRANSIT_LEVEL}; +use core::str::FromStr; #[cfg(feature = "serde")] use serde::{Serialize, Serializer}; use std::fmt::{Display, Formatter}; @@ -37,6 +38,31 @@ pub enum InvalidGraphIdError { InvalidGraphId, } +/// The reasons why a string is not a valid graph ID or tile. +#[derive(Debug, Error, PartialEq)] +pub enum ParseGraphIdError { + /// The input does not have the expected number of `/`-separated components. + #[error("'{input}' is not in the {expected} form")] + Format { + input: String, + /// The expected form (e.g. `level/tile_id`). + expected: &'static str, + }, + /// A component of the input is not a non-negative integer. + #[error("the {component} in '{input}' is not a valid number")] + Component { + input: String, + /// The name of the component (e.g. `level`). + component: &'static str, + }, + /// The components are numbers, but they do not make a valid graph ID. + #[error("'{input}' is not a valid graph ID: {source}")] + Invalid { + input: String, + source: InvalidGraphIdError, + }, +} + /// An Identifier of a node or an edge within the tiled, hierarchical graph. /// It packs a hierarchy level, tile ID, and an identifier within /// the tile/level into a 64-bit integer. @@ -209,19 +235,38 @@ impl GraphId { GraphId(U64::::new(self.value() & 0x01ff_ffff)) } - /// Constructs a relative path for the given tile. + /// Parses the base graph ID of a tile from `level/tile_id` (e.g. `2/762485`). + /// + /// Whitespace around each component is ignored. + /// + /// # Errors + /// + /// Returns an error if `input` does not have two `/`-separated components, + /// if a component is not a non-negative integer, + /// or if the level or the tile ID is out of range. + pub fn parse_tile(input: &str) -> Result { + let [level, tile_id] = parse_components(input, "level/tile_id", ["level", "tile ID"])?; + components_to_graph_id(input, level, tile_id, 0) + } + + /// Returns the relative path of the tile file for this graph ID, + /// with `/` separators on every platform (e.g. `2/000/762/485.gph`). + /// + /// Valhalla uses this path in tile directories and extract tarballs. + /// The feature index is not part of the path. /// /// # Errors /// /// This will fail if the tile is invalid for this level of tiling. /// TODO: It seems like we could do this check at tile creation time? - pub fn file_path(&self, extension: &str) -> Result { - // This is quite ugly and prone to failure; this is a pretty literal C++ translation + pub fn relative_path(&self, extension: &str) -> Result { let level_number = self.level(); let level = if level_number == TRANSIT_LEVEL.level { &*TRANSIT_LEVEL } else { - &STANDARD_LEVELS[level_number as usize] + STANDARD_LEVELS + .get(level_number as usize) + .ok_or(InvalidGraphIdError::Level)? }; let max_id = level.tiling_system.n_cols * level.tiling_system.n_rows - 1; @@ -230,26 +275,101 @@ impl GraphId { return Err(InvalidGraphIdError::GraphTileId); } - let l = max_id.max(1).ilog10() + 1; - let rem = l % 3; - let n_digits = if rem == 0 { l } else { l + (3 - rem) }; - debug_assert!(n_digits % 3 == 0); - - // This may not be as efficient as the Valhalla implementation, but it's more readable. - // Format tile_id with leading zeros to match max_length - let padded_id = format!("{:0>width$}", tile_id, width = n_digits as usize); - let tile_id_chars: Vec<_> = padded_id.chars().collect(); - // Create the final path using groups of threes - let tile_id_component = tile_id_chars - .rchunks(3) - .fold(PathBuf::new(), |acc, chunk| { - PathBuf::from(chunk.iter().collect::()).join(acc) - }) - .with_extension(extension); + // The width of the largest tile ID of the level, rounded up to a whole number of groups. + let n_digits = (max_id.max(1).ilog10() + 1).next_multiple_of(3) as usize; + let padded_id = format!("{tile_id:0>n_digits$}"); + + let mut path = level_number.to_string(); + // The padded ID is ASCII, and its length is a multiple of 3. + for group_start in (0..n_digits).step_by(3) { + path.push('/'); + path.push_str(&padded_id[group_start..group_start + 3]); + } + if !extension.is_empty() { + path.push('.'); + path.push_str(extension); + } + + Ok(path) + } + + /// Returns [`GraphId::relative_path`] as a [`PathBuf`], with the separators of the platform. + /// + /// # Errors + /// + /// This will fail if the tile is invalid for this level of tiling. + pub fn file_path(&self, extension: &str) -> Result { + Ok(self.relative_path(extension)?.split('/').collect()) + } +} + +impl FromStr for GraphId { + type Err = ParseGraphIdError; + + /// Parses a graph ID from its packed `u64` value (e.g. `16889572344463360`) + /// or from the `level/tile_id/index` form (e.g. `2/762485/42`). + /// + /// To parse a whole tile in the `level/tile_id` form, use [`GraphId::parse_tile`]. + fn from_str(input: &str) -> Result { + if let Ok(id) = input.trim().parse::() { + return GraphId::try_from_id(id).map_err(|source| ParseGraphIdError::Invalid { + input: input.to_string(), + source, + }); + } + + let [level, tile_id, index] = parse_components( + input, + "level/tile_id/index", + ["level", "tile ID", "feature index"], + )?; + components_to_graph_id(input, level, tile_id, index) + } +} - // Build and return the final string - Ok(PathBuf::from(self.level().to_string()).join(tile_id_component)) +/// Splits `input` into exactly `N` `/`-separated integer components. +fn parse_components( + input: &str, + expected: &'static str, + names: [&'static str; N], +) -> Result<[u64; N], ParseGraphIdError> { + let format_error = || ParseGraphIdError::Format { + input: input.to_string(), + expected, + }; + + let mut parts = input.split('/'); + let mut values = [0; N]; + for (value, component) in values.iter_mut().zip(names) { + let part = parts.next().ok_or_else(format_error)?; + *value = part + .trim() + .parse() + .map_err(|_| ParseGraphIdError::Component { + input: input.to_string(), + component, + })?; + } + if parts.next().is_some() { + return Err(format_error()); } + + Ok(values) +} + +/// Converts parsed components into a graph ID, and reports range errors against `input`. +fn components_to_graph_id( + input: &str, + level: u64, + tile_id: u64, + index: u64, +) -> Result { + let invalid = |source| ParseGraphIdError::Invalid { + input: input.to_string(), + source, + }; + let level = u8::try_from(level).map_err(|_| invalid(InvalidGraphIdError::Level))?; + GraphId::try_from_components(level, tile_id, index).map_err(invalid) } impl Ord for GraphId { @@ -423,4 +543,152 @@ mod tests { Ok("3/001/000/000.gph".into()) ); } + + fn tile(level: u8, tile_id: u64) -> GraphId { + GraphId::try_from_components(level, tile_id, 0).expect("valid tile ID in test") + } + + #[test] + fn test_relative_path() { + // Levels 0 and 1 pad the tile ID to 6 digits, and level 2 pads it to 9. + for (graph_id, expected) in [ + (tile(0, 3015), "0/003/015.gph"), + (tile(0, 49), "0/000/049.gph"), + (tile(1, 47_701), "1/047/701.gph"), + (tile(1, 0), "1/000/000.gph"), + (tile(2, 762_485), "2/000/762/485.gph"), + (tile(2, 1_036_799), "2/001/036/799.gph"), + ] { + assert_eq!(graph_id.relative_path("gph").as_deref(), Ok(expected)); + assert_eq!( + graph_id.file_path("gph"), + Ok(expected.split('/').collect::()) + ); + } + + let with_index = GraphId::try_from_components(2, 762_485, 42).unwrap(); + assert_eq!( + with_index.relative_path("gph").as_deref(), + Ok("2/000/762/485.gph") + ); + + assert_eq!( + tile(1, 47_701).relative_path("").as_deref(), + Ok("1/047/701") + ); + } + + #[test] + fn test_relative_path_invalid() { + // Level 0 has 45 × 90 tiles, so its largest tile ID is 4049. + assert_eq!( + tile(0, 4049).relative_path("gph").as_deref(), + Ok("0/004/049.gph") + ); + assert_eq!( + tile(0, 4050).relative_path("gph"), + Err(InvalidGraphIdError::GraphTileId) + ); + // Levels 4 to 7 fit in the bit field, but they have no tiling system. + assert_eq!( + tile(5, 0).relative_path("gph"), + Err(InvalidGraphIdError::Level) + ); + } + + #[test] + fn test_parse_tile() { + assert_eq!(GraphId::parse_tile("2/763926"), Ok(tile(2, 763_926))); + assert_eq!(GraphId::parse_tile(" 0 / 3015 "), Ok(tile(0, 3015))); + } + + #[test] + fn test_parse_tile_invalid() { + let format = |input: &str| ParseGraphIdError::Format { + input: input.to_string(), + expected: "level/tile_id", + }; + assert_eq!(GraphId::parse_tile("2"), Err(format("2"))); + assert_eq!(GraphId::parse_tile("2/763926/7"), Err(format("2/763926/7"))); + assert_eq!( + GraphId::parse_tile("2/"), + Err(ParseGraphIdError::Component { + input: "2/".to_string(), + component: "tile ID", + }) + ); + assert_eq!( + GraphId::parse_tile("x/763926"), + Err(ParseGraphIdError::Component { + input: "x/763926".to_string(), + component: "level", + }) + ); + assert_eq!( + GraphId::parse_tile("-1/0"), + Err(ParseGraphIdError::Component { + input: "-1/0".to_string(), + component: "level", + }) + ); + // Levels have 3 bits, and tile IDs have 22 bits. + for (input, source) in [ + ("8/0", InvalidGraphIdError::Level), + ("256/0", InvalidGraphIdError::Level), + ("0/4194304", InvalidGraphIdError::GraphTileId), + ] { + assert_eq!( + GraphId::parse_tile(input), + Err(ParseGraphIdError::Invalid { + input: input.to_string(), + source, + }) + ); + } + } + + #[test] + fn test_from_str() { + let expected = GraphId::try_from_components(2, 762_485, 42).unwrap(); + assert_eq!("2/762485/42".parse(), Ok(expected)); + assert_eq!(expected.value().to_string().parse(), Ok(expected)); + assert_eq!( + " 16889572344463360 " + .parse::() + .map(|id| id.value()), + Ok(16_889_572_344_463_360) + ); + } + + #[test] + fn test_from_str_invalid() { + assert_eq!( + "2/762485".parse::(), + Err(ParseGraphIdError::Format { + input: "2/762485".to_string(), + expected: "level/tile_id/index", + }) + ); + assert_eq!( + "2/762485/x".parse::(), + Err(ParseGraphIdError::Component { + input: "2/762485/x".to_string(), + component: "feature index", + }) + ); + assert_eq!( + INVALID_GRAPH_ID.to_string().parse::(), + Err(ParseGraphIdError::Invalid { + input: INVALID_GRAPH_ID.to_string(), + source: InvalidGraphIdError::InvalidGraphId, + }) + ); + assert_eq!( + "0/0/2097152".parse::(), + Err(ParseGraphIdError::Invalid { + input: "0/0/2097152".to_string(), + source: InvalidGraphIdError::TileIndex, + }) + ); + } } diff --git a/valhalla-graphtile/src/lib.rs b/valhalla-graphtile/src/lib.rs index 179c3c5..0db98cd 100644 --- a/valhalla-graphtile/src/lib.rs +++ b/valhalla-graphtile/src/lib.rs @@ -19,7 +19,7 @@ use serde::{Deserialize, Serialize}; // The implementations are sufficiently complex that we want to have lots of files, // But many of those only have one or two useful definitions to re-export, // so this flattens things for better ergonomics. -pub use graph_id::GraphId; +pub use graph_id::{GraphId, InvalidGraphIdError, ParseGraphIdError}; /// Road class; broad hierarchies of relative (and sometimes locally specific) importance. /// From 462597b77b143a767bb70afbf8faf1144d7b952f Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Mon, 2 Nov 2026 09:30:23 -0800 Subject: [PATCH 17/37] Move the valinor-cli changes to their own pull request This pull request now contains only the extract writer. The split of valinor-cli into subcommand modules moves to a separate pull request, which builds on this one. The extract subcommand moves there too. valinor-cli is the same as on main again. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 - valinor-cli/Cargo.toml | 1 - valinor-cli/README.md | 25 +-- valinor-cli/src/commands/extract.rs | 265 --------------------------- valinor-cli/src/commands/get_edge.rs | 127 ------------- valinor-cli/src/commands/mod.rs | 4 - valinor-cli/src/main.rs | 209 +++++++++++++-------- 7 files changed, 138 insertions(+), 494 deletions(-) delete mode 100644 valinor-cli/src/commands/extract.rs delete mode 100644 valinor-cli/src/commands/get_edge.rs delete mode 100644 valinor-cli/src/commands/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 1c1792d..a4c24cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2286,7 +2286,6 @@ dependencies = [ "anyhow", "clap", "serde_json", - "tempfile", "tracing", "tracing-subscriber", "valhalla-graphtile", diff --git a/valinor-cli/Cargo.toml b/valinor-cli/Cargo.toml index 1961b76..cc0b362 100644 --- a/valinor-cli/Cargo.toml +++ b/valinor-cli/Cargo.toml @@ -12,7 +12,6 @@ rust-version.workspace = true anyhow = { workspace = true } clap = { workspace = true, features = ["derive", "env"] } serde_json = { workspace = true } -tempfile = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter"] } valhalla-graphtile = { workspace = true, features = ["serde"] } diff --git a/valinor-cli/README.md b/valinor-cli/README.md index df299af..f620a82 100644 --- a/valinor-cli/README.md +++ b/valinor-cli/README.md @@ -1,8 +1,8 @@ # Valinor CLI A CLI multi-tool for quickly inspecting Valhalla graphs. -Most subcommands take a Valhalla JSON configuration as input -(`extract` is the exception; it takes a tile extract tarball or a tile directory directly). +This utility takes a Valhalla JSON configuration as input, +and has subcommands with various functionality. This is primarily meant as a low-level debugging tool for Valinor developers. If this looks useful to you, it probably is; otherwise, it's probably not. @@ -12,26 +12,9 @@ If this looks useful to you, it probably is; otherwise, it's probably not. As an example, you can get edge info like this (works with either the integer or slash separated form): ```shell -RUST_LOG=info cargo run --package valinor-cli -- get-edge ~/valhalla-docker/valhalla/valhalla.json 901881911882 +RUST_LOG=info cargo run --package valinor-cli -- ~/valhalla-docker/valhalla/valhalla.json get-edge 901881911882 ``` Then you'll get some JSON output on your terminal with the details. -## Example: writing an offline sub-extract - -You can write a smaller, fully spec-compliant extract -which contains only some tiles of a source (each tile is `level/tile_id`). -The source can be a tile extract tarball or a tile directory: - -```shell -cargo run --package valinor-cli -- extract valhalla-graphtile/fixtures/andorra-tiles.tar andorra-sub.tar --tile 0/3015,1/47701,2/763926 -cargo run --package valinor-cli -- extract valhalla-graphtile/fixtures/andorra-tiles andorra-sub.tar --tile 0/3015,1/47701,2/763926 -``` - -A one-line summary (tile count and output size) is printed on stderr. -The output is interchangeable with `valhalla_build_extract` output. -The command writes to a temporary file in the output directory, -and renames it to the output path only when the extract is complete. -So the output path never contains a partial extract. - -Run the `help` subcommand for more details (everything is automatically documented via clap). +Run the `help` subcommand for more details (everything is automatically documented via clap). \ No newline at end of file diff --git a/valinor-cli/src/commands/extract.rs b/valinor-cli/src/commands/extract.rs deleted file mode 100644 index 206e678..0000000 --- a/valinor-cli/src/commands/extract.rs +++ /dev/null @@ -1,265 +0,0 @@ -//! The `extract` subcommand: writes an offline sub-extract of a tile extract or tile directory. - -use std::{ - io::BufWriter, - num::NonZeroUsize, - path::{Path, PathBuf}, -}; - -#[cfg(unix)] -use std::{fs::Permissions, os::unix::fs::PermissionsExt}; - -use anyhow::Context; -use clap::Args; -use tempfile::NamedTempFile; -use valhalla_graphtile::GraphId; -use valhalla_graphtile::extract::write_extract; -use valhalla_graphtile::tile_provider::{ - DirectoryGraphTileProvider, TarballTileProvider, TileBytesSource, -}; - -/// Write an offline sub-extract of a tile extract tarball or a tile directory -/// -/// The output is an extract tarball in the format of `valhalla_build_extract`, -/// with the given tiles copied byte for byte. -#[derive(Args, Debug)] -pub(crate) struct ExtractArgs { - /// Path to the source: a tile extract tarball (e.g. tiles.tar) - /// or a tile directory (e.g. the `tile_dir` of a Valhalla config) - source: PathBuf, - - /// Path to write the output extract tarball to - output: PathBuf, - - /// A tile to copy, as level/tile_id (e.g. `--tile 2/763926`) - /// - /// Give the option once for each tile, - /// or give a comma-separated list (e.g. `--tile 0/3015,1/47701`). - #[arg( - long = "tile", - value_name = "LEVEL/TILE_ID", - required = true, - value_delimiter = ',', - value_parser = GraphId::parse_tile - )] - pub(crate) tiles: Vec, -} - -/// A successful extract, as reported on stderr. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct ExtractSummary { - tile_count: usize, - bytes_written: u64, -} - -/// Writes the sub-extract from the tile directory or the tarball at the source path. -fn write_sub_extract(args: &ExtractArgs) -> anyhow::Result { - let bytes_written = if args.source.is_dir() { - // The writer reads each tile once, so a cache is of no use. - let provider = DirectoryGraphTileProvider::new(args.source.clone(), NonZeroUsize::MIN); - write_atomically(&provider, &args.tiles, &args.output)? - } else { - let provider = TarballTileProvider::new_readonly(&args.source).with_context(|| { - format!( - "Failed to open source tile extract at {}", - args.source.display() - ) - })?; - write_atomically(&provider, &args.tiles, &args.output)? - }; - - Ok(ExtractSummary { - tile_count: args.tiles.len(), - bytes_written, - }) -} - -/// Writes the extract to a temporary file next to `output`, and then renames it to `output`. -/// -/// If the write fails, `output` keeps its previous contents. -fn write_atomically(source: &S, tiles: &[GraphId], output: &Path) -> anyhow::Result -where - S: TileBytesSource, - S::Error: Send + Sync, -{ - // The rename is atomic only within one file system. - let output_dir = output - .parent() - .filter(|parent| !parent.as_os_str().is_empty()) - .unwrap_or(Path::new(".")); - - let mut builder = tempfile::Builder::new(); - builder.prefix(".valinor-extract-").suffix(".tar.partial"); - // Use the permissions of `File::create`, not the owner-only default of `tempfile`. - #[cfg(unix)] - builder.permissions(Permissions::from_mode(0o666)); - let temp_file: NamedTempFile = builder.tempfile_in(output_dir).with_context(|| { - format!( - "Failed to create a temporary file in {}", - output_dir.display() - ) - })?; - - let mut dest = BufWriter::new(temp_file); - let bytes_written = - write_extract(source, tiles, &mut dest).context("Failed to write the extract")?; - let temp_file = dest - .into_inner() - .map_err(|e| e.into_error()) - .context("Failed to flush the output extract")?; - // Make sure that the data is on disk before the rename makes the file visible. - temp_file - .as_file() - .sync_all() - .context("Failed to sync the output extract to disk")?; - temp_file.persist(output).with_context(|| { - format!( - "Failed to move the output extract into place at {}", - output.display() - ) - })?; - - Ok(bytes_written) -} - -/// Runs the `extract` subcommand, and prints a one-line summary on stderr. -pub(crate) fn run(args: &ExtractArgs) -> anyhow::Result<()> { - let summary = write_sub_extract(args)?; - let noun = if summary.tile_count == 1 { - "tile" - } else { - "tiles" - }; - eprintln!( - "Extracted {} {noun} ({} bytes) to {}", - summary.tile_count, - summary.bytes_written, - args.output.display() - ); - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::fs; - - /// Path to a fixture in the sibling `valhalla-graphtile` crate. - fn fixture(name: &str) -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("..") - .join("valhalla-graphtile") - .join("fixtures") - .join(name) - } - - fn tile(level: u8, tile_id: u64) -> GraphId { - GraphId::try_from_components(level, tile_id, 0).expect("valid tile ID in test") - } - - // Tragically, memory maps are not supported by Miri! - #[cfg(not(miri))] - #[test] - fn extract_round_trips_from_a_tarball_and_a_directory() { - let temp_dir = tempfile::tempdir().expect("a temporary directory is created"); - let tiles = vec![tile(0, 3015), tile(1, 47_701), tile(2, 763_926)]; - - let mut outputs = Vec::new(); - for (source, name) in [ - (fixture("andorra-tiles.tar"), "from-tarball.tar"), - (fixture("andorra-tiles"), "from-directory.tar"), - ] { - let output = temp_dir.path().join(name); - let args = ExtractArgs { - source, - output: output.clone(), - tiles: tiles.clone(), - }; - - let summary = write_sub_extract(&args).expect("the extract writes cleanly"); - assert_eq!(summary.tile_count, tiles.len()); - - let on_disk = fs::metadata(&output).expect("the output exists").len(); - assert_eq!(summary.bytes_written, on_disk); - - let extract = TarballTileProvider::new_readonly(&output) - .expect("the generated extract opens cleanly"); - let mut extracted: Vec = extract.tile_ids().copied().collect(); - extracted.sort_unstable(); - let mut expected = tiles.clone(); - expected.sort_unstable(); - assert_eq!(extracted, expected); - - outputs.push(fs::read(&output).expect("the output is readable")); - } - - assert_eq!( - outputs[0], outputs[1], - "the tarball and directory fixtures give the same extract" - ); - - let mut names: Vec<_> = fs::read_dir(temp_dir.path()) - .expect("the directory is readable") - .map(|entry| entry.expect("the entry is readable").file_name()) - .collect(); - names.sort_unstable(); - assert_eq!(names, ["from-directory.tar", "from-tarball.tar"]); - } - - #[cfg(not(miri))] - #[test] - fn failed_extract_keeps_the_previous_output() { - let temp_dir = tempfile::tempdir().expect("a temporary directory is created"); - let output = temp_dir.path().join("andorra-sub.tar"); - fs::write(&output, b"previous contents").expect("the previous output is written"); - - let args = ExtractArgs { - source: fixture("andorra-tiles.tar"), - output: output.clone(), - // The fixture has no tile 2/0. - tiles: vec![tile(0, 3015), tile(2, 0)], - }; - - let error = write_sub_extract(&args).expect_err("a missing tile fails the extract"); - assert!( - format!("{error:#}").contains("is not in the source"), - "{error:#}" - ); - assert_eq!( - fs::read(&output).expect("the previous output still exists"), - b"previous contents" - ); - assert_eq!( - fs::read_dir(temp_dir.path()) - .expect("the directory is readable") - .count(), - 1, - "the temporary file is removed" - ); - } - - #[cfg(unix)] - #[cfg(not(miri))] - #[test] - fn output_has_the_permissions_of_a_new_file() { - let temp_dir = tempfile::tempdir().expect("a temporary directory is created"); - let output = temp_dir.path().join("andorra-sub.tar"); - let reference = temp_dir.path().join("reference"); - fs::File::create(&reference).expect("the reference file is created"); - - let args = ExtractArgs { - source: fixture("andorra-tiles.tar"), - output: output.clone(), - tiles: vec![tile(0, 3015)], - }; - write_sub_extract(&args).expect("the extract writes cleanly"); - - let mode = |path: &Path| { - fs::metadata(path) - .expect("the file exists") - .permissions() - .mode() - }; - assert_eq!(mode(&output), mode(&reference)); - } -} diff --git a/valinor-cli/src/commands/get_edge.rs b/valinor-cli/src/commands/get_edge.rs deleted file mode 100644 index 211cef9..0000000 --- a/valinor-cli/src/commands/get_edge.rs +++ /dev/null @@ -1,127 +0,0 @@ -//! The `get-edge` subcommand: pretty-prints information about a directed edge. - -use std::{ - fs, - path::{Path, PathBuf}, -}; - -use anyhow::{Context, anyhow}; -use clap::Args; -use serde_json::Value as JsonValue; -use tracing::info; -use valhalla_graphtile::tile_provider::TrafficTileProvider; -use valhalla_graphtile::{ - GraphId, - graph_tile::GraphTile, - tile_provider::{DirectoryGraphTileProvider, GraphTileProvider, TarballTileProvider}, -}; - -/// Pretty-print information about a directed edge (incl. live traffic, if available) -#[derive(Args, Debug)] -pub(crate) struct GetEdgeArgs { - /// Path to valhalla.json - #[arg(env)] - valhalla_config: PathBuf, - - /// Graph ID (u64) or slash-form level/tile_id/index - pub(crate) graph_id: GraphId, -} - -#[derive(Debug, Clone)] -struct DataSources { - routing_graph: Option, - traffic_extract: Option, -} - -#[derive(Debug, Clone)] -enum RoutingGraphDataSource { - Tarball(PathBuf), - TileDir(PathBuf), -} - -fn parse_valhalla_data_paths(path: &Path) -> anyhow::Result { - let bytes = - fs::read(path).with_context(|| format!("Failed to read config at {}", path.display()))?; - let json: JsonValue = - serde_json::from_slice(&bytes).context("Invalid JSON in valhalla config")?; - - let get_path_if_exists = |key: &str| -> Option { - if let JsonValue::String(s) = &json["mjolnir"][key] - && !s.is_empty() - && fs::exists(s).unwrap_or_default() - { - Some(PathBuf::from(s)) - } else { - None - } - }; - - let tile_extract = get_path_if_exists("tile_extract"); - let tile_dir = get_path_if_exists("tile_dir"); - let traffic_extract = get_path_if_exists("traffic_extract"); - Ok(DataSources { - routing_graph: match (tile_extract, tile_dir) { - (Some(tarball), _) => Some(RoutingGraphDataSource::Tarball(tarball)), - (_, Some(dir)) => Some(RoutingGraphDataSource::TileDir(dir)), - (None, None) => None, - }, - traffic_extract, - }) -} - -fn pretty_print_edge_info( - provider: &T, - traffic_provider: Option<&TrafficTileProvider>, - gid: GraphId, -) -> anyhow::Result<()> { - let output = provider.with_tile_containing(gid, |tile| { - let edge = tile.get_directed_edge(gid)?; - let edge_info = tile.get_edge_info(edge)?; - let traffic_info = traffic_provider.map(|tp| unsafe { tp.get_speeds_for_edge(gid).ok() }); - - Ok::(serde_json::json!({ - "graph_id": gid, - "directed_edge": edge, - "edge_info": edge_info, - "traffic": traffic_info, - })) - })??; - println!("{}", serde_json::to_string_pretty(&output)?); - Ok(()) -} - -/// Runs the `get-edge` subcommand. -pub(crate) fn run(args: &GetEdgeArgs) -> anyhow::Result<()> { - let gid = args.graph_id; - let sources = parse_valhalla_data_paths(&args.valhalla_config)?; - let traffic_extract = if let Some(path) = sources.traffic_extract { - info!(path = path.to_str(), "Using traffic extract"); - Some(TrafficTileProvider::new_readonly(path)?) - } else { - info!("No traffic extract could be found"); - None - }; - - // Prefer tarball if set, fall back to directory. - if let Some(graph) = sources.routing_graph { - match graph { - RoutingGraphDataSource::Tarball(path) => { - info!(path = path.to_str(), "Using tarball tile extract"); - - let provider = TarballTileProvider::new_readonly(&path)?; - pretty_print_edge_info(&provider, traffic_extract.as_ref(), gid) - } - RoutingGraphDataSource::TileDir(path) => { - info!(path = path.to_str(), "Using tile directory"); - - let provider = - DirectoryGraphTileProvider::new(path, std::num::NonZeroUsize::new(1).unwrap()); - pretty_print_edge_info(&provider, traffic_extract.as_ref(), gid) - } - } - } else { - Err(anyhow!( - "No routing graph data sources could be loaded. Expected a valid 'tile_extract' (tarball) or 'tile_dir' in the config." - )) - } -} diff --git a/valinor-cli/src/commands/mod.rs b/valinor-cli/src/commands/mod.rs deleted file mode 100644 index f23f125..0000000 --- a/valinor-cli/src/commands/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -//! The subcommands of the CLI, with one module for each subcommand. - -pub(crate) mod extract; -pub(crate) mod get_edge; diff --git a/valinor-cli/src/main.rs b/valinor-cli/src/main.rs index 7ed9e99..17eb3be 100644 --- a/valinor-cli/src/main.rs +++ b/valinor-cli/src/main.rs @@ -1,15 +1,26 @@ -mod commands; +use std::{fs, path::PathBuf, str::FromStr}; +use anyhow::{Context, anyhow}; use clap::{Parser, Subcommand}; +use serde_json::Value as JsonValue; +use tracing::info; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::{EnvFilter, Layer}; - -use commands::{extract, get_edge}; +use valhalla_graphtile::tile_provider::TrafficTileProvider; +use valhalla_graphtile::{ + GraphId, + graph_tile::GraphTile, + tile_provider::{DirectoryGraphTileProvider, GraphTileProvider, TarballTileProvider}, +}; #[derive(Parser, Debug)] #[command(name = "valinor-cli", author, version, about, long_about = None)] struct Cli { + /// Path to valhalla.json + #[arg(env)] + valhalla_config: PathBuf, + /// Subcommand/tool to run #[command(subcommand)] command: Commands, @@ -17,8 +28,94 @@ struct Cli { #[derive(Subcommand, Debug)] enum Commands { - GetEdge(get_edge::GetEdgeArgs), - Extract(extract::ExtractArgs), + /// Pretty-print information about a directed edge (incl. live traffic, if available) + GetEdge { + /// Graph ID (u64) or slash-form level/tile/index + graph_id: String, + }, +} + +fn parse_graph_id(input: &str) -> anyhow::Result { + // Try to parse as a pure integer. + if let Ok(id) = input.parse::() { + return GraphId::try_from_id(id).map_err(|e| anyhow!(e)); + } + + // Try slash-separated level/tile/index + let parts: Vec<&str> = input.split('/').collect(); + if parts.len() == 3 { + let level = u8::from_str(parts[0]).context("invalid level in graph id")?; + let tile_id = u64::from_str(parts[1]).context("invalid tile id in graph id")?; + let index = u64::from_str(parts[2]).context("invalid index in graph id")?; + return GraphId::try_from_components(level, tile_id, index).map_err(|e| anyhow!(e)); + } + + Err(anyhow!( + "Unrecognized graph id format. Use a u64 integer or level/tile/index" + )) +} + +#[derive(Debug, Clone)] +struct DataSources { + routing_graph: Option, + traffic_extract: Option, +} + +#[derive(Debug, Clone)] +enum RoutingGraphDataSource { + Tarball(PathBuf), + TileDir(PathBuf), +} + +fn parse_valhalla_data_paths(path: &PathBuf) -> anyhow::Result { + let bytes = + fs::read(path).with_context(|| format!("Failed to read config at {}", path.display()))?; + let json: JsonValue = + serde_json::from_slice(&bytes).context("Invalid JSON in valhalla config")?; + + let get_path_if_exists = |key: &str| -> Option { + if let JsonValue::String(s) = &json["mjolnir"][key] + && !s.is_empty() + && fs::exists(s).unwrap_or_default() + { + Some(PathBuf::from(s)) + } else { + None + } + }; + + let tile_extract = get_path_if_exists("tile_extract"); + let tile_dir = get_path_if_exists("tile_dir"); + let traffic_extract = get_path_if_exists("traffic_extract"); + Ok(DataSources { + routing_graph: match (tile_extract, tile_dir) { + (Some(tarball), _) => Some(RoutingGraphDataSource::Tarball(tarball)), + (_, Some(dir)) => Some(RoutingGraphDataSource::TileDir(dir)), + (None, None) => None, + }, + traffic_extract, + }) +} + +fn pretty_print_edge_info( + provider: &T, + traffic_provider: Option<&TrafficTileProvider>, + gid: GraphId, +) -> anyhow::Result<()> { + let output = provider.with_tile_containing(gid, |tile| { + let edge = tile.get_directed_edge(gid)?; + let edge_info = tile.get_edge_info(edge)?; + let traffic_info = traffic_provider.map(|tp| unsafe { tp.get_speeds_for_edge(gid).ok() }); + + Ok::(serde_json::json!({ + "graph_id": gid, + "directed_edge": edge, + "edge_info": edge_info, + "traffic": traffic_info, + })) + })??; + println!("{}", serde_json::to_string_pretty(&output)?); + Ok(()) } fn main() -> anyhow::Result<()> { @@ -30,79 +127,41 @@ fn main() -> anyhow::Result<()> { let cli = Cli::parse(); match cli.command { - Commands::GetEdge(args) => get_edge::run(&args), - Commands::Extract(args) => extract::run(&args), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use clap::CommandFactory; - use valhalla_graphtile::GraphId; - - fn tile(level: u8, tile_index: u64) -> GraphId { - GraphId::try_from_components(level, tile_index, 0).expect("valid tile ID in test") - } - - #[test] - fn clap_derive_is_consistent() { - Cli::command().debug_assert(); - } + Commands::GetEdge { graph_id } => { + let gid = parse_graph_id(&graph_id)?; + let sources = parse_valhalla_data_paths(&cli.valhalla_config)?; + let traffic_extract = if let Some(path) = sources.traffic_extract { + info!(path = path.to_str(), "Using traffic extract"); + Some(TrafficTileProvider::new_readonly(path)?) + } else { + info!("No traffic extract could be found"); + None + }; - #[test] - fn extract_tiles_accept_repeated_and_comma_separated_options() { - let cli = Cli::try_parse_from([ - "valinor-cli", - "extract", - "in.tar", - "out.tar", - "--tile", - "0/3015,1/47701", - "--tile", - "2/763926", - ]) - .expect("the arguments parse"); - let Commands::Extract(args) = cli.command else { - panic!("expected the extract subcommand"); - }; - assert_eq!( - args.tiles, - vec![tile(0, 3015), tile(1, 47_701), tile(2, 763_926)] - ); - } + // Prefer tarball if set, fall back to directory. + if let Some(graph) = sources.routing_graph { + match graph { + RoutingGraphDataSource::Tarball(path) => { + info!(path = path.to_str(), "Using tarball tile extract"); - #[test] - fn extract_tiles_reject_malformed_values() { - for input in ["2", "2/763926/7", "2/abc", "8/0"] { - let error = Cli::try_parse_from([ - "valinor-cli", - "extract", - "in.tar", - "out.tar", - "--tile", - input, - ]) - .expect_err("a malformed tile is rejected"); - assert!(error.to_string().contains(input), "{input}: {error}"); - } - } + let provider = TarballTileProvider::new_readonly(&path)?; + pretty_print_edge_info(&provider, traffic_extract.as_ref(), gid) + } + RoutingGraphDataSource::TileDir(path) => { + info!(path = path.to_str(), "Using tile directory"); - #[test] - fn get_edge_accepts_both_graph_id_forms() { - let expected = GraphId::try_from_components(2, 762_485, 42).expect("valid graph ID"); - for input in [expected.value().to_string(), "2/762485/42".to_string()] { - let cli = Cli::try_parse_from(["valinor-cli", "get-edge", "valhalla.json", &input]) - .expect("the arguments parse"); - let Commands::GetEdge(args) = cli.command else { - panic!("expected the get-edge subcommand"); - }; - assert_eq!(args.graph_id, expected); + let provider = DirectoryGraphTileProvider::new( + path, + std::num::NonZeroUsize::new(1).unwrap(), + ); + pretty_print_edge_info(&provider, traffic_extract.as_ref(), gid) + } + } + } else { + Err(anyhow!( + "No routing graph data sources could be loaded. Expected a valid 'tile_extract' (tarball) or 'tile_dir' in the config." + )) + } } } - - #[test] - fn extract_tiles_are_required() { - assert!(Cli::try_parse_from(["valinor-cli", "extract", "in.tar", "out.tar"]).is_err()); - } } From decc8fba1243d1e2c0edba110da816ec2e7e4385 Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Tue, 13 Oct 2026 20:40:51 -0700 Subject: [PATCH 18/37] Split valinor-cli into one module for each subcommand main.rs now contains only the clap entry point, the tracing set-up, and the dispatch. Each subcommand is a module in src/commands with its clap arguments and a run function, so a new subcommand does not make main.rs larger. This is a refactor only: get-edge moves into commands/get_edge.rs, and the help output is the same as before. The only code change is that parse_valhalla_data_paths takes a &Path (clippy: ptr_arg). Co-Authored-By: Claude Opus 5.5 --- valinor-cli/src/commands/get_edge.rs | 144 ++++++++++++++++++++++++++ valinor-cli/src/commands/mod.rs | 6 ++ valinor-cli/src/main.rs | 148 +++------------------------ 3 files changed, 166 insertions(+), 132 deletions(-) create mode 100644 valinor-cli/src/commands/get_edge.rs create mode 100644 valinor-cli/src/commands/mod.rs diff --git a/valinor-cli/src/commands/get_edge.rs b/valinor-cli/src/commands/get_edge.rs new file mode 100644 index 0000000..b201ca4 --- /dev/null +++ b/valinor-cli/src/commands/get_edge.rs @@ -0,0 +1,144 @@ +//! The `get-edge` subcommand: pretty-prints information about a directed edge. + +use std::{ + fs, + path::{Path, PathBuf}, + str::FromStr, +}; + +use anyhow::{Context, anyhow}; +use clap::Args; +use serde_json::Value as JsonValue; +use tracing::info; +use valhalla_graphtile::tile_provider::TrafficTileProvider; +use valhalla_graphtile::{ + GraphId, + graph_tile::GraphTile, + tile_provider::{DirectoryGraphTileProvider, GraphTileProvider, TarballTileProvider}, +}; + +/// Pretty-print information about a directed edge (incl. live traffic, if available) +#[derive(Args, Debug)] +pub(crate) struct GetEdgeArgs { + /// Graph ID (u64) or slash-form level/tile/index + graph_id: String, +} + +fn parse_graph_id(input: &str) -> anyhow::Result { + // Try to parse as a pure integer. + if let Ok(id) = input.parse::() { + return GraphId::try_from_id(id).map_err(|e| anyhow!(e)); + } + + // Try slash-separated level/tile/index + let parts: Vec<&str> = input.split('/').collect(); + if parts.len() == 3 { + let level = u8::from_str(parts[0]).context("invalid level in graph id")?; + let tile_id = u64::from_str(parts[1]).context("invalid tile id in graph id")?; + let index = u64::from_str(parts[2]).context("invalid index in graph id")?; + return GraphId::try_from_components(level, tile_id, index).map_err(|e| anyhow!(e)); + } + + Err(anyhow!( + "Unrecognized graph id format. Use a u64 integer or level/tile/index" + )) +} + +#[derive(Debug, Clone)] +struct DataSources { + routing_graph: Option, + traffic_extract: Option, +} + +#[derive(Debug, Clone)] +enum RoutingGraphDataSource { + Tarball(PathBuf), + TileDir(PathBuf), +} + +fn parse_valhalla_data_paths(path: &Path) -> anyhow::Result { + let bytes = + fs::read(path).with_context(|| format!("Failed to read config at {}", path.display()))?; + let json: JsonValue = + serde_json::from_slice(&bytes).context("Invalid JSON in valhalla config")?; + + let get_path_if_exists = |key: &str| -> Option { + if let JsonValue::String(s) = &json["mjolnir"][key] + && !s.is_empty() + && fs::exists(s).unwrap_or_default() + { + Some(PathBuf::from(s)) + } else { + None + } + }; + + let tile_extract = get_path_if_exists("tile_extract"); + let tile_dir = get_path_if_exists("tile_dir"); + let traffic_extract = get_path_if_exists("traffic_extract"); + Ok(DataSources { + routing_graph: match (tile_extract, tile_dir) { + (Some(tarball), _) => Some(RoutingGraphDataSource::Tarball(tarball)), + (_, Some(dir)) => Some(RoutingGraphDataSource::TileDir(dir)), + (None, None) => None, + }, + traffic_extract, + }) +} + +fn pretty_print_edge_info( + provider: &T, + traffic_provider: Option<&TrafficTileProvider>, + gid: GraphId, +) -> anyhow::Result<()> { + let output = provider.with_tile_containing(gid, |tile| { + let edge = tile.get_directed_edge(gid)?; + let edge_info = tile.get_edge_info(edge)?; + let traffic_info = traffic_provider.map(|tp| unsafe { tp.get_speeds_for_edge(gid).ok() }); + + Ok::(serde_json::json!({ + "graph_id": gid, + "directed_edge": edge, + "edge_info": edge_info, + "traffic": traffic_info, + })) + })??; + println!("{}", serde_json::to_string_pretty(&output)?); + Ok(()) +} + +/// Runs the `get-edge` subcommand with the given Valhalla config. +pub(crate) fn run(valhalla_config: &Path, args: &GetEdgeArgs) -> anyhow::Result<()> { + let gid = parse_graph_id(&args.graph_id)?; + let sources = parse_valhalla_data_paths(valhalla_config)?; + let traffic_extract = if let Some(path) = sources.traffic_extract { + info!(path = path.to_str(), "Using traffic extract"); + Some(TrafficTileProvider::new_readonly(path)?) + } else { + info!("No traffic extract could be found"); + None + }; + + // Prefer tarball if set, fall back to directory. + if let Some(graph) = sources.routing_graph { + match graph { + RoutingGraphDataSource::Tarball(path) => { + info!(path = path.to_str(), "Using tarball tile extract"); + + let provider = TarballTileProvider::new_readonly(&path)?; + pretty_print_edge_info(&provider, traffic_extract.as_ref(), gid) + } + RoutingGraphDataSource::TileDir(path) => { + info!(path = path.to_str(), "Using tile directory"); + + let provider = + DirectoryGraphTileProvider::new(path, std::num::NonZeroUsize::new(1).unwrap()); + pretty_print_edge_info(&provider, traffic_extract.as_ref(), gid) + } + } + } else { + Err(anyhow!( + "No routing graph data sources could be loaded. Expected a valid 'tile_extract' (tarball) or 'tile_dir' in the config." + )) + } +} diff --git a/valinor-cli/src/commands/mod.rs b/valinor-cli/src/commands/mod.rs new file mode 100644 index 0000000..24c714d --- /dev/null +++ b/valinor-cli/src/commands/mod.rs @@ -0,0 +1,6 @@ +//! The subcommands of the CLI. +//! +//! Each subcommand has its own module, +//! which contains its clap arguments and a `run` function. + +pub(crate) mod get_edge; diff --git a/valinor-cli/src/main.rs b/valinor-cli/src/main.rs index 17eb3be..02a1b68 100644 --- a/valinor-cli/src/main.rs +++ b/valinor-cli/src/main.rs @@ -1,18 +1,13 @@ -use std::{fs, path::PathBuf, str::FromStr}; +mod commands; + +use std::path::PathBuf; -use anyhow::{Context, anyhow}; use clap::{Parser, Subcommand}; -use serde_json::Value as JsonValue; -use tracing::info; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::{EnvFilter, Layer}; -use valhalla_graphtile::tile_provider::TrafficTileProvider; -use valhalla_graphtile::{ - GraphId, - graph_tile::GraphTile, - tile_provider::{DirectoryGraphTileProvider, GraphTileProvider, TarballTileProvider}, -}; + +use commands::get_edge; #[derive(Parser, Debug)] #[command(name = "valinor-cli", author, version, about, long_about = None)] @@ -28,94 +23,7 @@ struct Cli { #[derive(Subcommand, Debug)] enum Commands { - /// Pretty-print information about a directed edge (incl. live traffic, if available) - GetEdge { - /// Graph ID (u64) or slash-form level/tile/index - graph_id: String, - }, -} - -fn parse_graph_id(input: &str) -> anyhow::Result { - // Try to parse as a pure integer. - if let Ok(id) = input.parse::() { - return GraphId::try_from_id(id).map_err(|e| anyhow!(e)); - } - - // Try slash-separated level/tile/index - let parts: Vec<&str> = input.split('/').collect(); - if parts.len() == 3 { - let level = u8::from_str(parts[0]).context("invalid level in graph id")?; - let tile_id = u64::from_str(parts[1]).context("invalid tile id in graph id")?; - let index = u64::from_str(parts[2]).context("invalid index in graph id")?; - return GraphId::try_from_components(level, tile_id, index).map_err(|e| anyhow!(e)); - } - - Err(anyhow!( - "Unrecognized graph id format. Use a u64 integer or level/tile/index" - )) -} - -#[derive(Debug, Clone)] -struct DataSources { - routing_graph: Option, - traffic_extract: Option, -} - -#[derive(Debug, Clone)] -enum RoutingGraphDataSource { - Tarball(PathBuf), - TileDir(PathBuf), -} - -fn parse_valhalla_data_paths(path: &PathBuf) -> anyhow::Result { - let bytes = - fs::read(path).with_context(|| format!("Failed to read config at {}", path.display()))?; - let json: JsonValue = - serde_json::from_slice(&bytes).context("Invalid JSON in valhalla config")?; - - let get_path_if_exists = |key: &str| -> Option { - if let JsonValue::String(s) = &json["mjolnir"][key] - && !s.is_empty() - && fs::exists(s).unwrap_or_default() - { - Some(PathBuf::from(s)) - } else { - None - } - }; - - let tile_extract = get_path_if_exists("tile_extract"); - let tile_dir = get_path_if_exists("tile_dir"); - let traffic_extract = get_path_if_exists("traffic_extract"); - Ok(DataSources { - routing_graph: match (tile_extract, tile_dir) { - (Some(tarball), _) => Some(RoutingGraphDataSource::Tarball(tarball)), - (_, Some(dir)) => Some(RoutingGraphDataSource::TileDir(dir)), - (None, None) => None, - }, - traffic_extract, - }) -} - -fn pretty_print_edge_info( - provider: &T, - traffic_provider: Option<&TrafficTileProvider>, - gid: GraphId, -) -> anyhow::Result<()> { - let output = provider.with_tile_containing(gid, |tile| { - let edge = tile.get_directed_edge(gid)?; - let edge_info = tile.get_edge_info(edge)?; - let traffic_info = traffic_provider.map(|tp| unsafe { tp.get_speeds_for_edge(gid).ok() }); - - Ok::(serde_json::json!({ - "graph_id": gid, - "directed_edge": edge, - "edge_info": edge_info, - "traffic": traffic_info, - })) - })??; - println!("{}", serde_json::to_string_pretty(&output)?); - Ok(()) + GetEdge(get_edge::GetEdgeArgs), } fn main() -> anyhow::Result<()> { @@ -127,41 +35,17 @@ fn main() -> anyhow::Result<()> { let cli = Cli::parse(); match cli.command { - Commands::GetEdge { graph_id } => { - let gid = parse_graph_id(&graph_id)?; - let sources = parse_valhalla_data_paths(&cli.valhalla_config)?; - let traffic_extract = if let Some(path) = sources.traffic_extract { - info!(path = path.to_str(), "Using traffic extract"); - Some(TrafficTileProvider::new_readonly(path)?) - } else { - info!("No traffic extract could be found"); - None - }; - - // Prefer tarball if set, fall back to directory. - if let Some(graph) = sources.routing_graph { - match graph { - RoutingGraphDataSource::Tarball(path) => { - info!(path = path.to_str(), "Using tarball tile extract"); + Commands::GetEdge(args) => get_edge::run(&cli.valhalla_config, &args), + } +} - let provider = TarballTileProvider::new_readonly(&path)?; - pretty_print_edge_info(&provider, traffic_extract.as_ref(), gid) - } - RoutingGraphDataSource::TileDir(path) => { - info!(path = path.to_str(), "Using tile directory"); +#[cfg(test)] +mod tests { + use super::*; + use clap::CommandFactory; - let provider = DirectoryGraphTileProvider::new( - path, - std::num::NonZeroUsize::new(1).unwrap(), - ); - pretty_print_edge_info(&provider, traffic_extract.as_ref(), gid) - } - } - } else { - Err(anyhow!( - "No routing graph data sources could be loaded. Expected a valid 'tile_extract' (tarball) or 'tile_dir' in the config." - )) - } - } + #[test] + fn clap_derive_is_consistent() { + Cli::command().debug_assert(); } } From b2898cf65b17e77adfe560eb039cc2ccf003ecae Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Mon, 2 Nov 2026 09:37:30 -0800 Subject: [PATCH 19/37] Write extracts from GraphTileProvider handles The writer now reads tiles through GraphTileProvider, so the separate TileBytesSource trait is gone. It gets each tile size from GraphTileProvider::tile_size before the write, and the tile bytes from GraphTileHandle::tile_bytes during the write. The handles decode each tile, so a corrupt tile is a decoding error. - write_extract takes any iterator of graph IDs. It reduces each ID to its base tile, sorts the tiles in archive order, and removes duplicates in one place. - ExtractLayoutError is gone. ExtractWriteError has the new NoTiles and InvalidIndexBin variants, and Provider replaces Source. The DuplicateTile and EmptyTile errors are gone: duplicate IDs give one tile, and an empty tile file fails to decode. - CountingWriter is gone. The layout already has every offset and the total length, and the tests check the offsets with the tar crate. - INDEX_BIN_ENTRY_NAME is now INDEX_BIN_FILE_NAME, and END_OF_ARCHIVE_LEN is now END_OF_ARCHIVE_MARKER_LEN. - The tests use the Andorra tiles through a test provider that can report wrong sizes and fail reads. New tests cover a tile file that changes after the directory provider caches it, and an empty tile file. Every assert has a message. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/extract/mod.rs | 8 +- valhalla-graphtile/src/extract/writer.rs | 1019 +++++++++-------- .../src/tile_provider/directory.rs | 63 +- valhalla-graphtile/src/tile_provider/mod.rs | 26 +- .../src/tile_provider/tarball.rs | 67 +- 5 files changed, 592 insertions(+), 591 deletions(-) diff --git a/valhalla-graphtile/src/extract/mod.rs b/valhalla-graphtile/src/extract/mod.rs index 4168f49..f9de6c5 100644 --- a/valhalla-graphtile/src/extract/mod.rs +++ b/valhalla-graphtile/src/extract/mod.rs @@ -1,10 +1,6 @@ //! Offline tile extracts: tarballs with a subset of the tiles of a -//! [`TileBytesSource`](crate::tile_provider::TileBytesSource). -//! -//! [`write_extract`] writes an extract in the format of `valhalla_build_extract`, -//! so stock Valhalla and [`TarballTileProvider`](crate::tile_provider::TarballTileProvider) -//! can read it. +//! [`GraphTileProvider`](crate::tile_provider::GraphTileProvider). mod writer; -pub use writer::{ExtractLayoutError, ExtractWriteError, write_extract}; +pub use writer::{ExtractWriteError, write_extract}; diff --git a/valhalla-graphtile/src/extract/writer.rs b/valhalla-graphtile/src/extract/writer.rs index a6d1fbf..2b2c959 100644 --- a/valhalla-graphtile/src/extract/writer.rs +++ b/valhalla-graphtile/src/extract/writer.rs @@ -1,19 +1,20 @@ //! A streaming tar writer for tile extracts. //! //! The writer computes every offset from the tile sizes before it writes, -//! so it writes the archive in one pass. The archive contains: +//! and writes the archive in one pass. The archive contains: //! //! 1. `index.bin`, which Valhalla requires as the first entry of a remote tarball. //! 2. The tiles at the paths of [`GraphId::relative_path`], //! in ascending (level, tile index) order. -//! This is not the `Ord` order of [`GraphId`]. //! 3. The two-block end-of-archive marker. use crate::GraphId; use crate::graph_id::InvalidGraphIdError; +use crate::graph_tile::handles::GraphTileHandle; use crate::tile_provider::{ - GraphTileProviderError, INDEX_BIN_ENTRY_NAME, INDEX_ENTRY_SIZE, InvalidIndexBinError, - TAR_BLOCK_SIZE, TILE_FILE_EXTENSION, TileBytesSource, TileIndexBinEntry, serialize_index_bin, + GraphTileProvider, GraphTileProviderError, INDEX_BIN_FILE_NAME, INDEX_ENTRY_SIZE, + InvalidIndexBinError, TAR_BLOCK_SIZE, TILE_FILE_EXTENSION, TileIndexBinEntry, + serialize_index_bin, }; use std::io::Write; use tar::{EntryType, Header}; @@ -24,147 +25,177 @@ use thiserror::Error; const ZERO_BLOCK: [u8; TAR_BLOCK_SIZE as usize] = [0; TAR_BLOCK_SIZE as usize]; /// The length of the end-of-archive marker: two zero blocks. -const END_OF_ARCHIVE_LEN: u64 = 2 * TAR_BLOCK_SIZE; - -/// An error in the set of tiles of an extract. -/// -/// The writer finds these errors before it writes any bytes. -#[derive(Debug, Error, PartialEq, Eq)] -pub enum ExtractLayoutError { - /// The tile list is empty. - #[error("the extract contains no tiles")] - EmptyManifest, - /// The tile list contains a tile more than once. - /// - /// Graph IDs that differ only in the feature index are the same tile. - #[error("the extract contains tile {0} more than once")] - DuplicateTile(GraphId), - /// The source gives a tile a size of zero. - #[error("the source gives tile {0} a size of zero")] - EmptyTile(GraphId), - /// A tile cannot be indexed. - #[error("invalid index.bin: {0}")] - InvalidIndexBin(#[from] InvalidIndexBinError), -} +const END_OF_ARCHIVE_MARKER_LEN: u64 = 2 * TAR_BLOCK_SIZE; /// An error while writing a tile extract. -/// -/// `E` is the error type of the [`TileBytesSource`]. #[derive(Debug, Error)] -pub enum ExtractWriteError { - /// The requested tiles do not make a valid extract. - #[error("invalid extract: {0}")] - Layout(#[from] ExtractLayoutError), - /// A requested tile is not in the source. - #[error("tile {0} is not in the source")] +pub enum ExtractWriteError { + /// The tile list is empty. + #[error("the extract contains no tiles")] + NoTiles, + /// A requested tile is not in the provider. + #[error("tile {0} is not in the provider")] MissingTile(GraphId), /// A requested graph ID cannot be converted into a tar entry path. #[error("invalid graph ID: {0}")] InvalidGraphId(#[from] InvalidGraphIdError), - /// The source produced a tile whose length differs from the size that it reported, + /// A tile cannot be indexed. + #[error("invalid index.bin: {0}")] + InvalidIndexBin(#[from] InvalidIndexBinError), + /// The length of a tile differs from the size that the provider reported, /// for example because a tile file changed during the write. #[error( - "the source produced {actual} bytes for tile {graph_id}, but it reported a size of {expected} bytes" + "the provider gave {actual} bytes for tile {graph_id}, but it reported a size of {expected} bytes" )] SizeMismatch { - /// The base graph ID of the offending tile. + /// The base graph ID of the tile. graph_id: GraphId, - /// The size that the source reported before the write. + /// The size that the provider reported before the write. expected: u32, - /// The length of the bytes actually produced by the source. + /// The length of the tile bytes from the provider. actual: usize, }, - /// The source failed to give the size or the bytes of a tile. - #[error("the source failed to produce tile {graph_id}: {source}")] - Source { + /// The provider failed to give the size of a tile, or to read or decode a tile. + #[error("the provider failed to give tile {graph_id}: {source}")] + Provider { /// The base graph ID of the tile. graph_id: GraphId, - /// The error from the source. - source: E, + /// The error from the provider. + source: GraphTileProviderError, }, /// Writing to the destination failed. #[error("I/O error: {0}")] Io(#[from] std::io::Error), } -/// Writes an extract with the tiles of `tile_ids` from `source` into `dest`, +impl ExtractWriteError { + /// Wraps an error from the provider for the tile `graph_id`. + fn from_provider(graph_id: GraphId, source: GraphTileProviderError) -> Self { + match source { + GraphTileProviderError::TileDoesNotExist => Self::MissingTile(graph_id), + source => Self::Provider { graph_id, source }, + } + } +} + +/// Writes an extract with the tiles that contain `tile_ids` from `provider` into `dest`, /// and returns the number of bytes written. /// -/// The writer copies each tile verbatim, and it does not seek or buffer the archive. -/// The order of `tile_ids` and their feature indexes do not change the output, -/// so the same set of tiles always gives the same bytes. +/// The writer copies each tile verbatim. +/// The tiles are written in the standard order, +/// so a given input set of tile IDs always produces the same output bytes. +/// Each tile is written once, even if `tile_ids` contains more than one graph ID in it. +/// +/// For a [`DirectoryGraphTileProvider`](crate::tile_provider::DirectoryGraphTileProvider), +/// the tile files must not change during the write, +/// and the cache of the provider must not hold older versions of the tiles. +/// +/// # Performance +/// +/// The writer makes three small writes for each tile, and it does not buffer them. +/// Wrap a `dest` without a buffer, such as a [`File`](std::fs::File), in a +/// [`BufWriter`](std::io::BufWriter). /// /// # Errors /// -/// Returns an error before any write if `tile_ids` is empty, -/// names a tile more than once, or names a tile that is not in `source`. -/// Returns an error during the write if `source` fails, -/// if the length of a tile differs from its reported size, or if `dest` fails. -/// After an error during the write, `dest` contains a partial archive. -pub fn write_extract( - source: &S, - tile_ids: &[GraphId], - dest: W, -) -> Result> +/// The writer finds these errors before it writes to `dest`: +/// +/// - [`ExtractWriteError::NoTiles`] if `tile_ids` is empty. +/// - [`ExtractWriteError::MissingTile`] if the provider does not have a tile. +/// - [`ExtractWriteError::Provider`] if the provider cannot get the size of a tile. +/// +/// The writer finds these errors during the write, so `dest` then contains a partial archive: +/// +/// - [`ExtractWriteError::MissingTile`] or [`ExtractWriteError::Provider`] +/// if the provider cannot read or decode a tile. +/// - [`ExtractWriteError::SizeMismatch`] if the length of a tile differs from its size. +/// - [`ExtractWriteError::Io`] if a write to `dest` fails. +pub fn write_extract( + provider: &P, + tile_ids: I, + mut dest: W, +) -> Result where - S: TileBytesSource, + P: GraphTileProvider, + I: IntoIterator, W: Write, { - let manifest = tile_ids - .iter() + let mut tiles: Vec = tile_ids + .into_iter() + .map(|graph_id| graph_id.tile_base_id()) + .collect(); + // `GraphId` orders higher levels first, so it cannot give the archive order. + tiles.sort_unstable_by_key(|graph_id| (graph_id.level(), graph_id.tile_id())); + tiles.dedup(); + if tiles.is_empty() { + return Err(ExtractWriteError::NoTiles); + } + + let manifest = tiles + .into_iter() .map(|graph_id| { - let base_id = graph_id.tile_base_id(); - match source.tile_size(base_id) { - Ok(Some(size)) => Ok((base_id, size)), - Ok(None) => Err(ExtractWriteError::MissingTile(base_id)), - Err(source) => Err(ExtractWriteError::Source { - graph_id: base_id, - source, - }), - } + provider + .tile_size(graph_id) + .map(|size| (graph_id, size)) + .map_err(|source| ExtractWriteError::from_provider(graph_id, source)) }) .collect::, _>>()?; - let layout = ExtractLayout::from_manifest(&manifest)?; - write_layout(&layout, source, dest) + + let index_bytes = serialize_index_bin(&layout.entries)?; + write_entry(&mut dest, INDEX_BIN_FILE_NAME, index_bytes)?; + + for entry in &layout.entries { + let graph_id = entry.graph_id()?; + let path = graph_id.relative_path(TILE_FILE_EXTENSION)?; + let tile = provider + .get_handle_for_tile_containing(graph_id) + .map_err(|source| ExtractWriteError::from_provider(graph_id, source))?; + + let tile_bytes = tile.tile_bytes(); + let expected = entry.size(); + if tile_bytes.len() != expected as usize { + return Err(ExtractWriteError::SizeMismatch { + graph_id, + expected, + actual: tile_bytes.len(), + }); + } + write_entry(&mut dest, &path, tile_bytes)?; + } + + // Write the end of archive marker + dest.write_all(&ZERO_BLOCK)?; + dest.write_all(&ZERO_BLOCK)?; + dest.flush()?; + + Ok(layout.total_len) } -/// The payload offset of each tile and the total length of an extract. +/// Where each tile goes in an extract. +/// +/// The writer must write `index.bin` before the tiles, +/// so it computes the offset of every tile from the tile sizes before it writes. #[derive(Debug)] struct ExtractLayout { - /// One index entry for each tile, in archive order. + /// The `index.bin` entry of each tile, in archive order. entries: Vec, - /// The total length of the archive in bytes, including the end-of-archive marker. + /// The length of the archive in bytes, with the end-of-archive marker. total_len: u64, } impl ExtractLayout { - /// Computes the layout for `(GraphId, size)` pairs in any order. + /// Computes the layout for `(GraphId, size)` pairs of base tile IDs in archive order. /// /// # Errors /// - /// Returns an error if `manifest` is empty, contains a tile more than once, - /// contains a tile with a size of zero, or contains an ID that is not a base tile ID. - fn from_manifest(manifest: &[(GraphId, u32)]) -> Result { - if manifest.is_empty() { - return Err(ExtractLayoutError::EmptyManifest); - } - - let mut sorted = manifest.to_vec(); - sorted.sort_unstable_by_key(|(graph_id, _)| (graph_id.level(), graph_id.tile_id())); - - let index_len = (INDEX_ENTRY_SIZE * sorted.len()) as u64; + /// Returns an error if `manifest` contains an ID that is not a base tile ID. + fn from_manifest(manifest: &[(GraphId, u32)]) -> Result { + let index_len = (INDEX_ENTRY_SIZE * manifest.len()) as u64; let mut next_entry_offset = entry_len(index_len); - let mut entries = Vec::with_capacity(sorted.len()); - for (i, &(graph_id, size)) in sorted.iter().enumerate() { - if i > 0 && sorted[i - 1].0 == graph_id { - return Err(ExtractLayoutError::DuplicateTile(graph_id)); - } - if size == 0 { - return Err(ExtractLayoutError::EmptyTile(graph_id)); - } - + let mut entries = Vec::with_capacity(manifest.len()); + for &(graph_id, size) in manifest { let payload_offset = next_entry_offset + TAR_BLOCK_SIZE; entries.push(TileIndexBinEntry::new(graph_id, payload_offset, size)?); next_entry_offset += entry_len(u64::from(size)); @@ -172,21 +203,9 @@ impl ExtractLayout { Ok(Self { entries, - total_len: next_entry_offset + END_OF_ARCHIVE_LEN, + total_len: next_entry_offset + END_OF_ARCHIVE_MARKER_LEN, }) } - - /// The index entries, in archive order. - #[inline] - fn index_entries(&self) -> &[TileIndexBinEntry] { - &self.entries - } - - /// The total length of the archive in bytes. - #[inline] - fn total_len(&self) -> u64 { - self.total_len - } } /// The length of a tar entry with a payload of `payload_len` bytes: @@ -195,66 +214,8 @@ const fn entry_len(payload_len: u64) -> u64 { TAR_BLOCK_SIZE + payload_len.next_multiple_of(TAR_BLOCK_SIZE) } -/// Streams the archive of a layout into the destination, -/// and returns the number of bytes written. -fn write_layout( - layout: &ExtractLayout, - source: &S, - dest: W, -) -> Result> -where - S: TileBytesSource, - W: Write, -{ - let mut dest = CountingWriter { - inner: dest, - position: 0, - }; - - let index_bytes = - serialize_index_bin(layout.index_entries()).map_err(ExtractLayoutError::from)?; - write_entry(&mut dest, INDEX_BIN_ENTRY_NAME, index_bytes)?; - - for entry in layout.index_entries() { - // The layout holds only valid base tile IDs, so this does not fail. - let graph_id = entry.graph_id().map_err(ExtractLayoutError::from)?; - let path = graph_id.relative_path(TILE_FILE_EXTENSION)?; - let expected = entry.size(); - - source - .with_tile_bytes(graph_id, |tile_bytes| { - if tile_bytes.len() != expected as usize { - return Err(ExtractWriteError::SizeMismatch { - graph_id, - expected, - actual: tile_bytes.len(), - }); - } - let payload_offset = write_entry(&mut dest, &path, tile_bytes)?; - debug_assert_eq!( - payload_offset, - entry.offset(), - "the payload of {graph_id} is not where index.bin says it is" - ); - Ok(()) - }) - .map_err(|source| ExtractWriteError::Source { graph_id, source })??; - } - - dest.write_all(&ZERO_BLOCK)?; - dest.write_all(&ZERO_BLOCK)?; - dest.flush()?; - - debug_assert_eq!(dest.position, layout.total_len()); - Ok(dest.position) -} - -/// Writes a tar entry for `payload` at `path`, and returns the offset of the payload. -fn write_entry( - dest: &mut CountingWriter, - path: &str, - payload: &[u8], -) -> std::io::Result { +/// Writes a tar entry for `payload` at `path`. +fn write_entry(dest: &mut W, path: &str, payload: &[u8]) -> std::io::Result<()> { // Keep all metadata fixed, so that the same tiles always give the same bytes. let mut header = Header::new_ustar(); header.set_entry_type(EntryType::Regular); @@ -267,126 +228,126 @@ fn write_entry( header.set_cksum(); dest.write_all(header.as_bytes())?; - let payload_offset = dest.position; dest.write_all(payload)?; let padding = payload.len().next_multiple_of(ZERO_BLOCK.len()) - payload.len(); - dest.write_all(&ZERO_BLOCK[..padding])?; - - Ok(payload_offset) -} - -/// A writer that counts the bytes written through it. -struct CountingWriter { - inner: W, - /// The offset of the next byte in the archive. - position: u64, -} - -impl Write for CountingWriter { - #[inline] - fn write(&mut self, buf: &[u8]) -> std::io::Result { - let written = self.inner.write(buf)?; - self.position += written as u64; - Ok(written) - } - - #[inline] - fn flush(&mut self) -> std::io::Result<()> { - self.inner.flush() - } + dest.write_all(&ZERO_BLOCK[..padding]) } #[cfg(test)] mod tests { use super::*; + use crate::graph_tile::GraphTileView; + use crate::graph_tile::handles::OwnedGraphTileHandle; use crate::test_fixtures; use crate::tile_provider::{DirectoryGraphTileProvider, TarballTileProvider, parse_index_bin}; - use proptest::collection::btree_map; + use geo::{CoordFloat, Point}; + use num_traits::FromPrimitive; + use proptest::collection::vec; + use proptest::prelude::ProptestConfig; + use proptest::sample::select; use proptest::{prop_assert, prop_assert_eq, proptest}; use std::collections::{HashMap, HashSet}; use std::io::Read; use std::num::NonZeroUsize; + use std::path::Path; + use std::sync::Arc; use tar::Archive; - fn tile(level: u8, tile_id: u64) -> GraphId { + /// The tiles of the Andorra fixtures, in archive order. + const ANDORRA_TILES: [(u8, u64); 7] = [ + (0, 3015), + (1, 47_701), + (2, 762_485), + (2, 762_486), + (2, 763_925), + (2, 763_926), + (2, 763_927), + ]; + + /// The smallest Andorra tile, which keeps the tests that run under Miri fast. + const SMALL_TILE: (u8, u64) = (2, 763_927); + + fn tile((level, tile_id): (u8, u64)) -> GraphId { GraphId::try_from_components(level, tile_id, 0).expect("valid tile ID in test") } - #[derive(Default)] - struct MemorySource { - tiles: HashMap>, + fn andorra_tiles() -> Vec { + ANDORRA_TILES.into_iter().map(tile).collect() + } + + fn directory_provider(base: &Path) -> DirectoryGraphTileProvider { + DirectoryGraphTileProvider::new(base.to_path_buf(), NonZeroUsize::MIN) + } + + /// The Andorra tile directory, with sizes and read errors that a test can change. + struct TestProvider { + inner: DirectoryGraphTileProvider, /// Sizes which `tile_size` reports in place of the real sizes. reported_sizes: HashMap, - /// Tiles which the source has, but whose bytes it fails to produce. - unreadable: HashSet, + /// Tiles whose size the provider fails to get. + size_failures: HashSet, + /// Tiles which the provider has, but fails to read. + read_failures: HashSet, } - /// The error of a [`MemorySource`] when it cannot produce a tile. - #[derive(Debug, Error, PartialEq, Eq)] - #[error("the tile is not readable")] - struct Unreadable; - - impl MemorySource { - fn from_manifest(manifest: &[(GraphId, u32)]) -> Self { - let tiles = manifest - .iter() - .map(|(graph_id, size)| { - // A nonzero fill for each tile, so that tiles and padding differ. - let fill = - 0x11 * (1 + u8::try_from(graph_id.tile_id() % 13).expect("13 fits in u8")); - (*graph_id, vec![fill; *size as usize]) - }) - .collect(); + impl TestProvider { + fn new() -> Self { Self { - tiles, - ..Self::default() + inner: directory_provider(&test_fixtures::andorra_tiles_dir()), + reported_sizes: HashMap::new(), + size_failures: HashSet::new(), + read_failures: HashSet::new(), } } + } - fn tile_ids(&self) -> Vec { - self.tiles.keys().copied().collect() - } + fn test_failure() -> GraphTileProviderError { + GraphTileProviderError::TileFetchError("a failure from the test provider".to_string()) } - impl TileBytesSource for MemorySource { - type Error = Unreadable; + impl GraphTileProvider for TestProvider { + type TileHandle = Arc; - fn tile_size(&self, graph_id: GraphId) -> Result, Self::Error> { - let graph_id = graph_id.tile_base_id(); - if let Some(size) = self.reported_sizes.get(&graph_id) { - return Ok(Some(*size)); + fn with_tile_containing( + &self, + graph_id: GraphId, + process: F, + ) -> Result + where + F: FnOnce(&GraphTileView) -> T, + { + self.inner.with_tile_containing(graph_id, process) + } + + fn get_handle_for_tile_containing( + &self, + graph_id: GraphId, + ) -> Result { + if self.read_failures.contains(&graph_id.tile_base_id()) { + return Err(test_failure()); } - Ok(self - .tiles - .get(&graph_id) - .map(|tile_bytes| u32::try_from(tile_bytes.len()).expect("test tiles are small"))) + self.inner.get_handle_for_tile_containing(graph_id) } - fn with_tile_bytes(&self, graph_id: GraphId, process: F) -> Result - where - F: FnOnce(&[u8]) -> T, - { - let graph_id = graph_id.tile_base_id(); - if self.unreadable.contains(&graph_id) { - return Err(Unreadable); + fn tile_size(&self, graph_id: GraphId) -> Result { + let base_graph_id = graph_id.tile_base_id(); + if self.size_failures.contains(&base_graph_id) { + return Err(test_failure()); } - self.tiles - .get(&graph_id) - .map(|tile_bytes| process(tile_bytes)) - .ok_or(Unreadable) + if let Some(size) = self.reported_sizes.get(&base_graph_id) { + return Ok(*size); + } + self.inner.tile_size(graph_id) } - } - /// Tiles out of archive order, with sizes at, above, and below a block boundary. - fn scrambled_manifest() -> Vec<(GraphId, u32)> { - vec![ - (tile(2, 762_485), 100), - (tile(0, 3015), 512), - (tile(2, 762_484), 1024), - (tile(1, 47_701), 513), - (tile(1, 47_700), 511), - ] + fn tiles_within_radius( + &self, + center: Point, + radius: N, + ) -> Vec { + self.inner.tiles_within_radius(center, radius) + } } /// Returns the path and payload offset of each entry, as the `tar` crate reads them. @@ -419,58 +380,110 @@ mod tests { index_bytes } + /// Checks an archive with the `tar` crate as an independent reader, + /// and checks each tile against the tile file in `tile_dir`. + fn check_archive(archive_bytes: &[u8], expected_tiles: &[GraphId], tile_dir: &Path) { + assert!( + archive_bytes.len().is_multiple_of(ZERO_BLOCK.len()), + "The archive must be a whole number of tar blocks" + ); + let marker_start = archive_bytes.len() - ZERO_BLOCK.len() * 2; + assert!( + archive_bytes[marker_start..].iter().all(|byte| *byte == 0), + "The archive must end with two zero blocks" + ); + + let entries = read_entries(archive_bytes); + let mut expected_paths = vec![INDEX_BIN_FILE_NAME.to_string()]; + expected_paths.extend(expected_tiles.iter().map(|graph_id| { + graph_id + .relative_path(TILE_FILE_EXTENSION) + .expect("valid tile") + })); + let entry_paths: Vec<&String> = entries.iter().map(|(path, _)| path).collect(); + assert_eq!( + entry_paths, + expected_paths.iter().collect::>(), + "The archive must contain index.bin, then the tiles in archive order" + ); + + let index_bytes = read_index_bytes(archive_bytes); + let index_entries = parse_index_bin(&index_bytes).expect("the index parses"); + assert_eq!( + index_entries.len(), + expected_tiles.len(), + "The index must have one entry for each tile" + ); + for (index_entry, (path, payload_offset)) in index_entries.iter().zip(&entries[1..]) { + assert_eq!( + index_entry.offset(), + *payload_offset, + "The index offset of {path} must be where the tar reader finds its payload" + ); + + let start = usize::try_from(index_entry.offset()).expect("the offset fits in memory"); + let payload = &archive_bytes[start..start + index_entry.size() as usize]; + let tile_file = std::fs::read(tile_dir.join(path)).expect("the tile file is readable"); + assert!( + payload == tile_file.as_slice(), + "The payload of {path} must be the bytes of its tile file" + ); + } + } + #[test] fn test_layout_offsets() { const B: u64 = TAR_BLOCK_SIZE; - let layout = ExtractLayout::from_manifest(&scrambled_manifest()) - .expect("the manifest is well-formed"); + // Sizes at, below, and above a block boundary, in archive order. + let manifest = [ + (tile((0, 3015)), 512), + (tile((1, 47_700)), 511), + (tile((1, 47_701)), 513), + (tile((2, 762_484)), 1024), + (tile((2, 762_485)), 100), + ]; + let layout = ExtractLayout::from_manifest(&manifest).expect("the manifest is well-formed"); // The index entry (a header and 5 × 16 bytes) fills blocks 0 and 1. // Each payload starts one block after its header. - let expected = [ - // (graph_id, payload offset, size) - (tile(0, 3015), 3 * B, 512), // header at block 2; payload is 1 block - (tile(1, 47_700), 5 * B, 511), // header at block 4; payload is padded to 1 block - (tile(1, 47_701), 7 * B, 513), // header at block 6; payload is padded to 2 blocks - (tile(2, 762_484), 10 * B, 1024), // header at block 9; payload is 2 blocks - (tile(2, 762_485), 13 * B, 100), // header at block 12; payload is padded to 1 block + let expected_offsets = [ + 3 * B, // header at block 2; payload is 1 block + 5 * B, // header at block 4; payload is padded to 1 block + 7 * B, // header at block 6; payload is padded to 2 blocks + 10 * B, // header at block 9; payload is 2 blocks + 13 * B, // header at block 12; payload is padded to 1 block ]; - assert_eq!(layout.index_entries().len(), expected.len()); - for (entry, (graph_id, offset, size)) in layout.index_entries().iter().zip(expected) { - assert_eq!(entry.graph_id().unwrap(), graph_id); - assert_eq!(entry.offset(), offset); - assert_eq!(entry.size(), size); - } - - // The last payload ends at block 14, and the end-of-archive marker is 2 blocks. - assert_eq!(layout.total_len(), 16 * B); - } - - #[test] - fn test_layout_rejects_empty_manifest() { assert_eq!( - ExtractLayout::from_manifest(&[]).unwrap_err(), - ExtractLayoutError::EmptyManifest + layout.entries.len(), + manifest.len(), + "The layout must have one entry for each tile" ); - } - - #[test] - fn test_layout_rejects_duplicate_tiles() { - let manifest = [(tile(1, 47_701), 100), (tile(1, 47_701), 100)]; - assert_eq!( - ExtractLayout::from_manifest(&manifest).unwrap_err(), - ExtractLayoutError::DuplicateTile(tile(1, 47_701)) - ); - } + for ((entry, (graph_id, size)), offset) in + layout.entries.iter().zip(manifest).zip(expected_offsets) + { + assert_eq!( + entry.graph_id(), + Ok(graph_id), + "The layout must keep the order of the manifest" + ); + assert_eq!( + entry.offset(), + offset, + "The payload of {graph_id} must start one block after its header" + ); + assert_eq!( + entry.size(), + size, + "The index entry of {graph_id} must have its size" + ); + } - #[test] - fn test_layout_rejects_empty_tiles() { - let manifest = [(tile(0, 3015), 512), (tile(1, 47_701), 0)]; assert_eq!( - ExtractLayout::from_manifest(&manifest).unwrap_err(), - ExtractLayoutError::EmptyTile(tile(1, 47_701)) + layout.total_len, + 16 * B, + "The last payload ends at block 14, then the end-of-archive marker is 2 blocks" ); } @@ -480,159 +493,230 @@ mod tests { GraphId::try_from_components(1, 47_701, 7).expect("valid graph ID in test"); assert_eq!( ExtractLayout::from_manifest(&[(with_feature_index, 100)]).unwrap_err(), - ExtractLayoutError::InvalidIndexBin(InvalidIndexBinError::FeatureIndex( - with_feature_index - )) + InvalidIndexBinError::FeatureIndex(with_feature_index), + "Only base tile IDs can be indexed" ); } #[test] fn test_write_rejects_empty_tile_list() { - let source = MemorySource::default(); - assert!(matches!( - write_extract(&source, &[], Vec::new()), - Err(ExtractWriteError::Layout(ExtractLayoutError::EmptyManifest)) - )); + let mut dest = Vec::new(); + assert!( + matches!( + write_extract(&TestProvider::new(), [], &mut dest), + Err(ExtractWriteError::NoTiles) + ), + "An extract must contain at least one tile" + ); + assert!( + dest.is_empty(), + "An empty tile list is found before any write" + ); } #[test] - fn test_write_rejects_the_same_tile_twice() { - let source = MemorySource::from_manifest(&[(tile(1, 47_701), 100)]); - let with_feature_index = - GraphId::try_from_components(1, 47_701, 7).expect("valid graph ID in test"); - assert!(matches!( - write_extract(&source, &[tile(1, 47_701), with_feature_index], Vec::new()), - Err(ExtractWriteError::Layout(ExtractLayoutError::DuplicateTile(graph_id))) - if graph_id == tile(1, 47_701) - )); + fn test_write_rejects_missing_tiles() { + let missing = tile((2, 0)); + let mut dest = Vec::new(); + assert!( + matches!( + write_extract(&TestProvider::new(), [tile(SMALL_TILE), missing], &mut dest), + Err(ExtractWriteError::MissingTile(graph_id)) if graph_id == missing + ), + "A tile that is not in the provider must be a MissingTile error" + ); + assert!(dest.is_empty(), "A missing tile is found before any write"); } #[test] - fn test_write_rejects_missing_tiles() { - let source = MemorySource::from_manifest(&[(tile(0, 3015), 100)]); + fn test_write_reports_size_errors_before_write() { + let mut provider = TestProvider::new(); + provider.size_failures.insert(tile(SMALL_TILE)); + let mut dest = Vec::new(); - assert!(matches!( - write_extract(&source, &[tile(0, 3015), tile(2, 0)], &mut dest), - Err(ExtractWriteError::MissingTile(graph_id)) if graph_id == tile(2, 0) - )); - assert!(dest.is_empty(), "a missing tile is found before any write"); + assert!( + matches!( + write_extract(&provider, [tile(SMALL_TILE)], &mut dest), + Err(ExtractWriteError::Provider { + graph_id, + source: GraphTileProviderError::TileFetchError(_), + }) if graph_id == tile(SMALL_TILE) + ), + "A failure to get a tile size must be a Provider error" + ); + assert!( + dest.is_empty(), + "A tile size error is found before any write" + ); + } + + #[test] + fn test_write_reports_read_errors() { + let mut provider = TestProvider::new(); + provider.read_failures.insert(tile(SMALL_TILE)); + + assert!( + matches!( + write_extract(&provider, [tile(SMALL_TILE)], Vec::new()), + Err(ExtractWriteError::Provider { + graph_id, + source: GraphTileProviderError::TileFetchError(_), + }) if graph_id == tile(SMALL_TILE) + ), + "A failure to read a tile must be a Provider error" + ); } #[test] fn test_write_rejects_size_mismatch() { - let mut source = MemorySource::from_manifest(&[(tile(0, 3015), 99)]); - source.reported_sizes.insert(tile(0, 3015), 100); + let mut provider = TestProvider::new(); + let real_size = provider + .tile_size(tile(SMALL_TILE)) + .expect("the tile has a size"); + provider + .reported_sizes + .insert(tile(SMALL_TILE), real_size + 1); + + assert!( + matches!( + write_extract(&provider, [tile(SMALL_TILE)], Vec::new()), + Err(ExtractWriteError::SizeMismatch { + graph_id, + expected, + actual, + }) if graph_id == tile(SMALL_TILE) + && expected == real_size + 1 + && actual == real_size as usize + ), + "Tile bytes of a different length than the reported size must be an error" + ); + } - assert!(matches!( - write_extract(&source, &[tile(0, 3015)], Vec::new()), - Err(ExtractWriteError::SizeMismatch { - graph_id, - expected: 100, - actual: 99, - }) if graph_id == tile(0, 3015) - )); + /// Copies the small Andorra tile into a new tile directory. + fn copy_small_tile() -> (tempfile::TempDir, std::path::PathBuf) { + let base = tempfile::tempdir().expect("Unable to create a temporary directory"); + let relative_path = tile(SMALL_TILE) + .file_path(TILE_FILE_EXTENSION) + .expect("valid tile"); + let tile_path = base.path().join(&relative_path); + std::fs::create_dir_all(tile_path.parent().expect("the tile path has a parent")) + .expect("Unable to create the tile directory"); + std::fs::copy( + test_fixtures::andorra_tiles_dir().join(&relative_path), + &tile_path, + ) + .expect("Unable to copy the tile"); + (base, tile_path) } #[test] - fn test_write_reports_source_errors() { - let mut source = MemorySource::from_manifest(&[(tile(0, 3015), 100)]); - source.unreadable.insert(tile(0, 3015)); - - assert!(matches!( - write_extract(&source, &[tile(0, 3015)], Vec::new()), - Err(ExtractWriteError::Source { - graph_id, - source: Unreadable - }) if graph_id == tile(0, 3015) - )); + fn test_write_rejects_tile_changed_after_cache() { + let (base, tile_path) = copy_small_tile(); + let provider = directory_provider(base.path()); + let cached = provider + .get_handle_for_tile_containing(tile(SMALL_TILE)) + .expect("Unable to get tile"); + let cached_len = cached.tile_bytes().len(); + + // The file grows, but the cache of the provider keeps the old bytes. + let mut file = std::fs::OpenOptions::new() + .append(true) + .open(&tile_path) + .expect("Unable to open the tile file"); + file.write_all(&[0]).expect("Unable to grow the tile file"); + + assert!( + matches!( + write_extract(&provider, [tile(SMALL_TILE)], Vec::new()), + Err(ExtractWriteError::SizeMismatch { + expected, + actual, + .. + }) if expected as usize == cached_len + 1 && actual == cached_len + ), + "A tile file that changed after the provider cached it must be a SizeMismatch" + ); } #[test] - fn test_archive_invariants() { - let manifest = scrambled_manifest(); - let source = MemorySource::from_manifest(&manifest); + fn test_write_rejects_empty_tile_file() { + let (base, tile_path) = copy_small_tile(); + std::fs::File::create(&tile_path).expect("Unable to empty the tile file"); + + assert!( + matches!( + write_extract(&directory_provider(base.path()), [tile(SMALL_TILE)], Vec::new()), + Err(ExtractWriteError::Provider { + graph_id, + source: GraphTileProviderError::DecodingError(_), + }) if graph_id == tile(SMALL_TILE) + ), + "An empty tile file must fail to decode" + ); + } + #[test] + fn test_write_small_tile() { + let provider = TestProvider::new(); let mut archive_bytes = Vec::new(); - let written = write_extract(&source, &source.tile_ids(), &mut archive_bytes) - .expect("the source is well-formed"); - - assert_eq!(written, archive_bytes.len() as u64); - assert!(archive_bytes.len().is_multiple_of(ZERO_BLOCK.len())); + let written = write_extract(&provider, [tile(SMALL_TILE)], &mut archive_bytes) + .expect("the extract writes cleanly"); - let entries = read_entries(&archive_bytes); - let entry_paths: Vec<&str> = entries.iter().map(|(path, _)| path.as_str()).collect(); assert_eq!( - entry_paths, - vec![ - "index.bin", - "0/003/015.gph", - "1/047/700.gph", - "1/047/701.gph", - "2/000/762/484.gph", - "2/000/762/485.gph", - ] + written, + archive_bytes.len() as u64, + "The return value must be the number of bytes written" + ); + check_archive( + &archive_bytes, + &[tile(SMALL_TILE)], + &test_fixtures::andorra_tiles_dir(), ); - - let index_bytes = read_index_bytes(&archive_bytes); - let index_entries = parse_index_bin(&index_bytes).expect("the index parses"); - assert_eq!(index_entries.len(), manifest.len()); - - for (index_entry, (_, payload_offset)) in index_entries.iter().zip(&entries[1..]) { - assert_eq!(index_entry.offset(), *payload_offset); - - let graph_id = index_entry.graph_id().expect("the entry ID is valid"); - let start = usize::try_from(index_entry.offset()).expect("the offset fits in memory"); - let payload = &archive_bytes[start..start + index_entry.size() as usize]; - source - .with_tile_bytes(graph_id, |tile_bytes| assert_eq!(payload, tile_bytes)) - .expect("the tile is in the source"); - } } + // Miri is too slow to decode all of the Andorra tiles. + #[cfg(not(miri))] #[test] - fn test_determinism() { - let source = MemorySource::from_manifest(&scrambled_manifest()); - let tile_ids = source.tile_ids(); - - let mut first = Vec::new(); - write_extract(&source, &tile_ids, &mut first).expect("first write succeeds"); - - let mut reversed = tile_ids; - reversed.reverse(); - let mut second = Vec::new(); - write_extract(&source, &reversed, &mut second).expect("second write succeeds"); + fn test_write_all_andorra_tiles() { + let provider = TestProvider::new(); + let mut archive_bytes = Vec::new(); + let written = write_extract(&provider, andorra_tiles(), &mut archive_bytes) + .expect("the extract writes cleanly"); - assert_eq!(first, second); + assert_eq!( + written, + archive_bytes.len() as u64, + "The return value must be the number of bytes written" + ); + check_archive( + &archive_bytes, + &andorra_tiles(), + &test_fixtures::andorra_tiles_dir(), + ); } - proptest! { - #[test] - fn test_archive_matches_independent_reader( - // Unique tiles on levels 0 to 2, with sizes of one to just over four blocks. - tiles in btree_map((0u8..=2, 0u64..4050), 1u32..=2049, 1..32) - ) { - let manifest: Vec<(GraphId, u32)> = tiles - .into_iter() - .map(|((level, tile_id), size)| (tile(level, tile_id), size)) - .collect(); - let source = MemorySource::from_manifest(&manifest); - - let mut archive_bytes = Vec::new(); - let written = write_extract(&source, &source.tile_ids(), &mut archive_bytes) - .expect("the source is well-formed"); - let layout = ExtractLayout::from_manifest(&manifest).expect("the manifest is well-formed"); - prop_assert_eq!(written, archive_bytes.len() as u64); - prop_assert_eq!(written, layout.total_len()); - - let entries = read_entries(&archive_bytes); - prop_assert_eq!(entries.len(), manifest.len() + 1); - for (index_entry, (path, payload_offset)) in layout.index_entries().iter().zip(&entries[1..]) { - let graph_id = index_entry.graph_id().expect("the entry ID is valid"); - prop_assert_eq!(index_entry.offset(), *payload_offset); - prop_assert_eq!(path, &graph_id.relative_path(TILE_FILE_EXTENSION).expect("valid tile")); - prop_assert!(index_entry.offset().is_multiple_of(TAR_BLOCK_SIZE)); - } - } + // Miri is too slow to decode all of the Andorra tiles. + #[cfg(not(miri))] + #[test] + fn test_order_and_duplicates_do_not_change_output() { + let provider = TestProvider::new(); + let mut in_order = Vec::new(); + write_extract(&provider, andorra_tiles(), &mut in_order).expect("first write succeeds"); + + // Reverse order, with every tile twice: once as a base ID and once with a feature index. + let scrambled = andorra_tiles().into_iter().rev().flat_map(|graph_id| { + let with_feature_index = graph_id + .with_feature_index(42) + .expect("valid feature index"); + [graph_id, with_feature_index] + }); + let mut from_scrambled = Vec::new(); + write_extract(&provider, scrambled, &mut from_scrambled).expect("second write succeeds"); + + assert!( + in_order == from_scrambled, + "The order of the tile IDs, duplicates, and feature indexes must not change the output" + ); } // Tragically, memory maps are not supported by Miri! @@ -644,77 +728,100 @@ mod tests { // Out of archive order, and with one nonzero feature index. let tile_ids = [ - tile(2, 763_926), - tile(0, 3015), + tile((2, 763_926)), + tile((0, 3015)), GraphId::try_from_components(1, 47_701, 42).expect("valid graph ID in test"), - tile(2, 762_485), + tile((2, 762_485)), ]; - let excluded = tile(2, 763_925); - assert!(source_provider.tile_size(excluded).unwrap().is_some()); + let excluded = tile((2, 763_925)); + assert!( + source_provider.tile_size(excluded).is_ok(), + "The excluded tile must be in the source" + ); let mut archive_bytes = Vec::new(); - let written = write_extract(&source_provider, &tile_ids, &mut archive_bytes) + write_extract(&source_provider, tile_ids, &mut archive_bytes) .expect("the extract writes cleanly"); - assert_eq!(written, archive_bytes.len() as u64); - let mut second = Vec::new(); - write_extract(&source_provider, &tile_ids, &mut second).expect("second write succeeds"); - assert_eq!(archive_bytes, second); - - let mut extract_file = tempfile::NamedTempFile::new().expect("a temporary file is created"); - extract_file + let mut archive_file = tempfile::NamedTempFile::new().expect("a temporary file is created"); + archive_file .write_all(&archive_bytes) .expect("the extract is written to disk"); - - let extract_provider = TarballTileProvider::new_readonly(extract_file.path()) + let extract_provider = TarballTileProvider::new_readonly(archive_file.path()) .expect("the generated extract opens cleanly"); for graph_id in tile_ids { - extract_provider - .with_tile_bytes(graph_id, |extract_bytes| { - source_provider - .with_tile_bytes(graph_id, |source_bytes| { - assert_eq!(extract_bytes, source_bytes); - }) - .expect("the tile is in the source"); - }) + let extract_tile = extract_provider + .get_handle_for_tile_containing(graph_id) .expect("the tile is in the extract"); + let source_tile = source_provider + .get_handle_for_tile_containing(graph_id) + .expect("the tile is in the source"); + assert!( + extract_tile.tile_bytes() == source_tile.tile_bytes(), + "Tile {graph_id} must have the same bytes in the extract and in the source" + ); } - - assert!(extract_provider.tile_size(excluded).unwrap().is_none()); + assert!( + matches!( + extract_provider.tile_size(excluded), + Err(GraphTileProviderError::TileDoesNotExist) + ), + "A tile that was not requested must not be in the extract" + ); } // Tragically, memory maps are not supported by Miri! #[cfg(not(miri))] #[test] - fn test_directory_and_tarball_sources_write_the_same_extract() { + fn test_directory_and_tarball_providers_write_the_same_extract() { // The two fixtures contain the same tiles. let tarball = TarballTileProvider::new_readonly(test_fixtures::andorra_tiles_tar()) .expect("Unable to init the tarball tile provider"); - let directory = DirectoryGraphTileProvider::new( - test_fixtures::andorra_tiles_dir(), - NonZeroUsize::new(1).unwrap(), - ); - - let tile_ids: Vec = tarball.tile_ids().copied().collect(); - assert!(tile_ids.len() > 1); + let directory = directory_provider(&test_fixtures::andorra_tiles_dir()); let mut from_tarball = Vec::new(); - write_extract(&tarball, &tile_ids, &mut from_tarball) + write_extract(&tarball, andorra_tiles(), &mut from_tarball) .expect("the extract writes cleanly from the tarball"); let mut from_directory = Vec::new(); - write_extract(&directory, &tile_ids, &mut from_directory) + write_extract(&directory, andorra_tiles(), &mut from_directory) .expect("the extract writes cleanly from the directory"); - assert_eq!(from_tarball, from_directory); - - for result in [ - write_extract(&tarball, &[tile(0, 3015), tile(2, 0)], Vec::new()), - write_extract(&directory, &[tile(0, 3015), tile(2, 0)], Vec::new()), - ] { - assert!(matches!( - result, - Err(ExtractWriteError::MissingTile(graph_id)) if graph_id == tile(2, 0) - )); + assert!( + from_tarball == from_directory, + "The tarball and the directory of the same tiles must give the same extract" + ); + } + + // Miri is too slow to decode the Andorra tiles in many cases. + #[cfg(not(miri))] + proptest! { + #![proptest_config(ProptestConfig::with_cases(32))] + + #[test] + fn test_archive_matches_independent_reader( + // Andorra tiles in any order, with duplicates and feature indexes. + tile_ids in vec((select(&ANDORRA_TILES[..]), 0u64..4), 1..12) + ) { + let tile_ids: Vec = tile_ids + .into_iter() + .map(|(tile_key, feature_index)| { + tile(tile_key) + .with_feature_index(feature_index) + .expect("valid feature index") + }) + .collect(); + let mut expected_tiles: Vec = + tile_ids.iter().map(GraphId::tile_base_id).collect(); + expected_tiles.sort_unstable_by_key(|graph_id| (graph_id.level(), graph_id.tile_id())); + expected_tiles.dedup(); + + let mut archive_bytes = Vec::new(); + let written = write_extract(&TestProvider::new(), tile_ids, &mut archive_bytes) + .expect("the extract writes cleanly"); + + prop_assert_eq!(written, archive_bytes.len() as u64); + prop_assert!(!expected_tiles.is_empty()); + check_archive(&archive_bytes, &expected_tiles, &test_fixtures::andorra_tiles_dir()); } } } diff --git a/valhalla-graphtile/src/tile_provider/directory.rs b/valhalla-graphtile/src/tile_provider/directory.rs index d2476d1..1e8c9a5 100644 --- a/valhalla-graphtile/src/tile_provider/directory.rs +++ b/valhalla-graphtile/src/tile_provider/directory.rs @@ -5,7 +5,7 @@ use crate::graph_tile::{GraphTileView, handles::OwnedGraphTileHandle}; use crate::spatial::bbox_with_center; use crate::tile_hierarchy::STANDARD_LEVELS; use crate::tile_provider::{ - GraphTileProvider, GraphTileProviderError, LockTable, TILE_FILE_EXTENSION, TileBytesSource, + GraphTileProvider, GraphTileProviderError, LockTable, TILE_FILE_EXTENSION, }; use geo::{CoordFloat, Point}; use lru::LruCache; @@ -130,53 +130,6 @@ impl DirectoryGraphTileProvider { .base_directory .join(graph_id.file_path(TILE_FILE_EXTENSION)?)) } - - /// Reads the tile file for `base_graph_id`. - /// - /// The caller must hold the lock for `base_graph_id`. - fn read_tile_bytes(&self, base_graph_id: GraphId) -> Result, GraphTileProviderError> { - let path = self.path_for_graph_id(base_graph_id)?; - // NOTE: Does not handle compressed tiles - std::fs::read(path).map_err(|e| match e.kind() { - ErrorKind::NotFound => GraphTileProviderError::TileDoesNotExist, - _ => GraphTileProviderError::IoError(e), - }) - } -} - -impl TileBytesSource for DirectoryGraphTileProvider { - type Error = GraphTileProviderError; - - /// The size comes from the file system metadata, so this reads no tile data. - fn tile_size(&self, graph_id: GraphId) -> Result, Self::Error> { - let base_graph_id = graph_id.tile_base_id(); - let size = match std::fs::metadata(self.path_for_graph_id(base_graph_id)?) { - Ok(metadata) if metadata.is_file() => metadata.len(), - Ok(_) => return Ok(None), - Err(e) if e.kind() == ErrorKind::NotFound => return Ok(None), - Err(e) => return Err(e.into()), - }; - - u32::try_from(size) - .map(Some) - .map_err(|_| GraphTileProviderError::TileTooLarge { - graph_id: base_graph_id, - size, - }) - } - - fn with_tile_bytes(&self, graph_id: GraphId, process: F) -> Result - where - F: FnOnce(&[u8]) -> T, - { - let base_graph_id = graph_id.tile_base_id(); - let lock = self.lock_table.lock_for(base_graph_id); - let _guard = lock.lock(); - - // Bypass the tile cache, so that a copy of many tiles does not evict hot tiles. - let tile_bytes = self.read_tile_bytes(base_graph_id)?; - Ok(process(&tile_bytes)) - } } impl GraphTileProvider for DirectoryGraphTileProvider { @@ -211,7 +164,13 @@ impl GraphTileProvider for DirectoryGraphTileProvider { .map_err(|e| GraphTileProviderError::PoisonedCacheLock(e.to_string()))?; let tile = cache .try_get_or_insert(base_graph_id, || { - let data = self.read_tile_bytes(base_graph_id)?; + let path = self.path_for_graph_id(base_graph_id)?; + // Open the file and read all bytes into a buffer + // NOTE: Does not handle compressed tiles + let data = std::fs::read(path).map_err(|e| match e.kind() { + ErrorKind::NotFound => GraphTileProviderError::TileDoesNotExist, + _ => GraphTileProviderError::IoError(e), + })?; let tile = OwnedGraphTileHandle::try_from(data)?; Ok::<_, GraphTileProviderError>(Arc::new(tile)) }) @@ -272,7 +231,7 @@ mod test { use crate::graph_tile::handles::GraphTileHandle; use crate::test_fixtures; use crate::tile_hierarchy::STANDARD_LEVELS; - use crate::tile_provider::{GraphTileProvider, GraphTileProviderError}; + use crate::tile_provider::{GraphTileProvider, GraphTileProviderError, TILE_FILE_EXTENSION}; use core::num::NonZeroUsize; use rand::{ distr::{Distribution, Uniform}, @@ -542,7 +501,7 @@ mod test { let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); let tile_path = base.path().join( graph_id - .file_path("gph") + .file_path(TILE_FILE_EXTENSION) .expect("Unable to get the tile path"), ); std::fs::create_dir_all(tile_path).expect("Unable to create a directory at the tile path"); @@ -566,7 +525,7 @@ mod test { let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); let tile_path = base.path().join( graph_id - .file_path("gph") + .file_path(TILE_FILE_EXTENSION) .expect("Unable to get the tile path"), ); let parent = tile_path.parent().expect("The tile path has a parent"); diff --git a/valhalla-graphtile/src/tile_provider/mod.rs b/valhalla-graphtile/src/tile_provider/mod.rs index 8ea3b68..bfdf5d9 100644 --- a/valhalla-graphtile/src/tile_provider/mod.rs +++ b/valhalla-graphtile/src/tile_provider/mod.rs @@ -27,7 +27,7 @@ use crate::graph_tile::{ pub use directory::DirectoryGraphTileProvider; pub use iterators::GraphSearchResult; use iterators::ResultsWithinRadius; -pub(crate) use tarball::{INDEX_BIN_ENTRY_NAME, INDEX_ENTRY_SIZE, TAR_BLOCK_SIZE}; +pub(crate) use tarball::{INDEX_BIN_FILE_NAME, INDEX_ENTRY_SIZE, TAR_BLOCK_SIZE}; /// The extension of graph tile files, in tile directories and in extract tarballs. pub(crate) const TILE_FILE_EXTENSION: &str = "gph"; @@ -63,30 +63,6 @@ pub enum GraphTileProviderError { TileTooLarge { graph_id: GraphId, size: u64 }, } -/// Access to the raw bytes of graph tiles, for code that copies tiles verbatim. -pub trait TileBytesSource { - /// The error type of this source. - type Error: core::error::Error + 'static; - - /// Returns the size in bytes of the tile that contains `graph_id`, - /// or [`None`] if the source does not contain the tile. - /// - /// # Errors - /// - /// Returns an error if the source cannot find out whether it contains the tile. - fn tile_size(&self, graph_id: GraphId) -> Result, Self::Error>; - - /// Calls `process` with the raw bytes of the tile that contains `graph_id`, - /// and returns its result. - /// - /// # Errors - /// - /// Returns an error if the source does not contain the tile, or if it cannot read the tile. - fn with_tile_bytes(&self, graph_id: GraphId, process: F) -> Result - where - F: FnOnce(&[u8]) -> T; -} - /// A generic trait for all graph tile providers. /// /// This provides a standard interface so users don't need to care about the details diff --git a/valhalla-graphtile/src/tile_provider/tarball.rs b/valhalla-graphtile/src/tile_provider/tarball.rs index 55163c0..3ec5b74 100644 --- a/valhalla-graphtile/src/tile_provider/tarball.rs +++ b/valhalla-graphtile/src/tile_provider/tarball.rs @@ -1,4 +1,4 @@ -use super::{GraphTileProvider, GraphTileProviderError, TileBytesSource}; +use super::{GraphTileProvider, GraphTileProviderError}; use crate::GraphId; use crate::graph_tile::handles::MmapGraphTileHandle; use crate::graph_tile::{ @@ -114,9 +114,9 @@ impl TarballTileProvider { }; let mut entry = entry?; - if entry.path()?.to_string_lossy() != INDEX_BIN_ENTRY_NAME { + if entry.path()?.to_string_lossy() != INDEX_BIN_FILE_NAME { return Err(GraphTileProviderError::InvalidTarball(format!( - "Expected {INDEX_BIN_ENTRY_NAME} at the start of the archive" + "Expected {INDEX_BIN_FILE_NAME} at the start of the archive" ))); } @@ -186,51 +186,6 @@ impl TarballTileProvider { pub fn tile_ids(&self) -> impl Iterator { self.tile_index.keys() } - - /// Gets a pointer for the tile containing this graph ID. - /// - /// # Errors - /// - /// If the tile is not part of the tarball archive, - /// this returns [`GraphTileProviderError::TileDoesNotExist`]. - #[inline] - fn tile_pointer(&self, graph_id: GraphId) -> Result { - let base_graph_id = graph_id.tile_base_id(); - let Some(offsets) = self.tile_index.get(&base_graph_id) else { - return Err(GraphTileProviderError::TileDoesNotExist); - }; - - Ok(MmapTilePointer { - mmap: self.mmap.clone(), - offsets: *offsets, - }) - } -} - -impl TileBytesSource for TarballTileProvider { - type Error = GraphTileProviderError; - - /// The size comes from the `index.bin` of the tarball, so this reads no tile data. - #[inline] - fn tile_size(&self, graph_id: GraphId) -> Result, Self::Error> { - Ok(self - .tile_index - .get(&graph_id.tile_base_id()) - .map(|tile_offset| tile_offset.size)) - } - - #[inline] - fn with_tile_bytes(&self, graph_id: GraphId, process: F) -> Result - where - F: FnOnce(&[u8]) -> T, - { - let tile_pointer = self.tile_pointer(graph_id)?; - // SAFETY: The provider checks at construction that every index range lies inside the memory map. - // We assume that the underlying memory map has not changed in a way that would render the tile invalid - // (see the safety discussion on TarballTileProvider). - let tile_bytes = unsafe { tile_pointer.as_tile_bytes() }; - Ok(process(tile_bytes)) - } } impl GraphTileProvider for TarballTileProvider { @@ -371,7 +326,15 @@ impl TarballTileProvider { &self, graph_id: GraphId, ) -> Result { - self.tile_pointer(graph_id) + let base_graph_id = graph_id.tile_base_id(); + let Some(offsets) = self.tile_index.get(&base_graph_id) else { + return Err(GraphTileProviderError::TileDoesNotExist); + }; + + Ok(MmapTilePointer { + mmap: self.mmap.clone(), + offsets: *offsets, + }) } } @@ -504,8 +467,8 @@ pub(crate) const INDEX_ENTRY_SIZE: usize = size_of::(); /// Each entry is a header block, then its data, padded with zeros to whole blocks. pub(crate) const TAR_BLOCK_SIZE: u64 = 512; -/// The name of the tile index entry, which is the first entry of an extract tarball. -pub(crate) const INDEX_BIN_ENTRY_NAME: &str = "index.bin"; +/// The file name of the tile index, which is the first entry of an extract tarball. +pub(crate) const INDEX_BIN_FILE_NAME: &str = "index.bin"; /// The reasons why an `index.bin` file or one of its entries is not valid. #[derive(Debug, Error, PartialEq, Eq)] @@ -1089,7 +1052,7 @@ mod test { ( "directory", directory_tile.tile_bytes(), - GraphTileProvider::tile_size(&directory_provider, *graph_id), + directory_provider.tile_size(*graph_id), ), ( "tarball", From 2d468acdfac151e61850efb7a523bb98816bca38 Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Wed, 21 Oct 2026 12:00:31 -0700 Subject: [PATCH 20/37] Add integration tests of offline extracts with C++ Valhalla The integration tests run with one command: uv run --project scripts/integration valinor-integration scripts/integration is a uv project (valinor-integration). It uses docker-py to start valhalla_service, so it needs only uv and Docker (and cargo, if VALINOR_CLI is not set). - `run` (the default) writes a sub-extract of the Andorra fixture with valinor-cli extract. It serves the full extract and the sub-extract at the same time on free ports, sends each request to both, and compares the responses. It makes sure that the sub-extract is a proper subset of the full extract, so the comparison can never be empty. It also makes sure that both logs have no [ERROR] lines. - The checks compare the route shape, the maneuvers, and the summary length and time of each /route response, and the edges of each /locate response. - `serve` serves both extracts on fixed ports (8002 and 8003) until Ctrl-C or SIGTERM, for requests by hand. - The requests are a Bruno collection in the OpenCollection YAML format (scripts/integration/bruno). The harness reads the same files, so a request that you try in Bruno is the request that CI sends. The harness refuses Bruno features that it does not run, such as scripts and assertions. - The tiles of the sub-extract are in sub-extract-tiles.txt: every fixture tile except the north-east corner (2/763927), which no request uses. - The image's own valhalla_build_config makes the Valhalla config. The harness sets mjolnir.shortcut_caching to false, because Valhalla asserts at start that all level 0 and level 1 tiles exist when it is true. - The Valhalla version comes from VALHALLA_VERSION, a readable tag. CI pins it to 3.8.2 in integration.yml. If it is not set, the harness uses `latest` and logs a warning. - The harness copies the extract and the config into the container in place of a bind mount, so a work directory outside the home directory (such as /tmp) works with colima and Docker Desktop. - If DOCKER_HOST is not set and the default socket fails, the harness uses the endpoint of the current Docker CLI context (for colima). - The harness has 37 unit tests (pytest) and a ruff configuration. CI runs both before the integration test. .github/workflows/integration.yml runs the harness on pull requests that change the extract code, the tile providers, valinor-cli, or the harness. Verified locally: PASS against 3.8.2 and against latest (3.9.1), with VALINOR_CLI set and with cargo. With 2/763926 removed from the tile list, the harness fails as expected (4 routes return HTTP 400, and /locate and the fifth route differ). In serve mode, both ports answer, SIGTERM removes both containers, and a second start removes the containers that a killed run left. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/integration.yml | 50 +++ scripts/integration/.gitignore | 3 + scripts/integration/README.md | 94 +++++ .../integration/bruno/environments/full.yml | 4 + .../integration/bruno/environments/sub.yml | 4 + .../bruno/locate/andorra-la-vella-center.yml | 28 ++ scripts/integration/bruno/locate/folder.yml | 8 + scripts/integration/bruno/opencollection.yml | 10 + .../andorra-la-vella-center-pedestrian.yml | 32 ++ .../route/andorra-la-vella-to-encamp-auto.yml | 32 ++ .../andorra-la-vella-to-la-massana-auto.yml | 32 ++ .../bruno/route/encamp-to-canillo-auto.yml | 32 ++ scripts/integration/bruno/route/folder.yml | 8 + .../route/ordino-to-la-massana-bicycle.yml | 32 ++ scripts/integration/pyproject.toml | 33 ++ .../src/valinor_integration/__init__.py | 1 + .../src/valinor_integration/__main__.py | 5 + .../src/valinor_integration/checks.py | 120 ++++++ .../src/valinor_integration/cli.py | 109 ++++++ .../src/valinor_integration/collection.py | 169 ++++++++ .../src/valinor_integration/errors.py | 5 + .../src/valinor_integration/extract.py | 97 +++++ .../src/valinor_integration/runner.py | 201 ++++++++++ .../src/valinor_integration/settings.py | 92 +++++ .../src/valinor_integration/valhalla.py | 253 ++++++++++++ scripts/integration/sub-extract-tiles.txt | 14 + scripts/integration/tests/test_checks.py | 71 ++++ scripts/integration/tests/test_cli.py | 36 ++ scripts/integration/tests/test_collection.py | 95 +++++ scripts/integration/tests/test_extract.py | 52 +++ scripts/integration/tests/test_settings.py | 56 +++ scripts/integration/uv.lock | 370 ++++++++++++++++++ 32 files changed, 2148 insertions(+) create mode 100644 .github/workflows/integration.yml create mode 100644 scripts/integration/.gitignore create mode 100644 scripts/integration/README.md create mode 100644 scripts/integration/bruno/environments/full.yml create mode 100644 scripts/integration/bruno/environments/sub.yml create mode 100644 scripts/integration/bruno/locate/andorra-la-vella-center.yml create mode 100644 scripts/integration/bruno/locate/folder.yml create mode 100644 scripts/integration/bruno/opencollection.yml create mode 100644 scripts/integration/bruno/route/andorra-la-vella-center-pedestrian.yml create mode 100644 scripts/integration/bruno/route/andorra-la-vella-to-encamp-auto.yml create mode 100644 scripts/integration/bruno/route/andorra-la-vella-to-la-massana-auto.yml create mode 100644 scripts/integration/bruno/route/encamp-to-canillo-auto.yml create mode 100644 scripts/integration/bruno/route/folder.yml create mode 100644 scripts/integration/bruno/route/ordino-to-la-massana-bicycle.yml create mode 100644 scripts/integration/pyproject.toml create mode 100644 scripts/integration/src/valinor_integration/__init__.py create mode 100644 scripts/integration/src/valinor_integration/__main__.py create mode 100644 scripts/integration/src/valinor_integration/checks.py create mode 100644 scripts/integration/src/valinor_integration/cli.py create mode 100644 scripts/integration/src/valinor_integration/collection.py create mode 100644 scripts/integration/src/valinor_integration/errors.py create mode 100644 scripts/integration/src/valinor_integration/extract.py create mode 100644 scripts/integration/src/valinor_integration/runner.py create mode 100644 scripts/integration/src/valinor_integration/settings.py create mode 100644 scripts/integration/src/valinor_integration/valhalla.py create mode 100644 scripts/integration/sub-extract-tiles.txt create mode 100644 scripts/integration/tests/test_checks.py create mode 100644 scripts/integration/tests/test_cli.py create mode 100644 scripts/integration/tests/test_collection.py create mode 100644 scripts/integration/tests/test_extract.py create mode 100644 scripts/integration/tests/test_settings.py create mode 100644 scripts/integration/uv.lock diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000..befadfb --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,50 @@ +# Integration tests of offline extracts with C++ Valhalla. See scripts/integration/README.md. +name: Integration + +on: + pull_request: + branches: [ "main" ] + paths: + - "valhalla-graphtile/src/extract/**" + # The tar layout and the index.bin code are in the tile providers. + - "valhalla-graphtile/src/tile_provider/**" + - "valinor-cli/**" + - "scripts/integration/**" + - ".github/workflows/integration.yml" + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + # The Valhalla release under test. Without this pin, the harness uses `latest`. + # When you change it, examine the differences in the responses and logs before you merge. + VALHALLA_VERSION: "3.8.2" + +jobs: + integration: + # Linux only: the job tests the Docker image, not the platform. + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - uses: astral-sh/setup-uv@v10.3.0 + + - name: Check the harness + working-directory: scripts/integration + run: | + uv run --locked ruff check + uv run --locked ruff format --check + uv run --locked pytest + + - name: Run the integration tests with C++ Valhalla + run: uv run --locked --project scripts/integration valinor-integration + + - name: Upload harness artifacts on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: integration-workdir + path: | + target/integration/responses + target/integration/valhalla-*.log + target/integration/valhalla.json diff --git a/scripts/integration/.gitignore b/scripts/integration/.gitignore new file mode 100644 index 0000000..5831ca4 --- /dev/null +++ b/scripts/integration/.gitignore @@ -0,0 +1,3 @@ +.venv/ +__pycache__/ +.pytest_cache/ diff --git a/scripts/integration/README.md b/scripts/integration/README.md new file mode 100644 index 0000000..709c9a1 --- /dev/null +++ b/scripts/integration/README.md @@ -0,0 +1,94 @@ +# Integration tests with C++ Valhalla + +This harness makes sure that a sub-extract from `valinor-cli extract` +works in stock C++ Valhalla the same way as the full extract. + +The harness does these steps: + +1. It writes a sub-extract of the full extract with `valinor-cli extract`. + The tiles are in [`sub-extract-tiles.txt`](sub-extract-tiles.txt). + The sub-extract must be a proper subset of the full extract, + or the comparison would prove nothing. +2. It starts two `valhalla_service` containers with docker-py: + one for the full extract and one for the sub-extract. + The image's own `valhalla_build_config` makes the config. + The harness sets `mjolnir.shortcut_caching` to `false`, + because Valhalla stops at startup with a partial extract when it is `true`. +3. It sends each request of the [Bruno collection](bruno/) to both servers. + It saves each response, and it compares the two responses of each request + (see [`checks.py`](src/valinor_integration/checks.py) for the fields that it compares). +4. It makes sure that the logs of both servers contain no `[ERROR]` lines. + +The default full extract is the Andorra fixture +(`valhalla-graphtile/fixtures/andorra-tiles.tar`). + +## Requirements + +- [uv](https://docs.astral.sh/uv/) +- Docker (Docker Desktop, colima, or a Docker daemon on Linux) +- cargo, if you do not set `VALINOR_CLI` + +## Run the test + +From the root of the repository: + +```bash +uv run --project scripts/integration valinor-integration +``` + +The command prints `integration: PASS`, or a list of failures with a diff for each difference. +The work directory (`target/integration` by default) contains: + +- `sub-extract.tar`: the sub-extract. +- `valhalla.json`: the Valhalla config. +- `responses/full/` and `responses/sub/`: the raw response of each request. +- `valhalla-full.log` and `valhalla-sub.log`: the logs of the two servers. + +Use `--help` for the options (a different full extract, work directory, tile list, or collection). + +## Send requests by hand with Bruno + +```bash +uv run --project scripts/integration valinor-integration serve +``` + +This serves the full extract at `http://localhost:8002` +and the sub-extract at `http://localhost:8003` until you push Ctrl-C. +Open `scripts/integration/bruno` in [Bruno](https://www.usebruno.com/), +and select the `full` or the `sub` environment. + +## Environment variables + +All variables are optional. + +| Variable | Default | Description | +| --- | --- | --- | +| `VALHALLA_VERSION` | `latest` | The tag of the Valhalla image, such as `3.8.2`. CI sets it in [`integration.yml`](../../.github/workflows/integration.yml). If it is not set, the harness uses `latest` and logs a warning. | +| `VALHALLA_IMAGE` | `ghcr.io/valhalla/valhalla` | The image repository. | +| `VALINOR_CLI` | (none) | The path of a `valinor-cli` binary. If it is not set, the harness runs `cargo run --release -p valinor-cli`. | +| `VALHALLA_PORT` | `8002` | The port of the full extract in `serve` mode. The sub-extract uses the next port. If you change it, change the Bruno environments too. | + +## Add a request + +1. Add the request to the Bruno collection. + You can do this in the Bruno app, or you can copy a file in `bruno/route/`. + Keep all coordinates far inside the tiles of the sub-extract, + so that the request never needs a tile which the sub-extract does not contain. +2. If the request uses a new endpoint, add a check for the endpoint to + [`checks.py`](src/valinor_integration/checks.py). + The harness refuses a request for an endpoint that has no check. + +The harness reads a subset of the Bruno YAML format: +the method, the URL (with `{{baseUrl}}`), headers, and a JSON body. +It refuses requests with Bruno scripts or assertions, +because it does not run them; put such checks in `checks.py`. + +## Unit tests of the harness + +```bash +uv run --project scripts/integration pytest +``` + +```bash +uv run --project scripts/integration ruff check +``` diff --git a/scripts/integration/bruno/environments/full.yml b/scripts/integration/bruno/environments/full.yml new file mode 100644 index 0000000..bab1789 --- /dev/null +++ b/scripts/integration/bruno/environments/full.yml @@ -0,0 +1,4 @@ +name: full +variables: + - name: baseUrl + value: http://localhost:8002 diff --git a/scripts/integration/bruno/environments/sub.yml b/scripts/integration/bruno/environments/sub.yml new file mode 100644 index 0000000..3ec4348 --- /dev/null +++ b/scripts/integration/bruno/environments/sub.yml @@ -0,0 +1,4 @@ +name: sub +variables: + - name: baseUrl + value: http://localhost:8003 diff --git a/scripts/integration/bruno/locate/andorra-la-vella-center.yml b/scripts/integration/bruno/locate/andorra-la-vella-center.yml new file mode 100644 index 0000000..5746419 --- /dev/null +++ b/scripts/integration/bruno/locate/andorra-la-vella-center.yml @@ -0,0 +1,28 @@ +info: + name: andorra-la-vella-center + type: http + seq: 1 + +http: + method: POST + url: "{{baseUrl}}/locate" + body: + type: json + data: |- + { + "locations": [ + { + "lat": 42.5063, + "lon": 1.5218 + } + ], + "costing": "auto", + "verbose": true + } + auth: inherit + +settings: + encodeUrl: true + timeout: 0 + followRedirects: true + maxRedirects: 5 diff --git a/scripts/integration/bruno/locate/folder.yml b/scripts/integration/bruno/locate/folder.yml new file mode 100644 index 0000000..9ee0e40 --- /dev/null +++ b/scripts/integration/bruno/locate/folder.yml @@ -0,0 +1,8 @@ +info: + name: locate + type: folder + seq: 2 + +request: + auth: inherit + diff --git a/scripts/integration/bruno/opencollection.yml b/scripts/integration/bruno/opencollection.yml new file mode 100644 index 0000000..2246956 --- /dev/null +++ b/scripts/integration/bruno/opencollection.yml @@ -0,0 +1,10 @@ +opencollection: 1.0.0 + +info: + name: Valinor integration +bundled: false +extensions: + bruno: + ignore: + - node_modules + - .git diff --git a/scripts/integration/bruno/route/andorra-la-vella-center-pedestrian.yml b/scripts/integration/bruno/route/andorra-la-vella-center-pedestrian.yml new file mode 100644 index 0000000..591ffe9 --- /dev/null +++ b/scripts/integration/bruno/route/andorra-la-vella-center-pedestrian.yml @@ -0,0 +1,32 @@ +info: + name: andorra-la-vella-center-pedestrian + type: http + seq: 5 + +http: + method: POST + url: "{{baseUrl}}/route" + body: + type: json + data: |- + { + "locations": [ + { + "lat": 42.5063, + "lon": 1.5218 + }, + { + "lat": 42.5103, + "lon": 1.5336 + } + ], + "costing": "pedestrian", + "units": "kilometers" + } + auth: inherit + +settings: + encodeUrl: true + timeout: 0 + followRedirects: true + maxRedirects: 5 diff --git a/scripts/integration/bruno/route/andorra-la-vella-to-encamp-auto.yml b/scripts/integration/bruno/route/andorra-la-vella-to-encamp-auto.yml new file mode 100644 index 0000000..31c3dfa --- /dev/null +++ b/scripts/integration/bruno/route/andorra-la-vella-to-encamp-auto.yml @@ -0,0 +1,32 @@ +info: + name: andorra-la-vella-to-encamp-auto + type: http + seq: 1 + +http: + method: POST + url: "{{baseUrl}}/route" + body: + type: json + data: |- + { + "locations": [ + { + "lat": 42.5063, + "lon": 1.5218 + }, + { + "lat": 42.536, + "lon": 1.5836 + } + ], + "costing": "auto", + "units": "kilometers" + } + auth: inherit + +settings: + encodeUrl: true + timeout: 0 + followRedirects: true + maxRedirects: 5 diff --git a/scripts/integration/bruno/route/andorra-la-vella-to-la-massana-auto.yml b/scripts/integration/bruno/route/andorra-la-vella-to-la-massana-auto.yml new file mode 100644 index 0000000..11ad474 --- /dev/null +++ b/scripts/integration/bruno/route/andorra-la-vella-to-la-massana-auto.yml @@ -0,0 +1,32 @@ +info: + name: andorra-la-vella-to-la-massana-auto + type: http + seq: 2 + +http: + method: POST + url: "{{baseUrl}}/route" + body: + type: json + data: |- + { + "locations": [ + { + "lat": 42.5078, + "lon": 1.5211 + }, + { + "lat": 42.5449, + "lon": 1.5148 + } + ], + "costing": "auto", + "units": "kilometers" + } + auth: inherit + +settings: + encodeUrl: true + timeout: 0 + followRedirects: true + maxRedirects: 5 diff --git a/scripts/integration/bruno/route/encamp-to-canillo-auto.yml b/scripts/integration/bruno/route/encamp-to-canillo-auto.yml new file mode 100644 index 0000000..179452e --- /dev/null +++ b/scripts/integration/bruno/route/encamp-to-canillo-auto.yml @@ -0,0 +1,32 @@ +info: + name: encamp-to-canillo-auto + type: http + seq: 3 + +http: + method: POST + url: "{{baseUrl}}/route" + body: + type: json + data: |- + { + "locations": [ + { + "lat": 42.536, + "lon": 1.5836 + }, + { + "lat": 42.5676, + "lon": 1.5976 + } + ], + "costing": "auto", + "units": "kilometers" + } + auth: inherit + +settings: + encodeUrl: true + timeout: 0 + followRedirects: true + maxRedirects: 5 diff --git a/scripts/integration/bruno/route/folder.yml b/scripts/integration/bruno/route/folder.yml new file mode 100644 index 0000000..c3c95fb --- /dev/null +++ b/scripts/integration/bruno/route/folder.yml @@ -0,0 +1,8 @@ +info: + name: route + type: folder + seq: 1 + +request: + auth: inherit + diff --git a/scripts/integration/bruno/route/ordino-to-la-massana-bicycle.yml b/scripts/integration/bruno/route/ordino-to-la-massana-bicycle.yml new file mode 100644 index 0000000..32baceb --- /dev/null +++ b/scripts/integration/bruno/route/ordino-to-la-massana-bicycle.yml @@ -0,0 +1,32 @@ +info: + name: ordino-to-la-massana-bicycle + type: http + seq: 4 + +http: + method: POST + url: "{{baseUrl}}/route" + body: + type: json + data: |- + { + "locations": [ + { + "lat": 42.5562, + "lon": 1.5332 + }, + { + "lat": 42.5449, + "lon": 1.5148 + } + ], + "costing": "bicycle", + "units": "kilometers" + } + auth: inherit + +settings: + encodeUrl: true + timeout: 0 + followRedirects: true + maxRedirects: 5 diff --git a/scripts/integration/pyproject.toml b/scripts/integration/pyproject.toml new file mode 100644 index 0000000..b196414 --- /dev/null +++ b/scripts/integration/pyproject.toml @@ -0,0 +1,33 @@ +[project] +name = "valinor-integration" +version = "0.1.0" +description = "Integration tests of Valinor tile extracts with stock C++ Valhalla" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "docker>=7.2.0", + "pyyaml>=6.0.3", + "requests>=2.34.2", +] + +[project.scripts] +valinor-integration = "valinor_integration.cli:main" + +[build-system] +requires = ["uv_build>=0.11.29,<0.12"] +build-backend = "uv_build" + +[dependency-groups] +dev = [ + "pytest>=9.1.1", + "ruff>=0.17.1", +] + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.ruff] +line-length = 100 + +[tool.ruff.lint] +select = ["E", "F", "I", "B", "UP", "SIM", "RUF"] diff --git a/scripts/integration/src/valinor_integration/__init__.py b/scripts/integration/src/valinor_integration/__init__.py new file mode 100644 index 0000000..07d3792 --- /dev/null +++ b/scripts/integration/src/valinor_integration/__init__.py @@ -0,0 +1 @@ +"""Integration tests of Valinor tile extracts with stock C++ Valhalla.""" diff --git a/scripts/integration/src/valinor_integration/__main__.py b/scripts/integration/src/valinor_integration/__main__.py new file mode 100644 index 0000000..18ac038 --- /dev/null +++ b/scripts/integration/src/valinor_integration/__main__.py @@ -0,0 +1,5 @@ +import sys + +from valinor_integration.cli import main + +sys.exit(main()) diff --git a/scripts/integration/src/valinor_integration/checks.py b/scripts/integration/src/valinor_integration/checks.py new file mode 100644 index 0000000..9e18c56 --- /dev/null +++ b/scripts/integration/src/valinor_integration/checks.py @@ -0,0 +1,120 @@ +"""The checks of the responses, for each Valhalla endpoint. + +A check has ``validate``, which finds problems in one response, +and ``project``, which keeps only the fields that the comparison uses. +To test a new endpoint, add a request to the Bruno collection and a check here. +""" + +from __future__ import annotations + +import difflib +import json +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Any + +# Per-maneuver fields included in the compared projection. +MANEUVER_FIELDS = ( + "type", + "instruction", + "verbal_pre_transition_instruction", + "street_names", + "length", + "time", + "begin_shape_index", + "end_shape_index", + "travel_mode", + "travel_type", +) + + +@dataclass(frozen=True) +class Check: + validate: Callable[[Any], list[str]] + project: Callable[[Any], Any] + + +def _validate_route(response: Any) -> list[str]: + trip = response.get("trip") if isinstance(response, dict) else None + if not isinstance(trip, dict) or not trip.get("legs"): + return ["the response contains no trip legs"] + return [] + + +def _project_route(response: Any) -> Any: + """Projects a /route response onto its status, units, shapes, summaries, and maneuvers.""" + trip = response["trip"] + return { + "status": trip.get("status"), + "units": trip.get("units"), + "summary": { + "length": trip["summary"]["length"], + "time": trip["summary"]["time"], + }, + "legs": [ + { + "shape": leg["shape"], + "summary": { + "length": leg["summary"]["length"], + "time": leg["summary"]["time"], + }, + "maneuvers": [ + {name: maneuver.get(name) for name in MANEUVER_FIELDS} + for maneuver in leg["maneuvers"] + ], + } + for leg in trip["legs"] + ], + } + + +def _validate_locate(response: Any) -> list[str]: + if not isinstance(response, list) or not response: + return ["the response is not a non-empty list of locations"] + if sum(len(location.get("edges") or []) for location in response) == 0: + return ["the response contains no edges"] + return [] + + +def _project_locate(response: Any) -> Any: + """Projects a /locate response onto the edges found at each location.""" + return [ + sorted( + ( + { + "edge_id": edge.get("edge_id"), + # With `verbose`, the OSM way ID is in the edge info. + "way_id": (edge.get("edge_info") or {}).get("way_id"), + "correlated_lat": edge.get("correlated_lat"), + "correlated_lon": edge.get("correlated_lon"), + "percent_along": edge.get("percent_along"), + } + for edge in location.get("edges") or [] + ), + key=lambda edge: json.dumps(edge, sort_keys=True), + ) + for location in response + ] + + +#: The checks, by the path of the endpoint. +CHECKS: Mapping[str, Check] = { + "/route": Check(validate=_validate_route, project=_project_route), + "/locate": Check(validate=_validate_locate, project=_project_locate), +} + + +def diff_projections( + name: str, expected: Any, actual: Any, expected_label: str, actual_label: str +) -> str: + """A unified diff of two projections, or an empty string if they are equal.""" + if expected == actual: + return "" + return "".join( + difflib.unified_diff( + json.dumps(expected, indent=2, sort_keys=True).splitlines(keepends=True), + json.dumps(actual, indent=2, sort_keys=True).splitlines(keepends=True), + fromfile=f"{expected_label}/{name}", + tofile=f"{actual_label}/{name}", + ) + ) diff --git a/scripts/integration/src/valinor_integration/cli.py b/scripts/integration/src/valinor_integration/cli.py new file mode 100644 index 0000000..b12cb7f --- /dev/null +++ b/scripts/integration/src/valinor_integration/cli.py @@ -0,0 +1,109 @@ +"""Integration tests of Valinor tile extracts with stock C++ Valhalla. + +``valinor-integration run`` (the default) compares a sub-extract with the full extract. +``valinor-integration serve`` serves both extracts until Ctrl-C. +See README.md for the environment variables. +""" + +from __future__ import annotations + +import argparse +import logging +import signal +import sys +from pathlib import Path + +from valinor_integration import runner +from valinor_integration.errors import HarnessError +from valinor_integration.settings import ( + DEFAULT_COLLECTION, + DEFAULT_FULL_EXTRACT, + DEFAULT_TILE_LIST, + DEFAULT_WORKDIR, + Settings, +) + +log = logging.getLogger("valinor_integration") + +COMMANDS = ("run", "serve") + + +def _parser() -> argparse.ArgumentParser: + common = argparse.ArgumentParser(add_help=False) + common.add_argument( + "--extract", + type=Path, + default=DEFAULT_FULL_EXTRACT, + help="the full extract tarball (default: the Andorra fixture)", + ) + common.add_argument( + "--workdir", + type=Path, + default=DEFAULT_WORKDIR, + help="the directory for the sub-extract, responses, and logs (default: target/integration)", + ) + common.add_argument( + "--tiles", + type=Path, + default=DEFAULT_TILE_LIST, + help="the tile list of the sub-extract (default: sub-extract-tiles.txt)", + ) + common.add_argument( + "--collection", + type=Path, + default=DEFAULT_COLLECTION, + help="the Bruno collection of requests (default: bruno/)", + ) + + parser = argparse.ArgumentParser( + prog="valinor-integration", + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + subparsers = parser.add_subparsers(dest="command", required=True) + subparsers.add_parser("run", parents=[common], help="run the integration test (the default)") + subparsers.add_parser("serve", parents=[common], help="serve both extracts until Ctrl-C") + return parser + + +def _interrupt(signum: int, frame: object) -> None: + raise KeyboardInterrupt + + +def main(argv: list[str] | None = None) -> int: + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") + # Remove the containers on SIGTERM (for example, a cancelled CI job), as on Ctrl-C. + signal.signal(signal.SIGTERM, _interrupt) + argv = sys.argv[1:] if argv is None else argv + # Make `run` the default command. + if not argv or argv[0] not in (*COMMANDS, "-h", "--help"): + argv = ["run", *argv] + args = _parser().parse_args(argv) + inputs = runner.Inputs( + full_extract=args.extract.resolve(), + workdir=args.workdir.resolve(), + tile_list=args.tiles.resolve(), + collection=args.collection.resolve(), + ) + + try: + settings = Settings.from_env() + log.info("Valhalla image: %s", settings.image_ref) + if args.command == "serve": + runner.serve(settings, inputs) + return 0 + if runner.run(settings, inputs): + log.info("integration: PASS") + return 0 + log.error("integration: FAIL (responses and logs are in %s)", inputs.workdir) + return 1 + except HarnessError as e: + log.error("%s", e) + return 1 + except KeyboardInterrupt: + log.info("stopped") + return 130 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/integration/src/valinor_integration/collection.py b/scripts/integration/src/valinor_integration/collection.py new file mode 100644 index 0000000..81b14c5 --- /dev/null +++ b/scripts/integration/src/valinor_integration/collection.py @@ -0,0 +1,169 @@ +"""Reads the requests of a Bruno collection in the OpenCollection YAML format. + +The harness reads an HTTP method, a URL with ``{{variable}}`` placeholders, headers, +and a JSON body. It refuses a request that uses more, such as scripts or assertions, +so that it never sends a different request than Bruno. +""" + +from __future__ import annotations + +import json +import re +from collections.abc import Mapping +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit + +import yaml + +from valinor_integration.errors import HarnessError + +COLLECTION_FILE = "opencollection.yml" +FOLDER_FILE = "folder.yml" +ENVIRONMENTS_DIR = "environments" + +# The keys of a request file that the harness reads, or ignores (`settings` and `docs`). +_REQUEST_KEYS = {"info", "http", "settings", "docs"} +_HTTP_KEYS = {"method", "url", "body", "headers", "auth"} +_VARIABLE = re.compile(r"\{\{\s*([A-Za-z0-9_.-]+)\s*\}\}") + + +@dataclass(frozen=True) +class BrunoRequest: + """One HTTP request of the collection.""" + + #: The request name, from ``info.name``. + name: str + #: The path of the request file, relative to the collection root. + path: Path + seq: int + method: str + #: The URL, with ``{{variable}}`` placeholders. + url: str + headers: Mapping[str, str] = field(default_factory=dict) + #: The parsed JSON body, or ``None`` if the request has no body. + body: Any = None + + @property + def id(self) -> str: + """A unique ID: the path of the request file without the extension.""" + return self.path.with_suffix("").as_posix() + + def resolve_url(self, variables: Mapping[str, str]) -> str: + """Replaces the ``{{variable}}`` placeholders of the URL.""" + + def replace(match: re.Match[str]) -> str: + name = match.group(1) + if name not in variables: + raise HarnessError(f"{self.path}: the URL uses the unknown variable {{{{{name}}}}}") + return variables[name] + + return _VARIABLE.sub(replace, self.url) + + def endpoint(self, variables: Mapping[str, str]) -> str: + """The path of the resolved URL (for example, ``/route``).""" + return urlsplit(self.resolve_url(variables)).path + + +def load_collection(root: Path) -> list[BrunoRequest]: + """Reads every request of the collection at ``root``, in the order of the Bruno app.""" + if not (root / COLLECTION_FILE).is_file(): + raise HarnessError(f"{root} is not a Bruno collection: it has no {COLLECTION_FILE}") + + requests = [] + for path in sorted(root.rglob("*.yml")): + relative = path.relative_to(root) + if relative.name in (COLLECTION_FILE, FOLDER_FILE) or relative.parts[0] == ENVIRONMENTS_DIR: + continue + requests.append(parse_request(relative, path.read_text(encoding="utf-8"))) + + if not requests: + raise HarnessError(f"the Bruno collection at {root} contains no requests") + ids = [request.id for request in requests] + if len(set(ids)) != len(ids): + raise HarnessError(f"the Bruno collection at {root} contains duplicate request IDs") + + requests.sort(key=lambda request: (request.path.parent.as_posix(), request.seq, request.name)) + return requests + + +def parse_request(path: Path, text: str) -> BrunoRequest: + """Parses one request file. ``path`` is only used in error messages and the request ID.""" + + def fail(message: str) -> HarnessError: + return HarnessError(f"{path}: {message}") + + try: + document = yaml.safe_load(text) + except yaml.YAMLError as e: + raise fail(f"invalid YAML: {e}") from None + if not isinstance(document, dict): + raise fail("a request file must be a YAML mapping") + + unsupported = set(document) - _REQUEST_KEYS + if unsupported: + raise fail( + f"unsupported sections {sorted(unsupported)}; the harness does not run Bruno " + "scripts or assertions (put checks in valinor_integration.checks)" + ) + + info = document.get("info") or {} + if info.get("type") != "http": + raise fail(f"unsupported request type {info.get('type')!r}; expected 'http'") + name = info.get("name") + if not isinstance(name, str) or not name: + raise fail("info.name must be a non-empty string") + seq = info.get("seq", 0) + if not isinstance(seq, int): + raise fail("info.seq must be an integer") + + http = document.get("http") or {} + unsupported = set(http) - _HTTP_KEYS + if unsupported: + raise fail(f"unsupported http fields {sorted(unsupported)}") + method = http.get("method") + url = http.get("url") + if not isinstance(method, str) or not isinstance(url, str): + raise fail("http.method and http.url must be strings") + if http.get("auth", "none") not in ("none", "inherit"): + raise fail("authentication is not supported") + + return BrunoRequest( + name=name, + path=path, + seq=seq, + method=method.upper(), + url=url, + headers=_parse_headers(http.get("headers"), fail), + body=_parse_body(http.get("body"), fail), + ) + + +def _parse_headers(headers: Any, fail: Any) -> dict[str, str]: + if headers is None: + return {} + if not isinstance(headers, list): + raise fail("http.headers must be a list") + parsed = {} + for header in headers: + if not isinstance(header, dict) or not isinstance(header.get("name"), str): + raise fail("each header must have a name") + if header.get("disabled", False): + continue + parsed[header["name"]] = str(header.get("value", "")) + return parsed + + +def _parse_body(body: Any, fail: Any) -> Any: + if body is None: + return None + if not isinstance(body, dict) or body.get("type") != "json": + raise fail("only JSON bodies (http.body.type: json) are supported") + data = body.get("data") + if not isinstance(data, str): + raise fail("http.body.data must be a string of JSON") + try: + return json.loads(data) + except json.JSONDecodeError as e: + raise fail(f"http.body.data is not valid JSON: {e}") from None diff --git a/scripts/integration/src/valinor_integration/errors.py b/scripts/integration/src/valinor_integration/errors.py new file mode 100644 index 0000000..5ff783b --- /dev/null +++ b/scripts/integration/src/valinor_integration/errors.py @@ -0,0 +1,5 @@ +"""The error type of the harness.""" + + +class HarnessError(Exception): + """A failure that the harness reports with a message, and no traceback.""" diff --git a/scripts/integration/src/valinor_integration/extract.py b/scripts/integration/src/valinor_integration/extract.py new file mode 100644 index 0000000..2b552a8 --- /dev/null +++ b/scripts/integration/src/valinor_integration/extract.py @@ -0,0 +1,97 @@ +"""Writes the sub-extract with ``valinor-cli extract``, and checks it against the full extract.""" + +from __future__ import annotations + +import logging +import subprocess +import tarfile +from pathlib import Path + +from valinor_integration.errors import HarnessError +from valinor_integration.settings import REPO_ROOT, Settings + +log = logging.getLogger(__name__) + + +def read_tile_list(path: Path) -> list[str]: + """Reads a tile list: one ``level/tile_id`` on each line. + + ``#`` starts a comment, and blank lines are ignored. + ``valinor-cli`` checks the form of each tile. + """ + tiles = [] + for line in path.read_text(encoding="utf-8").splitlines(): + tile = line.split("#", 1)[0].strip() + if tile: + tiles.append(tile.replace(" ", "")) + if not tiles: + raise HarnessError(f"the tile list contains no tiles: {path}") + return tiles + + +def valinor_cli_command(settings: Settings) -> list[str]: + """The command which runs ``valinor-cli``.""" + if settings.valinor_cli is not None: + return [str(settings.valinor_cli)] + return [ + "cargo", + "run", + "--quiet", + "--release", + "--manifest-path", + str(REPO_ROOT / "Cargo.toml"), + "--package", + "valinor-cli", + "--", + ] + + +def write_sub_extract( + settings: Settings, full_extract: Path, tiles: list[str], output: Path +) -> None: + """Writes a sub-extract of ``full_extract`` which contains ``tiles``.""" + command = [ + *valinor_cli_command(settings), + "extract", + str(full_extract), + str(output), + "--tile", + ",".join(tiles), + ] + log.info("writing the sub-extract: %s", " ".join(command)) + try: + subprocess.run(command, check=True) + except FileNotFoundError as e: + raise HarnessError(f"cannot run {command[0]}: {e}") from None + except subprocess.CalledProcessError as e: + raise HarnessError(f"valinor-cli extract failed with exit status {e.returncode}") from None + + +def check_proper_subset(full_extract: Path, sub_extract: Path) -> list[str]: + """Makes sure that the sub-extract is a proper subset of the full extract. + + Returns the entries that are only in the full extract. + """ + full_entries = _entry_names(full_extract) + sub_entries = _entry_names(sub_extract) + + extra = sorted(sub_entries - full_entries) + if extra: + raise HarnessError( + f"the sub-extract contains entries which are not in the full extract: {extra}" + ) + removed = sorted(full_entries - sub_entries) + if not removed: + raise HarnessError( + "the sub-extract contains every entry of the full extract, " + "so the comparison would prove nothing" + ) + return removed + + +def _entry_names(archive: Path) -> set[str]: + try: + with tarfile.open(archive) as tar: + return set(tar.getnames()) + except (OSError, tarfile.TarError) as e: + raise HarnessError(f"cannot read the tar archive {archive}: {e}") from None diff --git a/scripts/integration/src/valinor_integration/runner.py b/scripts/integration/src/valinor_integration/runner.py new file mode 100644 index 0000000..d612a4e --- /dev/null +++ b/scripts/integration/src/valinor_integration/runner.py @@ -0,0 +1,201 @@ +"""The two modes of the harness: ``run`` (the test) and ``serve`` (for manual requests).""" + +from __future__ import annotations + +import json +import logging +import shutil +import time +from contextlib import ExitStack +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import requests + +from valinor_integration import valhalla +from valinor_integration.checks import CHECKS, diff_projections +from valinor_integration.collection import BrunoRequest, load_collection +from valinor_integration.errors import HarnessError +from valinor_integration.extract import check_proper_subset, read_tile_list, write_sub_extract +from valinor_integration.settings import Settings + +log = logging.getLogger(__name__) + +#: The names of the two extracts. +FULL = "full" +SUB = "sub" + + +@dataclass(frozen=True) +class Inputs: + """The files of a harness run.""" + + full_extract: Path + workdir: Path + tile_list: Path + collection: Path + + @property + def sub_extract(self) -> Path: + return self.workdir / "sub-extract.tar" + + +def run(settings: Settings, inputs: Inputs) -> bool: + """Serves both extracts, sends every request to both, and compares the responses. + + Returns ``True`` if every check passes. + The responses, the logs, and the config are saved in the work directory. + """ + collection = _load_checked_collection(inputs.collection) + client, config = _prepare(settings, inputs) + + responses_dir = inputs.workdir / "responses" + shutil.rmtree(responses_dir, ignore_errors=True) + + failures: list[str] = [] + responses: dict[str, dict[str, Any]] = {} + with ExitStack() as stack: + servers = { + name: stack.enter_context( + valhalla.ValhallaServer(client, settings.image_ref, name, extract, config) + ) + for name, extract in ((FULL, inputs.full_extract), (SUB, inputs.sub_extract)) + } + + for name, server in servers.items(): + responses[name] = _send_all(server, collection, responses_dir / name, failures) + + # Capture the logs before the containers are removed. + for name, server in servers.items(): + logs = server.logs() + log_file = inputs.workdir / f"valhalla-{name}.log" + log_file.write_text(logs, encoding="utf-8") + error_lines = [line for line in logs.splitlines() if "[ERROR]" in line] + if error_lines: + failures.append( + f"{name}: {len(error_lines)} [ERROR] lines in the valhalla_service log " + f"({log_file}); the first is: {error_lines[0]}" + ) + + for request in collection: + full, sub = responses[FULL].get(request.id), responses[SUB].get(request.id) + if full is None or sub is None: + continue # The send failure is already in `failures`. + check = CHECKS[request.endpoint(_variables(""))] + diff = diff_projections(request.id, check.project(full), check.project(sub), FULL, SUB) + if diff: + failures.append(f"{request.id}: the responses differ:\n{diff}") + else: + log.info("ok: %s is identical for both extracts", request.id) + + for failure in failures: + log.error("FAIL: %s", failure) + return not failures + + +def serve(settings: Settings, inputs: Inputs) -> None: + """Serves both extracts on fixed ports until Ctrl-C or SIGTERM raises ``KeyboardInterrupt``.""" + client, config = _prepare(settings, inputs) + ports = {FULL: settings.serve_port, SUB: settings.serve_port + 1} + + with ExitStack() as stack: + servers = [ + stack.enter_context( + valhalla.ValhallaServer( + client, settings.image_ref, name, extract, config, host_port=ports[name] + ) + ) + for name, extract in ((FULL, inputs.full_extract), (SUB, inputs.sub_extract)) + ] + log.info( + "serving the full extract at %s and the sub-extract at %s; " + "in Bruno, open %s and select the 'full' or 'sub' environment; press Ctrl-C to stop", + servers[0].base_url, + servers[1].base_url, + inputs.collection, + ) + while True: + time.sleep(1) + + +def _load_checked_collection(root: Path) -> list[BrunoRequest]: + """Loads the collection, and makes sure that every request has a check.""" + collection = load_collection(root) + for request in collection: + endpoint = request.endpoint(_variables("")) + if endpoint not in CHECKS: + raise HarnessError( + f"{request.path}: the harness has no check for {endpoint}; " + f"add one to valinor_integration.checks (known: {sorted(CHECKS)})" + ) + log.info("loaded %d requests from %s", len(collection), root) + return collection + + +def _prepare(settings: Settings, inputs: Inputs) -> tuple[Any, dict[str, Any]]: + """Writes the sub-extract, and prepares Docker and the Valhalla config.""" + if not inputs.full_extract.is_file(): + raise HarnessError(f"the full extract is not a file: {inputs.full_extract}") + inputs.workdir.mkdir(parents=True, exist_ok=True) + + tiles = read_tile_list(inputs.tile_list) + inputs.sub_extract.unlink(missing_ok=True) + write_sub_extract(settings, inputs.full_extract, tiles, inputs.sub_extract) + for entry in check_proper_subset(inputs.full_extract, inputs.sub_extract): + log.info("only in the full extract: %s", entry) + + client = valhalla.connect() + valhalla.ensure_image(client, settings) + config = valhalla.build_config(client, settings.image_ref) + (inputs.workdir / "valhalla.json").write_text(json.dumps(config, indent=2), encoding="utf-8") + return client, config + + +def _variables(base_url: str) -> dict[str, str]: + """The variables of the Bruno environments, for one server.""" + return {"baseUrl": base_url} + + +def _send_all( + server: valhalla.ValhallaServer, + collection: list[BrunoRequest], + out_dir: Path, + failures: list[str], +) -> dict[str, Any]: + """Sends every request to one server, and saves and validates each response.""" + variables = _variables(server.base_url) + responses = {} + for request in collection: + label = f"{server.name}: {request.id}" + try: + response = requests.request( + request.method, + request.resolve_url(variables), + json=request.body, + headers=dict(request.headers), + timeout=valhalla.REQUEST_TIMEOUT_SECONDS, + ) + except requests.RequestException as e: + failures.append(f"{label}: the request failed: {e}") + continue + + out_path = out_dir / f"{request.id}.json" + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(response.text, encoding="utf-8") + + if response.status_code != 200: + failures.append(f"{label}: HTTP {response.status_code}: {response.text[:500]}") + continue + try: + body = response.json() + except ValueError: + failures.append(f"{label}: the response is not JSON") + continue + + problems = CHECKS[request.endpoint(variables)].validate(body) + if problems: + failures.extend(f"{label}: {problem}" for problem in problems) + continue + responses[request.id] = body + return responses diff --git a/scripts/integration/src/valinor_integration/settings.py b/scripts/integration/src/valinor_integration/settings.py new file mode 100644 index 0000000..895943f --- /dev/null +++ b/scripts/integration/src/valinor_integration/settings.py @@ -0,0 +1,92 @@ +"""Settings of the harness, from environment variables. + +Every setting has a default, so ``valinor-integration`` runs with no configuration. +See README.md for the variables. +""" + +from __future__ import annotations + +import os +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path + +from valinor_integration.errors import HarnessError + +DEFAULT_VALHALLA_IMAGE = "ghcr.io/valhalla/valhalla" +LATEST_TAG = "latest" +DEFAULT_SERVE_PORT = 8002 + +# This file is scripts/integration/src/valinor_integration/settings.py. +HARNESS_DIR = Path(__file__).resolve().parents[2] +REPO_ROOT = HARNESS_DIR.parents[1] + +DEFAULT_FULL_EXTRACT = REPO_ROOT / "valhalla-graphtile" / "fixtures" / "andorra-tiles.tar" +DEFAULT_WORKDIR = REPO_ROOT / "target" / "integration" +DEFAULT_TILE_LIST = HARNESS_DIR / "sub-extract-tiles.txt" +DEFAULT_COLLECTION = HARNESS_DIR / "bruno" + + +@dataclass(frozen=True) +class Settings: + """The settings which come from environment variables.""" + + valhalla_image: str + #: The image tag, or ``None`` to use ``latest``. + valhalla_version: str | None + valinor_cli: Path | None + serve_port: int + + @property + def image_tag(self) -> str: + return self.valhalla_version or LATEST_TAG + + @property + def image_ref(self) -> str: + return f"{self.valhalla_image}:{self.image_tag}" + + @property + def is_pinned(self) -> bool: + return self.valhalla_version is not None + + @classmethod + def from_env(cls, env: Mapping[str, str] | None = None) -> Settings: + """Reads the settings from ``env`` (by default, the process environment). + + An empty variable is the same as an unset variable. + """ + env = os.environ if env is None else env + + def get(name: str) -> str | None: + value = env.get(name, "").strip() + return value or None + + version = get("VALHALLA_VERSION") + if version is not None and ("/" in version or ":" in version or "@" in version): + raise HarnessError( + f"VALHALLA_VERSION must be an image tag such as 3.8.2, not {version!r}; " + "set VALHALLA_IMAGE to change the repository" + ) + + cli = get("VALINOR_CLI") + valinor_cli = Path(cli).expanduser() if cli is not None else None + if valinor_cli is not None and not valinor_cli.is_file(): + raise HarnessError(f"VALINOR_CLI is not a file: {valinor_cli}") + + port_value = get("VALHALLA_PORT") + serve_port = DEFAULT_SERVE_PORT + if port_value is not None: + try: + serve_port = int(port_value) + except ValueError: + raise HarnessError(f"VALHALLA_PORT is not a number: {port_value!r}") from None + # serve mode also uses the next port, for the sub-extract. + if not 1 <= serve_port <= 65534: + raise HarnessError(f"VALHALLA_PORT must be from 1 to 65534, not {serve_port}") + + return cls( + valhalla_image=get("VALHALLA_IMAGE") or DEFAULT_VALHALLA_IMAGE, + valhalla_version=version, + valinor_cli=valinor_cli, + serve_port=serve_port, + ) diff --git a/scripts/integration/src/valinor_integration/valhalla.py b/scripts/integration/src/valinor_integration/valhalla.py new file mode 100644 index 0000000..4f8a7ff --- /dev/null +++ b/scripts/integration/src/valinor_integration/valhalla.py @@ -0,0 +1,253 @@ +"""Runs stock C++ Valhalla in Docker (with docker-py) against a tile extract.""" + +from __future__ import annotations + +import io +import json +import logging +import subprocess +import tarfile +import tempfile +import time +from pathlib import Path +from types import TracebackType +from typing import IO, Any + +import docker +import docker.errors +import requests +from docker.models.containers import Container + +from valinor_integration.errors import HarnessError +from valinor_integration.settings import Settings + +log = logging.getLogger(__name__) + +#: The directory in the container that holds the extract and the config. +CONTAINER_DATA_DIR = "/valinor" +CONTAINER_EXTRACT = f"{CONTAINER_DATA_DIR}/tiles.tar" +CONTAINER_CONFIG = f"{CONTAINER_DATA_DIR}/valhalla.json" +#: The port of valhalla_service in the container. +SERVICE_PORT = 8002 + +READINESS_TIMEOUT_SECONDS = 90 +REQUEST_TIMEOUT_SECONDS = 30 + + +def connect() -> docker.DockerClient: + """Connects to the Docker daemon. + + If the default connection fails, this uses the endpoint of the current Docker CLI context, + which docker-py does not read (for example, the socket of colima). + """ + try: + client = docker.from_env() + client.ping() + return client + except docker.errors.DockerException as default_error: + try: + endpoint = subprocess.run( + ["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + client = docker.DockerClient(base_url=endpoint) + client.ping() + log.info("connected to Docker through the CLI context endpoint %s", endpoint) + return client + except (OSError, subprocess.CalledProcessError, docker.errors.DockerException): + raise HarnessError( + f"cannot connect to Docker ({default_error}); is the Docker daemon running?" + ) from None + + +def ensure_image(client: docker.DockerClient, settings: Settings) -> None: + """Pulls ``latest`` every time, and a pinned tag only if it is not present.""" + if not settings.is_pinned: + log.warning( + "VALHALLA_VERSION is not set, so the harness uses %s; " + "set VALHALLA_VERSION to pin a release", + settings.image_ref, + ) + else: + try: + client.images.get(settings.image_ref) + return + except docker.errors.ImageNotFound: + pass + + log.info("pulling %s", settings.image_ref) + try: + client.images.pull(settings.valhalla_image, tag=settings.image_tag) + except docker.errors.APIError as e: + raise HarnessError(f"cannot pull {settings.image_ref}: {e}") from None + + +def build_config(client: docker.DockerClient, image: str) -> dict[str, Any]: + """Makes a config for the extract at ``CONTAINER_EXTRACT`` with the image's own tool.""" + try: + output = client.containers.run( + image, + entrypoint="valhalla_build_config", + command=["--mjolnir-tile-extract", CONTAINER_EXTRACT], + remove=True, + stdout=True, + stderr=False, + ) + except docker.errors.DockerException as e: + raise HarnessError(f"valhalla_build_config failed: {e}") from None + config = json.loads(output) + + mjolnir = config["mjolnir"] + mjolnir["tile_extract"] = CONTAINER_EXTRACT + # Do not enable shortcut_caching: Valhalla then asserts at startup that every + # level 0 and level 1 tile exists, so a partial extract fails to start. + mjolnir["shortcut_caching"] = False + return config + + +class ValhallaServer: + """A ``valhalla_service`` container which serves one extract. + + Use it as a context manager: the container starts on entry, + and is removed on exit (also after an error or Ctrl-C). + Call ``logs`` before the exit to keep the logs. + """ + + def __init__( + self, + client: docker.DockerClient, + image: str, + name: str, + extract: Path, + config: dict[str, Any], + host_port: int | None = None, + ) -> None: + """``host_port`` ``None`` lets Docker choose a free port.""" + self.client = client + self.image = image + self.name = name + self.extract = extract + self.config = config + self.host_port = host_port + self.container_name = f"valinor-integration-{name}" + self._container: Container | None = None + self.base_url = "" + + def __enter__(self) -> ValhallaServer: + try: + self._start() + except BaseException: + self._remove() + raise + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + traceback: TracebackType | None, + ) -> None: + self._remove() + + def _start(self) -> None: + self._remove_stale_container() + log.info("starting valhalla_service for the %s extract", self.name) + self._container = self.client.containers.create( + self.image, + name=self.container_name, + entrypoint="valhalla_service", + command=[CONTAINER_CONFIG, "1"], + ports={f"{SERVICE_PORT}/tcp": self.host_port}, + ) + # Copy the data in place of a bind mount, which fails for a directory + # that the Docker VM does not share (such as /tmp with colima). + with tempfile.TemporaryFile() as archive: + self._write_data_archive(archive) + archive.seek(0) + if not self._container.put_archive("/", archive): + raise HarnessError(f"cannot copy the {self.name} extract into the container") + self._container.start() + + self._container.reload() + bindings = self._container.ports.get(f"{SERVICE_PORT}/tcp") or [] + if not bindings: + raise HarnessError(f"Docker did not publish the port of the {self.name} container") + self.base_url = f"http://localhost:{bindings[0]['HostPort']}" + self._wait_until_ready() + + def _write_data_archive(self, archive: IO[bytes]) -> None: + """Writes a tar archive with the extract and the config, in ``CONTAINER_DATA_DIR``.""" + with tarfile.open(fileobj=archive, mode="w") as tar: + directory = tarfile.TarInfo(CONTAINER_DATA_DIR.lstrip("/")) + directory.type = tarfile.DIRTYPE + directory.mode = 0o755 + tar.addfile(directory) + + tar.add(self.extract, arcname=CONTAINER_EXTRACT.lstrip("/"), filter=_readable) + + config_bytes = json.dumps(self.config, indent=2).encode("utf-8") + config_info = tarfile.TarInfo(CONTAINER_CONFIG.lstrip("/")) + config_info.size = len(config_bytes) + config_info.mode = 0o644 + tar.addfile(config_info, io.BytesIO(config_bytes)) + + def _wait_until_ready(self) -> None: + assert self._container is not None + deadline = time.monotonic() + READINESS_TIMEOUT_SECONDS + while time.monotonic() < deadline: + try: + status = requests.get(f"{self.base_url}/status", timeout=2) + if status.ok: + version = status.json().get("version", "unknown") + log.info( + "the %s extract is served at %s (Valhalla %s)", + self.name, + self.base_url, + version, + ) + return + except (requests.RequestException, ValueError): + pass + self._container.reload() + if self._container.status not in ("created", "running"): + break + time.sleep(0.5) + + raise HarnessError( + f"valhalla_service did not become ready for the {self.name} extract; " + f"logs:\n{self.logs()}" + ) + + def logs(self) -> str: + if self._container is None: + return "" + return self._container.logs().decode("utf-8", errors="replace") + + def _remove_stale_container(self) -> None: + """Removes a container with the same name, left by an earlier run that crashed.""" + try: + self.client.containers.get(self.container_name).remove(force=True) + log.info("removed the stale container %s", self.container_name) + except docker.errors.NotFound: + pass + + def _remove(self) -> None: + if self._container is None: + return + try: + self._container.remove(force=True) + except docker.errors.NotFound: + pass + except docker.errors.APIError as e: + log.warning("cannot remove the container %s: %s", self.container_name, e) + self._container = None + + +def _readable(info: tarfile.TarInfo) -> tarfile.TarInfo: + """Makes a file in the data archive readable by any user of the container.""" + info.mode = 0o644 + info.uid = info.gid = 0 + info.uname = info.gname = "" + return info diff --git a/scripts/integration/sub-extract-tiles.txt b/scripts/integration/sub-extract-tiles.txt new file mode 100644 index 0000000..2cdfe7a --- /dev/null +++ b/scripts/integration/sub-extract-tiles.txt @@ -0,0 +1,14 @@ +# The tiles of the integration sub-extract, one level/tile_id per line. +# valinor-integration gives these to `valinor-cli extract --tile`. +# +# The list contains every tile of the Andorra fixture except the north-east corner +# (2/763927: longitude 1.75 to 2.0, latitude 42.5 to 42.75), +# so the sub-extract is a proper subset of the fixture. +# All coordinates in the Bruno collection (bruno/) are far from the removed tile. +# If you change this list, make sure that the requests stay inside the remaining tiles. +0/3015 +1/47701 +2/762485 +2/762486 +2/763925 +2/763926 diff --git a/scripts/integration/tests/test_checks.py b/scripts/integration/tests/test_checks.py new file mode 100644 index 0000000..39da2c5 --- /dev/null +++ b/scripts/integration/tests/test_checks.py @@ -0,0 +1,71 @@ +import copy + +from valinor_integration.checks import CHECKS, diff_projections + +ROUTE = { + "trip": { + "status": 0, + "units": "kilometers", + "locations": [{"lat": 1.0, "lon": 2.0}], + "summary": {"length": 1.5, "time": 90.0, "min_lat": 1.0}, + "legs": [ + { + "shape": "abc", + "summary": {"length": 1.5, "time": 90.0}, + "maneuvers": [{"type": 1, "instruction": "Go.", "length": 1.5, "extra": "x"}], + } + ], + } +} + +LOCATE = [ + { + "edges": [ + {"edge_id": {"value": 2}, "edge_info": {"way_id": 7}, "percent_along": 0.5}, + {"edge_id": {"value": 1}, "edge_info": {"way_id": 7}, "percent_along": 0.0}, + ] + } +] + + +def test_route_projection_ignores_excluded_fields() -> None: + other = copy.deepcopy(ROUTE) + other["trip"]["locations"] = [] + other["trip"]["summary"]["min_lat"] = 9.0 + other["trip"]["legs"][0]["maneuvers"][0]["extra"] = "y" + project = CHECKS["/route"].project + assert project(ROUTE) == project(other) + + +def test_route_projection_finds_differences() -> None: + other = copy.deepcopy(ROUTE) + other["trip"]["legs"][0]["shape"] = "abd" + project = CHECKS["/route"].project + diff = diff_projections("route/x", project(ROUTE), project(other), "full", "sub") + assert "--- full/route/x" in diff + assert '"shape": "abd"' in diff + + +def test_route_validation() -> None: + validate = CHECKS["/route"].validate + assert validate(ROUTE) == [] + assert validate({"trip": {"legs": []}}) != [] + assert validate([]) != [] + + +def test_locate_projection_is_order_independent() -> None: + reordered = [{"edges": list(reversed(LOCATE[0]["edges"]))}] + project = CHECKS["/locate"].project + assert project(LOCATE) == project(reordered) + assert project(LOCATE)[0][0]["way_id"] == 7 + + +def test_locate_validation_rejects_empty_results() -> None: + validate = CHECKS["/locate"].validate + assert validate(LOCATE) == [] + assert validate([{"edges": []}]) != [] + assert validate([]) != [] + + +def test_equal_projections_have_no_diff() -> None: + assert diff_projections("x", {"a": 1}, {"a": 1}, "full", "sub") == "" diff --git a/scripts/integration/tests/test_cli.py b/scripts/integration/tests/test_cli.py new file mode 100644 index 0000000..98b5f8f --- /dev/null +++ b/scripts/integration/tests/test_cli.py @@ -0,0 +1,36 @@ +from pathlib import Path + +import pytest + +from valinor_integration import cli, runner + + +@pytest.fixture +def calls(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, runner.Inputs]]: + recorded: list[tuple[str, runner.Inputs]] = [] + monkeypatch.setattr( + runner, "run", lambda settings, inputs: recorded.append(("run", inputs)) or True + ) + monkeypatch.setattr( + runner, "serve", lambda settings, inputs: recorded.append(("serve", inputs)) + ) + monkeypatch.delenv("VALINOR_CLI", raising=False) + return recorded + + +def test_run_is_the_default_command(calls: list[tuple[str, runner.Inputs]]) -> None: + assert cli.main([]) == 0 + assert cli.main(["--workdir", "somewhere"]) == 0 + assert [command for command, _ in calls] == ["run", "run"] + assert calls[1][1].workdir == Path("somewhere").resolve() + + +def test_serve_keeps_its_options(calls: list[tuple[str, runner.Inputs]]) -> None: + assert cli.main(["serve", "--workdir", "somewhere"]) == 0 + assert calls == [("serve", calls[0][1])] + assert calls[0][1].workdir == Path("somewhere").resolve() + + +def test_failed_run_exits_with_one(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(runner, "run", lambda settings, inputs: False) + assert cli.main([]) == 1 diff --git a/scripts/integration/tests/test_collection.py b/scripts/integration/tests/test_collection.py new file mode 100644 index 0000000..bcaff1c --- /dev/null +++ b/scripts/integration/tests/test_collection.py @@ -0,0 +1,95 @@ +from pathlib import Path + +import pytest + +from valinor_integration.collection import load_collection, parse_request +from valinor_integration.errors import HarnessError +from valinor_integration.settings import DEFAULT_COLLECTION + +REQUEST = """\ +info: + name: example + type: http + seq: 2 + +http: + method: post + url: "{{baseUrl}}/route" + headers: + - name: X-Enabled + value: "yes" + - name: X-Disabled + value: "no" + disabled: true + body: + type: json + data: |- + {"costing": "auto"} + auth: inherit + +settings: + encodeUrl: true +""" + + +def test_parse_request() -> None: + request = parse_request(Path("route/example.yml"), REQUEST) + assert request.name == "example" + assert request.id == "route/example" + assert request.seq == 2 + assert request.method == "POST" + assert request.headers == {"X-Enabled": "yes"} + assert request.body == {"costing": "auto"} + assert request.resolve_url({"baseUrl": "http://localhost:1"}) == "http://localhost:1/route" + assert request.endpoint({"baseUrl": "http://localhost:1"}) == "/route" + assert request.endpoint({"baseUrl": ""}) == "/route" + + +def test_unknown_variable_is_an_error() -> None: + request = parse_request(Path("route/example.yml"), REQUEST) + with pytest.raises(HarnessError, match="unknown variable"): + request.resolve_url({}) + + +@pytest.mark.parametrize( + ("replace", "by", "message"), + [ + ("settings:", "runtime:\n scripts: []\nsettings:", "unsupported sections"), + (" auth: inherit", " params: []", "unsupported http fields"), + (" auth: inherit", " auth:\n type: bearer", "authentication"), + (" type: json", " type: xml", "only JSON bodies"), + ('{"costing": "auto"}', '{"costing": ', "not valid JSON"), + (" type: http", " type: graphql", "unsupported request type"), + (" name: example", " name: ''", "info.name"), + ], +) +def test_unsupported_requests_are_rejected(replace: str, by: str, message: str) -> None: + assert replace in REQUEST + with pytest.raises(HarnessError, match=message): + parse_request(Path("route/example.yml"), REQUEST.replace(replace, by, 1)) + + +def test_load_collection_order_and_skipped_files(tmp_path: Path) -> None: + (tmp_path / "opencollection.yml").write_text("opencollection: 1.0.0\n") + (tmp_path / "environments").mkdir() + (tmp_path / "environments" / "local.yml").write_text("name: local\n") + (tmp_path / "b").mkdir() + (tmp_path / "b" / "folder.yml").write_text("info:\n type: folder\n") + (tmp_path / "b" / "second.yml").write_text(REQUEST.replace("name: example", "name: second")) + (tmp_path / "b" / "first.yml").write_text( + REQUEST.replace("name: example", "name: first").replace("seq: 2", "seq: 1") + ) + + assert [request.id for request in load_collection(tmp_path)] == ["b/first", "b/second"] + + +def test_load_collection_requires_a_collection_file(tmp_path: Path) -> None: + with pytest.raises(HarnessError, match="not a Bruno collection"): + load_collection(tmp_path) + + +def test_the_repository_collection_loads() -> None: + requests = load_collection(DEFAULT_COLLECTION) + endpoints = {request.endpoint({"baseUrl": ""}) for request in requests} + assert endpoints == {"/route", "/locate"} + assert all(request.body for request in requests) diff --git a/scripts/integration/tests/test_extract.py b/scripts/integration/tests/test_extract.py new file mode 100644 index 0000000..e539262 --- /dev/null +++ b/scripts/integration/tests/test_extract.py @@ -0,0 +1,52 @@ +import tarfile +from pathlib import Path + +import pytest + +from valinor_integration.errors import HarnessError +from valinor_integration.extract import check_proper_subset, read_tile_list +from valinor_integration.settings import DEFAULT_TILE_LIST + + +def _archive(path: Path, names: list[str]) -> Path: + with tarfile.open(path, "w") as tar: + for name in names: + tar.addfile(tarfile.TarInfo(name)) + return path + + +def test_read_tile_list(tmp_path: Path) -> None: + tile_list = tmp_path / "tiles.txt" + tile_list.write_text("# comment\n\n0/3015\n 2 / 762485 # inline comment\n") + assert read_tile_list(tile_list) == ["0/3015", "2/762485"] + + +def test_empty_tile_list_is_an_error(tmp_path: Path) -> None: + tile_list = tmp_path / "tiles.txt" + tile_list.write_text("# only a comment\n") + with pytest.raises(HarnessError, match="no tiles"): + read_tile_list(tile_list) + + +def test_the_repository_tile_list_reads() -> None: + assert len(read_tile_list(DEFAULT_TILE_LIST)) == 6 + + +def test_proper_subset(tmp_path: Path) -> None: + full = _archive(tmp_path / "full.tar", ["index.bin", "0/a.gph", "2/b.gph"]) + sub = _archive(tmp_path / "sub.tar", ["index.bin", "0/a.gph"]) + assert check_proper_subset(full, sub) == ["2/b.gph"] + + +def test_identical_extracts_prove_nothing(tmp_path: Path) -> None: + full = _archive(tmp_path / "full.tar", ["index.bin", "0/a.gph"]) + sub = _archive(tmp_path / "sub.tar", ["index.bin", "0/a.gph"]) + with pytest.raises(HarnessError, match="would prove nothing"): + check_proper_subset(full, sub) + + +def test_extra_entries_are_an_error(tmp_path: Path) -> None: + full = _archive(tmp_path / "full.tar", ["index.bin", "0/a.gph"]) + sub = _archive(tmp_path / "sub.tar", ["index.bin", "1/c.gph"]) + with pytest.raises(HarnessError, match="not in the full extract"): + check_proper_subset(full, sub) diff --git a/scripts/integration/tests/test_settings.py b/scripts/integration/tests/test_settings.py new file mode 100644 index 0000000..06b444d --- /dev/null +++ b/scripts/integration/tests/test_settings.py @@ -0,0 +1,56 @@ +from pathlib import Path + +import pytest + +from valinor_integration.errors import HarnessError +from valinor_integration.settings import DEFAULT_COLLECTION, DEFAULT_FULL_EXTRACT, Settings + + +def test_defaults_use_latest() -> None: + settings = Settings.from_env({}) + assert settings.image_ref == "ghcr.io/valhalla/valhalla:latest" + assert not settings.is_pinned + assert settings.valinor_cli is None + assert settings.serve_port == 8002 + + +def test_empty_values_are_unset() -> None: + assert Settings.from_env({"VALHALLA_VERSION": " ", "VALINOR_CLI": ""}) == Settings.from_env({}) + + +def test_pinned_version_and_image(tmp_path: Path) -> None: + cli = tmp_path / "valinor-cli" + cli.touch() + settings = Settings.from_env( + { + "VALHALLA_VERSION": "3.8.2", + "VALHALLA_IMAGE": "example.com/valhalla", + "VALINOR_CLI": str(cli), + "VALHALLA_PORT": "9000", + } + ) + assert settings.image_ref == "example.com/valhalla:3.8.2" + assert settings.is_pinned + assert settings.valinor_cli == cli + assert settings.serve_port == 9000 + + +@pytest.mark.parametrize( + "env", + [ + {"VALHALLA_VERSION": "ghcr.io/valhalla/valhalla:3.8.2"}, + {"VALHALLA_VERSION": "3.8.2@sha256:abc"}, + {"VALHALLA_PORT": "http"}, + {"VALHALLA_PORT": "65535"}, + {"VALHALLA_PORT": "0"}, + {"VALINOR_CLI": "/does/not/exist"}, + ], +) +def test_invalid_values(env: dict[str, str]) -> None: + with pytest.raises(HarnessError): + Settings.from_env(env) + + +def test_default_paths_exist() -> None: + assert DEFAULT_FULL_EXTRACT.is_file() + assert DEFAULT_COLLECTION.is_dir() diff --git a/scripts/integration/uv.lock b/scripts/integration/uv.lock new file mode 100644 index 0000000..bab11d4 --- /dev/null +++ b/scripts/integration/uv.lock @@ -0,0 +1,370 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" + +[[package]] +name = "certifi" +version = "2026.10.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/33/1c/f41d4e74c28ab327ff3acd36053f7ea506c55872d7a90b0fa71aa3ab0c89/charset_normalizer-3.5.2.tar.gz", hash = "sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef", size = 172659, upload-time = "2026-09-30T04:39:23.398Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/c8/693809898870237d82785a03f3b2b58fe4c9f14669f84a7d4e623c92a59e/charset_normalizer-3.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491", size = 367780, upload-time = "2026-09-30T04:35:30.888Z" }, + { url = "https://files.pythonhosted.org/packages/c9/87/2fea8c13dc24b3ca9c6f803a5b2dfdeae73eb4f9e12c7885ed908ff0433c/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c", size = 246730, upload-time = "2026-09-30T04:35:32.286Z" }, + { url = "https://files.pythonhosted.org/packages/a8/9e/09efac30b937722f46d3110ba30b875b24b2e3a266ed746cc4e376a94d80/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0", size = 237707, upload-time = "2026-09-30T04:35:33.709Z" }, + { url = "https://files.pythonhosted.org/packages/9e/18/70d76670b13686237863a379928d60bd10e021f17d243ab3d7014c4a5f4e/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51", size = 273050, upload-time = "2026-09-30T04:35:35.138Z" }, + { url = "https://files.pythonhosted.org/packages/54/e2/77a8b09d5adc013ed07b95b01b8b8fa5441c4e810e83ee7e4aae2fa4d91a/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5", size = 270345, upload-time = "2026-09-30T04:35:36.502Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c5/38806a25ab5e65fc178f39affeda20858efafede2fce1ffc2556cfc9fe73/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649", size = 257601, upload-time = "2026-09-30T04:35:37.919Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8d/213565184708fdb263ae55e2c04ee1ff748129dd65d48ed0e3502da9c85a/charset_normalizer-3.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e", size = 252222, upload-time = "2026-09-30T04:35:39.544Z" }, + { url = "https://files.pythonhosted.org/packages/7e/24/76d2cefc25472531e4c5c7dfff68865eb1c39b78482f0fdc15b46f047830/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346", size = 248482, upload-time = "2026-09-30T04:35:41.088Z" }, + { url = "https://files.pythonhosted.org/packages/7d/dc/65a801b66ab4c197e22c433ab25e7ac24324ac6f45a2269aca42cce309bf/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1", size = 241206, upload-time = "2026-09-30T04:35:42.59Z" }, + { url = "https://files.pythonhosted.org/packages/a7/95/ca9b5eabde673002c6f1e7ada1b223916fe18f6d661da7aabd4d643718f1/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875", size = 273190, upload-time = "2026-09-30T04:35:44.347Z" }, + { url = "https://files.pythonhosted.org/packages/2d/8b/803b4d2a3f6e1740f63f1e87b04d14b42f3d4fdfe6ed7d4db2d34102b14f/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1", size = 253527, upload-time = "2026-09-30T04:35:45.915Z" }, + { url = "https://files.pythonhosted.org/packages/a9/55/93c0e5dbd085ae0471346026abbe7e0db9ea2d6fea74e51f0b5a46f233a7/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413", size = 271285, upload-time = "2026-09-30T04:35:47.49Z" }, + { url = "https://files.pythonhosted.org/packages/95/69/0dbd0e0b9b16cfa816cdfcb3e2e3854a1f680dc07fb1245ea125e7448060/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869", size = 260010, upload-time = "2026-09-30T04:35:48.996Z" }, + { url = "https://files.pythonhosted.org/packages/58/9d/e7b88e7b1bf403590c3b573277b5e1e488c68c7a6fbacca310a2c324e90c/charset_normalizer-3.5.2-cp312-cp312-win32.whl", hash = "sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e", size = 184125, upload-time = "2026-09-30T04:35:50.777Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e6/e6e083884cbcfd49c64865af05027fe7011be7b2d9179524f099a1b611f3/charset_normalizer-3.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc", size = 207486, upload-time = "2026-09-30T04:35:52.194Z" }, + { url = "https://files.pythonhosted.org/packages/c4/e3/017aea0911ada7405a825c7d937eb3a13009664e2f5b38e8c4bbf2abf894/charset_normalizer-3.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3", size = 196734, upload-time = "2026-09-30T04:35:53.636Z" }, + { url = "https://files.pythonhosted.org/packages/c5/34/68292d68512768591aaff07c59bb53ee31341c87759433a859c4641a50c2/charset_normalizer-3.5.2-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5", size = 225946, upload-time = "2026-09-30T04:35:55.313Z" }, + { url = "https://files.pythonhosted.org/packages/e3/80/bee0b01b90ccd5322ae1d0abb33fab1bd95b7c2eadaf02aeccf22e04ee83/charset_normalizer-3.5.2-cp313-cp313-android_24_x86_64.whl", hash = "sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e", size = 238619, upload-time = "2026-09-30T04:35:56.863Z" }, + { url = "https://files.pythonhosted.org/packages/78/6e/60ce52a85a7fd631ae8482ae6d74521014ca2f255892679484dc04d7ef56/charset_normalizer-3.5.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a", size = 206701, upload-time = "2026-09-30T04:35:58.639Z" }, + { url = "https://files.pythonhosted.org/packages/36/8c/71aafad23f971afc84c2b295bc0c560739ce1dac558aad9fec22e39f3639/charset_normalizer-3.5.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:d376bbd28b3a8999db1a103b3b388aee6f1ddeb3e51bc2172993efdcd86e064d", size = 210506, upload-time = "2026-09-30T04:36:00.147Z" }, + { url = "https://files.pythonhosted.org/packages/91/da/3c5a7798c046df7d2d68ad653cf5b6c5a8bfee225055a843c6f2f42aac1a/charset_normalizer-3.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6045373d5a89a5ec71afde535db987ca28e76dfa276c2d4c818265b375d4b055", size = 366626, upload-time = "2026-09-30T04:36:01.77Z" }, + { url = "https://files.pythonhosted.org/packages/e1/16/710ac3de2ee354e2bd1a9c94efe45a2d27b5c6ad39b2d6a905be2c094b6c/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:849df64e889b2e17230d58410a03dba311a65b163508fd33679b2b737d4b7858", size = 244624, upload-time = "2026-09-30T04:36:03.389Z" }, + { url = "https://files.pythonhosted.org/packages/d6/39/45c7439f5b63d24f7d5b2a1d760f34af7628782d7144b4cc8ded45c2d4bc/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:15c44f7edfd477b06f517a5cc317fc1707edb9de2c865f43d4b6513907473234", size = 235731, upload-time = "2026-09-30T04:36:04.987Z" }, + { url = "https://files.pythonhosted.org/packages/4d/34/38f3154785ce92e9f56eb226f4d35bdfae6b008480dd055f58837a89c810/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a89012d6d5476ee112d20d998570ed58df2260a852afb1758809cd6900411d21", size = 270352, upload-time = "2026-09-30T04:36:06.412Z" }, + { url = "https://files.pythonhosted.org/packages/04/f3/859f74e7babc977705026b30593b3be04049632a522fb7000f83c033d747/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0c951d5e6dd9c2ff60609476752bee49da4206adde960ebc247766937f72e718", size = 267393, upload-time = "2026-09-30T04:36:07.865Z" }, + { url = "https://files.pythonhosted.org/packages/4b/85/41d27f234b82e47c167a5f6c0f62501dc0c640585ff4aba79e08a390336a/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4", size = 254660, upload-time = "2026-09-30T04:36:09.248Z" }, + { url = "https://files.pythonhosted.org/packages/58/ca/5d1a997587febe5b26d8daffe363b5c1a091cece19828eec6502fd09c5ef/charset_normalizer-3.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a19a731138fc27d5682277d3b9df22855cea1239bce7fcec5f78f42ef2d1f3c3", size = 249996, upload-time = "2026-09-30T04:36:10.73Z" }, + { url = "https://files.pythonhosted.org/packages/b3/1f/d1e78246f7ed60c8c8d606b4ac27f66ce49cc3e95f24893ccbeba9f77302/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:62603db9a7caa0802eaa28c1c46fecd7b3a263a774069c24c3c28c302448721c", size = 247028, upload-time = "2026-09-30T04:36:12.294Z" }, + { url = "https://files.pythonhosted.org/packages/8e/37/eba316edd4f0c4d3a5d945924c4eeeae59abac4056aa815d8a4268f863a2/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:b6856554c4f44d79fc2307d5768854310a8f0096e501c75637542c82292b0429", size = 240562, upload-time = "2026-09-30T04:36:13.887Z" }, + { url = "https://files.pythonhosted.org/packages/c8/8e/aaa037d40ca9ef045977f1a661048b1aa33f223adfce3452fe9be9f79d14/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:1bc0baf5ef96b6ede57d47f4b8fe4d9d84019c3bfcbeb20a41edc6a6ee341f1f", size = 271699, upload-time = "2026-09-30T04:36:15.41Z" }, + { url = "https://files.pythonhosted.org/packages/26/19/1c1c9f75974adf523b87f34b8a2adc5a435cd65916812bcbd0dfa45f9a29/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:56bc200a365efb37383b7852e4cc5898d3b2da5987289b543956cf8cad71018a", size = 252030, upload-time = "2026-09-30T04:36:16.839Z" }, + { url = "https://files.pythonhosted.org/packages/bc/90/0660ef18e18df0a4d2a1a0edff7dfbba42d4e50ef2425557a5bb7051f77b/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:2c9ad19a6cfcd5ea5c0d41161d22f9df1dcc277e9bef2751391334546a314c00", size = 268565, upload-time = "2026-09-30T04:36:18.468Z" }, + { url = "https://files.pythonhosted.org/packages/79/ba/57adc269824e8658f1a0f97a9e514c247445a9632b3419b97e0ba37f16dc/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e243bd13217235fc7290c621941c3f5cc8b66e4872495be821d7436ba2fb838d", size = 257627, upload-time = "2026-09-30T04:36:19.938Z" }, + { url = "https://files.pythonhosted.org/packages/9a/85/33abd4315c052d3d4f54c92b1ee49bfbc0dc7115a981e462a793b6d2ab87/charset_normalizer-3.5.2-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:a090bb2c68df85450502e3e20d665e3a5af9c65a84d6508ed477badd49166fd3", size = 143407, upload-time = "2026-09-30T04:36:21.376Z" }, + { url = "https://files.pythonhosted.org/packages/4f/de/6435e18d1aaa5d910b896d551411c96af1f42a0c56c29afc2016c61ccc2e/charset_normalizer-3.5.2-cp313-cp313-win32.whl", hash = "sha256:2b7b3bbfb4fe8ef40600792d762fbaa9057559f9d3fad209525b7a22b99e91fd", size = 183207, upload-time = "2026-09-30T04:36:22.776Z" }, + { url = "https://files.pythonhosted.org/packages/9c/76/b8ec57f4e9ee3253541abf95e4a462c0175fe8032dcd070f1f2421240942/charset_normalizer-3.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:78456a747de8dc58360ffa581f30a002baf5aa28cb262536545e91f113ed7639", size = 206489, upload-time = "2026-09-30T04:36:24.306Z" }, + { url = "https://files.pythonhosted.org/packages/3e/60/c647c6ae47480221e875ea5d743ff94946f7416e3c69415ab772928e8d32/charset_normalizer-3.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:11912e4bb14baae7c5d8791aa55ba0a3a03ec6729073307b0f57270abaa713d3", size = 195501, upload-time = "2026-09-30T04:36:25.846Z" }, + { url = "https://files.pythonhosted.org/packages/58/ca/7aa91362a2f77ac8e9e28a9b902a74f7d0e11a851ef0d27a74308da8cd90/charset_normalizer-3.5.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:1afb975bd5d68d5ce9f6b6d44fdf2f7e34b895a35e95708a7a91b20a3b51d187", size = 226010, upload-time = "2026-09-30T04:36:27.669Z" }, + { url = "https://files.pythonhosted.org/packages/a8/cf/ac8878d0322cf88a1aad4c7b147db32ca0bd806eb0060957b2e31486dbe6/charset_normalizer-3.5.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:bbbfc8e28816f19d7c0f1816664980c0a9875d01b27cdf8eedddb639d9e108ad", size = 239266, upload-time = "2026-09-30T04:36:29.434Z" }, + { url = "https://files.pythonhosted.org/packages/c9/6d/9a08d7e0b29b7208e2c6c01dc56c8e0520e7c7beadbbfb024b58fd69c8a5/charset_normalizer-3.5.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7967d08cf06dee78443b874f98c98036f624f3a4e73e11f9f64f5be4d25393cf", size = 206783, upload-time = "2026-09-30T04:36:30.872Z" }, + { url = "https://files.pythonhosted.org/packages/82/44/b0aa350280e6ff5a5492d17cf10460dd39d5ee848f872f7ba2df10607f60/charset_normalizer-3.5.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4c2b5031f63e331e3839b40aed2dd6f191e9c07edbde303e7876846ea1946995", size = 210449, upload-time = "2026-09-30T04:36:32.625Z" }, + { url = "https://files.pythonhosted.org/packages/7c/8a/40db9aa9f5907bb0e6f8b6d64064bf8852fb33d4b813ff9414911df7647c/charset_normalizer-3.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424", size = 367815, upload-time = "2026-09-30T04:36:34.197Z" }, + { url = "https://files.pythonhosted.org/packages/7f/72/9c5e7707b57c8ddfa9ddf7b0b1d009d7fbab9e9e887d5b721060f37e307d/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8d86d6fc60743dc916eb79e2eb1ec4818e21e427731543af40a3021851174a13", size = 247095, upload-time = "2026-09-30T04:36:35.803Z" }, + { url = "https://files.pythonhosted.org/packages/83/09/71e453691e927de4ddf792770cfaab3f49d494e222f66ea5e404bbd5e39c/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7a881931aa470808df94a8c380eed2bbbc76cd9dc622310f99665658c821eb6d", size = 233942, upload-time = "2026-09-30T04:36:37.407Z" }, + { url = "https://files.pythonhosted.org/packages/9f/86/85c84e4da8b27dd409577d9437926ff581c5f9d3c66038dc68c1a526de51/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8024d00c3faf3fc0c16e07a69f4405e8eac7cc0ab15f65fe6cf43827c4cf72b4", size = 273762, upload-time = "2026-09-30T04:36:38.904Z" }, + { url = "https://files.pythonhosted.org/packages/92/08/564955a4b5f2ccb410ab480bbe8c6a18063ff27f2d35458731c4a5335df9/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4d48f2d08b9de5864e2c8744d4461b862fb149a18274abc8b698c45975573438", size = 269514, upload-time = "2026-09-30T04:36:40.469Z" }, + { url = "https://files.pythonhosted.org/packages/18/24/bad3ac4271589df29cf5ce2f5ae490518a5739358052bd0d61209e6fea54/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34276fd796040bf0993ab33a369aa572e6979c7aab225a88893667ad8eac8f7a", size = 255056, upload-time = "2026-09-30T04:36:42.02Z" }, + { url = "https://files.pythonhosted.org/packages/d6/3e/350d89ad49916b86554d6f5f2d03ec1152148f87e5ff735106c6a03b1a36/charset_normalizer-3.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0521c5665880b33d603717defa76c094048900010897909952397feb3039da56", size = 251311, upload-time = "2026-09-30T04:36:43.577Z" }, + { url = "https://files.pythonhosted.org/packages/56/5b/4970a2d154df502e133402906dd04e3ae7cada7b3011283c88d0479a2585/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:eff0ac9dbe711a4aee69bf04a83896aa9b85f19641264053a9f6d48573abb7dd", size = 250190, upload-time = "2026-09-30T04:36:45.185Z" }, + { url = "https://files.pythonhosted.org/packages/88/8c/f1a91bddc8fb47c2889e29ea7ea49a194eb0d9868675d786806519c00d76/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1503bccbeb36d5527790c3930327704c39af22de3112f1b1666a9f3ce15ee204", size = 237098, upload-time = "2026-09-30T04:36:46.689Z" }, + { url = "https://files.pythonhosted.org/packages/24/0e/bb5dace3cc7e79068425386a6589c19b5a2ab5fefc2a46abea6919683332/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:52aa6992700996af31f375de0c6bacd402b0097fe40b53c426b9f51a90ebabc7", size = 274228, upload-time = "2026-09-30T04:36:48.31Z" }, + { url = "https://files.pythonhosted.org/packages/9d/79/b849ad523017ea9f5a45581bbebed91439e0cf42fd2860a6f64e358eb5a6/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e09a3942ecbdee5cce73ea9d42da82b81b72ac1bf031ce069b93b5adf4eac8cd", size = 252401, upload-time = "2026-09-30T04:36:50.091Z" }, + { url = "https://files.pythonhosted.org/packages/89/8c/75469d690cf47200bce8f6cad7655724fc23148e147abfc5ce78b5f65863/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:c7c9ab723cde841fefb34efbad91e87f00a674b1fe1cd0784fde742bf2c154dc", size = 271846, upload-time = "2026-09-30T04:36:51.719Z" }, + { url = "https://files.pythonhosted.org/packages/26/cd/6d52d3c7437cdcf2e310ce9f28f282e733d4ef60ed19105d1819c356255f/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ddc7dacc8ece3a182e7f15cb862d1fd616b46d076cb1ae9dd232b2c38b655874", size = 258151, upload-time = "2026-09-30T04:36:53.234Z" }, + { url = "https://files.pythonhosted.org/packages/f7/4c/070b38bdb5f49a70199fce923ec0726a49536a63ab262abbfcaaf351110b/charset_normalizer-3.5.2-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:ee43c17b173d46a3212baa6ead3ae258eeabdae48c263a01ccf0218c366dd655", size = 143570, upload-time = "2026-09-30T04:36:54.816Z" }, + { url = "https://files.pythonhosted.org/packages/81/84/9ebfc8ed6c8c4fcd8e726ff6bf220cc8deb3966e31dce9be8dd8aa017e64/charset_normalizer-3.5.2-cp314-cp314-win32.whl", hash = "sha256:4f87960d57feabfb618e4e0af6e7371645fa26a277860739d6e5d6e0012c92f0", size = 186412, upload-time = "2026-09-30T04:36:56.643Z" }, + { url = "https://files.pythonhosted.org/packages/d1/78/5ed86f743d4bc350db307e7636419a0a5ee1d91806d30c7f667bd5c80dae/charset_normalizer-3.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:e4e81e09c1578b8df602e3db08b0b3ea0a6947ad612f52bf8dc5ea8d47691f0c", size = 210965, upload-time = "2026-09-30T04:36:58.205Z" }, + { url = "https://files.pythonhosted.org/packages/53/94/a3a7698e9b1a395e1eb99ccd9a324be9347973bff4e72db2a06496d7cd27/charset_normalizer-3.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:80d02b6f04e92601a081dd97b23d3128033098bff5d35d392ddcc0476ea11253", size = 200406, upload-time = "2026-09-30T04:36:59.764Z" }, + { url = "https://files.pythonhosted.org/packages/c1/48/c5dd00d5ef7791f02666de250a5bb6071e29b7e133cf4b835800b6d3bc27/charset_normalizer-3.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:dca9ab98072a5a54ebacebdc45f53e645336b320c667410b061be1ca588ae709", size = 396871, upload-time = "2026-09-30T04:37:01.543Z" }, + { url = "https://files.pythonhosted.org/packages/12/c8/8379554b42e8368161d898476686947a0fdbd3e8865170d7909dcabfdee8/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0aa869112ef88429ae17820d99c3dd9504c9e9c671d3c246f3d7442cb051084", size = 249711, upload-time = "2026-09-30T04:37:03.111Z" }, + { url = "https://files.pythonhosted.org/packages/4a/eb/2ddb1035d17320caa9f41682935123a9a250277b261c3efc86b2d2a21343/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c0afc6800ba57ccc350374c5bd6150419915d95ce93cdbab2d783d75eaf30ecb", size = 231146, upload-time = "2026-09-30T04:37:04.721Z" }, + { url = "https://files.pythonhosted.org/packages/4a/24/2ecb4bde104322cd7859d6594fcfa74649f8d90b3221c9feecbef149875b/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7dcd882da75ef9adf94903b1e3b9419e8aa8fb4c7396822b834b9ef7fb96954f", size = 271552, upload-time = "2026-09-30T04:37:06.295Z" }, + { url = "https://files.pythonhosted.org/packages/3f/98/9d5f6ebc3aee9fef5d30b4aff11fb2ab7a1222b4064f8ef2c7c87cde217a/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2e06a3a98f916dd41d27f3105e02e7a40181c98c94b9158733d03a6f80506c09", size = 269810, upload-time = "2026-09-30T04:37:07.905Z" }, + { url = "https://files.pythonhosted.org/packages/09/e1/a3b06a10461b1b7628853c934c644e03bc28e42767116afb52f19a56519b/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6bd128f206a7752ae1f2ab6c61bf8a24ba28913a10df8b14c2637b973ff97a80", size = 259845, upload-time = "2026-09-30T04:37:09.554Z" }, + { url = "https://files.pythonhosted.org/packages/fd/d3/6f561f74a296cf27d61775a1dc665ad13f3bff6a798810ca05907f37a7c4/charset_normalizer-3.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c8f3d67aeaf55f017982b73683f0e7342ba2f6635a78f69ce89ebb26aa411e5c", size = 252825, upload-time = "2026-09-30T04:37:11.274Z" }, + { url = "https://files.pythonhosted.org/packages/26/9f/69e13ca3b18f43e0eafcd34c04a45b732ae22a43b54a5fc9e119103356eb/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f", size = 252659, upload-time = "2026-09-30T04:37:12.941Z" }, + { url = "https://files.pythonhosted.org/packages/73/a9/ace29806a0dae18939919c76ba526472d83214afa101105fabff2cf30625/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:92888bb3187c5ba50500b00b3b310c9f2c651709d28036077680cb5255450a03", size = 238819, upload-time = "2026-09-30T04:37:14.659Z" }, + { url = "https://files.pythonhosted.org/packages/f8/c1/6116d52a2e3311ec80f21f5fb5e17b27405f10b9608af8f6e69516841a1b/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:d008d90a7f2471519aef0c90dfbe73b3e6e4d5e66ac48e19154c17e89e98b604", size = 272973, upload-time = "2026-09-30T04:37:16.346Z" }, + { url = "https://files.pythonhosted.org/packages/19/aa/9955c7e93bba10a9c7e8f7a5031b7ced66f3a1883a55c00712b8d5850ff3/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:31f3930700408d211f13378ccbe1c40845d8da54bd0681fac3a9b5aae81c7aa8", size = 253692, upload-time = "2026-09-30T04:37:18.212Z" }, + { url = "https://files.pythonhosted.org/packages/bb/33/2a6ae7fdc1b10cb581cef91addd8cdfc5f40d50abb5702309369d5834579/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:2a925889534b3748302dae5dead07cc13480de1dac3aea80a941b729b471ef93", size = 268757, upload-time = "2026-09-30T04:37:19.877Z" }, + { url = "https://files.pythonhosted.org/packages/a2/22/80992720a0282cd39bba1db35868e6b9c22f41281160143a836544bc1d8a/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f5ec61164adcec446f8969a3358ec3f9b26bbda3b9213e5586d219afa8df2915", size = 262220, upload-time = "2026-09-30T04:37:21.583Z" }, + { url = "https://files.pythonhosted.org/packages/92/9f/181fd07e1bffea1d95cd80c84ac537354f50699c22cfc4d3c02b6fc16208/charset_normalizer-3.5.2-cp314-cp314t-win32.whl", hash = "sha256:598a11a2c7ebaa5334bf698bf29568c9c390abac6a154d8170fedecd1cea38c5", size = 197797, upload-time = "2026-09-30T04:37:23.235Z" }, + { url = "https://files.pythonhosted.org/packages/49/1c/25d8415ec1c4f2f41f1680435e4c87cfb378ff2f677d950946f2a45d0632/charset_normalizer-3.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:7fdde2c9fd9e3eca40631e024664cf2584272cc8f96308cbe5fdfc930f51d8bc", size = 222765, upload-time = "2026-09-30T04:37:24.891Z" }, + { url = "https://files.pythonhosted.org/packages/3e/b4/46b48f013dadfc0d0d33b375438e31bdf5a989dc68389c6bf627054d4df9/charset_normalizer-3.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d1befeed746d247c81127bb14de9dc3d30edb6e5976d34f83f86ed262b1d9105", size = 210985, upload-time = "2026-09-30T04:37:26.634Z" }, + { url = "https://files.pythonhosted.org/packages/ca/e9/34e597dee616d0b8ee4b34d29399e85c2204ade174157a48505d42baa4ff/charset_normalizer-3.5.2-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26", size = 367676, upload-time = "2026-09-30T04:37:28.329Z" }, + { url = "https://files.pythonhosted.org/packages/60/9f/a5d1c91c0263745e2cd344c5a4415d787c575501ab1d449f1148ac6b495d/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9409a8bf35cf78353942504b24a57de3d75b708997a1e4bd8db71ac8633ce364", size = 248414, upload-time = "2026-09-30T04:37:30.167Z" }, + { url = "https://files.pythonhosted.org/packages/26/79/e697f77464748a3ee3cf490c83d592459400d4898380d66c38366b03080c/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:498dc3188ca05a68231ac3fdbfc7f57eb67e1343c30e0fea17f8218c1599b253", size = 244775, upload-time = "2026-09-30T04:37:31.964Z" }, + { url = "https://files.pythonhosted.org/packages/ca/87/3d42a42e18ea066e2513936fd678a00696e77878b5ae04528976abdbcb83/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e242bb1c5e76e97dfa9e7f209a71e93a01d7f19ffdd5cfbb2e2d55b4f08f8ab0", size = 273877, upload-time = "2026-09-30T04:37:33.661Z" }, + { url = "https://files.pythonhosted.org/packages/c3/76/8a28136f3938ba9836f84280ce0c4d61ed1cf15a036b2034900c62634162/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:def79fa35ef0cef8d2accec024f4fdc7ead3012ff02f5215c783f39f03ef8cfc", size = 269455, upload-time = "2026-09-30T04:37:35.573Z" }, + { url = "https://files.pythonhosted.org/packages/a0/a1/4fbf5d0f0f1b2a080474c1cf9a2f12c4c6531bb0e8ba591055e846d2b4e9/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229", size = 257336, upload-time = "2026-09-30T04:37:37.397Z" }, + { url = "https://files.pythonhosted.org/packages/ba/a2/8b50aa320adb880ad579518e6f718f24944804b42a88b83d267d5d444125/charset_normalizer-3.5.2-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:04851f73ae72b8413dddadb16a49dfee95263553741fd42d546f7d66907e6be5", size = 257584, upload-time = "2026-09-30T04:37:39.522Z" }, + { url = "https://files.pythonhosted.org/packages/a5/57/50e3fed84e175f40349bd0da7a4fce94c87f0378f52d74f511d89e0bdc20/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:183b88127acdb4fabe59d951ab424faf1af7b63cdbb5f776186c1ea2ffcaed98", size = 251854, upload-time = "2026-09-30T04:37:41.23Z" }, + { url = "https://files.pythonhosted.org/packages/d6/54/f7fbb3493c9f49091213b9c2d6dd65800696f1ce1a3f196a4205f50417b1/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:16fa0eccf81304b79c5cd87f9271c3b85dd9dd99245e4422ae9c0dd45e0f99d3", size = 248767, upload-time = "2026-09-30T04:37:42.883Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/b3a6385acc5a1e45b39ae9c90bfb9cf838a09b9dd37ef2740ab4c6b4a2eb/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:7441d755b7ab94f8d4eb3e43ec05482d760842fd263d003a99102d742cd835e2", size = 274479, upload-time = "2026-09-30T04:37:44.658Z" }, + { url = "https://files.pythonhosted.org/packages/89/44/809913e2cfd279e635a9294fdbbfb1b1dc62a8189d473d561f649fce98d8/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:ca403d7e4798f525fdfc78e258820419cbbd0f0ecbab9de7840e3c017cf6b8cf", size = 258310, upload-time = "2026-09-30T04:37:46.529Z" }, + { url = "https://files.pythonhosted.org/packages/af/a2/f28400ab13359d91bd39179df8e149376b9bf36588e739a3a4f9de2b84b2/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:df29a0a7107f7011e77f4eebdddec4c7331e24d787a0b21a46d63bdf7445da95", size = 271100, upload-time = "2026-09-30T04:37:48.399Z" }, + { url = "https://files.pythonhosted.org/packages/e9/89/9bab37955edf0adb3b66f8a3a6617d9f2f487e0d56f295a6a286cb640aa6/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:f3c96f633825733f735c5a9cf21d21a257d8e1edf0b1cee0a064b9c424ca0f7d", size = 259936, upload-time = "2026-09-30T04:37:50.023Z" }, + { url = "https://files.pythonhosted.org/packages/23/b5/4459e08d45a679f903d50fea08bc52cfa728cca4d7bd02c757b5e5abda2e/charset_normalizer-3.5.2-cp315-cp315-win32.whl", hash = "sha256:281cb91036248400f4cc957495cccd44c275c2e0c5854f7e45ac5cf7dc193847", size = 186301, upload-time = "2026-09-30T04:37:51.722Z" }, + { url = "https://files.pythonhosted.org/packages/98/e8/55d5fd3935b4bce6da4fe0df61898e8c82653e317e677bd58aceb9c60f13/charset_normalizer-3.5.2-cp315-cp315-win_amd64.whl", hash = "sha256:89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8", size = 210858, upload-time = "2026-09-30T04:37:53.427Z" }, + { url = "https://files.pythonhosted.org/packages/a9/5b/974423c2fd8e524c7a7f64318c1e02240ef954912fa2b4d70344107b9c68/charset_normalizer-3.5.2-cp315-cp315-win_arm64.whl", hash = "sha256:6be488a102b8cf28d0391d8c4ba7748938ae28b78ad901f8585520fca33ead1a", size = 200309, upload-time = "2026-09-30T04:37:55.015Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f9/00ee0195db1013d8f7c416fd770fbeb560bb46eb2e36b054d05cb56f6cfa/charset_normalizer-3.5.2-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:915563965d418f986e7e145accc592eae9e1a1be3566ff98a05d7a9ec42a76e1", size = 395343, upload-time = "2026-09-30T04:37:56.743Z" }, + { url = "https://files.pythonhosted.org/packages/04/3a/c00b50e94c964cf934c7899cd47c97952fc11dad71cc5884b3c61795b09b/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:65cd72beeeca9d3aaea1201e5923859f308f952f9c71de93f06063c79f0f7a3b", size = 250114, upload-time = "2026-09-30T04:37:58.607Z" }, + { url = "https://files.pythonhosted.org/packages/50/27/d102dc880bbcffd0479ab64dfc1fb96777a854355a55e2bda72a71efadcb/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b7fd005a73d9e657273b7a10dc71a9e03c8fb9ee6999798d6918ce095b81ac7f", size = 239997, upload-time = "2026-09-30T04:38:00.511Z" }, + { url = "https://files.pythonhosted.org/packages/a5/4a/bf7ef45794dd293fab5f98a9309817977fbb845b9998f171b8cc5d8437a3/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e54da4baf05720032d527874d40b65fa4d7e5c6c6a43d0c3adbeffcaf275a2b3", size = 272313, upload-time = "2026-09-30T04:38:02.509Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ee/008a2837737991474c5754bb3191010007663860979701990982a502cbaf/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:124fbf1a8ff966d87ae05bb8bd45a71f966055ed8bba320d0c7cf450bc5f4d0e", size = 269986, upload-time = "2026-09-30T04:38:04.435Z" }, + { url = "https://files.pythonhosted.org/packages/93/ad/bd74a283940dc910c5b14f8e4f80a248082bc9c0fcbe1f54530cb6d9cc5e/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:28b4f0d66fb834ff90f28209ac7bce77868c45d8c93e26f906709d9b7c2e1af9", size = 260202, upload-time = "2026-09-30T04:38:06.549Z" }, + { url = "https://files.pythonhosted.org/packages/8a/7b/ed341c66f69f688723501fac752be3d63c7159ca0d0d4174fc611e5710bb/charset_normalizer-3.5.2-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:58ca3755ee7ff7f59b57789ec9833c9de9ea275405cdd240eda1f193112e398a", size = 259083, upload-time = "2026-09-30T04:38:08.311Z" }, + { url = "https://files.pythonhosted.org/packages/cc/9d/e41588b777965e5031a43128a1e96173ebb35ac75fc53ec3b517e7c21cd4/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:443eae2bf318abeaf6f15d785138f71fd6de770e99a92158b8b814265e079115", size = 253081, upload-time = "2026-09-30T04:38:10.402Z" }, + { url = "https://files.pythonhosted.org/packages/81/35/b761eb6d8c1eb218b9b42b9b4d5ac902afdc399fb6dac6f9a9aac7bda589/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:58f361dcbab699cf8f42db3f47c8e7fd1036f138c23a5d08de9fde5f425a730c", size = 248712, upload-time = "2026-09-30T04:38:12.317Z" }, + { url = "https://files.pythonhosted.org/packages/4d/2c/147169a041b747759f37405c0a97157e8e92de967968373101ff14915cba/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:1b4cbc7c3491ccb4aa17fcd8165649d01cf39f76de1696da8631b5f71b85401d", size = 274176, upload-time = "2026-09-30T04:38:14.138Z" }, + { url = "https://files.pythonhosted.org/packages/f0/2d/0ff8db0d373ba8538db686db11cd7e8912031490b9e4f383b41912e8d594/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:ba0b1d2620edf869789c3879223f52bf2afc5d31b3cb47cc57b3a12c05e2aa9d", size = 259413, upload-time = "2026-09-30T04:38:15.841Z" }, + { url = "https://files.pythonhosted.org/packages/8a/8e/b4a085fb47c9d3a7e43576a4784fdd8fe23f907514a972de8086edaf7a48/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:5e2b6b57e9733d39f0c9fd3185efa6b8e29652c4cd8fe94180272cf6ed9a78c4", size = 269434, upload-time = "2026-09-30T04:38:17.626Z" }, + { url = "https://files.pythonhosted.org/packages/83/1c/d8d8d7322a7c3eecdf3237a4a419cf41d2eaad8e006ce7dfdd9d4c8fa2eb/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:51cf45226a9b588d0d2b4880c62d686934b63ab0bd79ca23ab0e9762eb27441b", size = 262716, upload-time = "2026-09-30T04:38:19.214Z" }, + { url = "https://files.pythonhosted.org/packages/a0/16/0e4c6ba9b44e97a2da150e52d331e8f9c968b21b358fbffa6c856cebcd89/charset_normalizer-3.5.2-cp315-cp315t-win32.whl", hash = "sha256:5fb29fb8cd1a46c27a1bf9613ad5ec2599310d46b4025d9556404a6b6a292800", size = 197159, upload-time = "2026-09-30T04:38:21.037Z" }, + { url = "https://files.pythonhosted.org/packages/be/33/e90bc2b1374f7f36ef106f56620de5a783907e19ca857efe2277e31cac3e/charset_normalizer-3.5.2-cp315-cp315t-win_amd64.whl", hash = "sha256:a192e2c40070d92c3ccf777e3a5c4ff515573cd2bb7ed0c537fdadbbec5bbf21", size = 221322, upload-time = "2026-09-30T04:38:22.886Z" }, + { url = "https://files.pythonhosted.org/packages/66/89/dfa6dcb08c200b7830ab56439e8c1890f2971d51aafbb3937894a2e7fcfc/charset_normalizer-3.5.2-cp315-cp315t-win_arm64.whl", hash = "sha256:749e97e1b32313717a565abbe321bc2190bc8b35f1a67e4cdbc7c56c8d8ffe58", size = 209192, upload-time = "2026-09-30T04:38:24.648Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ab/176fbfd5b64939c55d652366aa5b9ef1d767af207a3aa6ebeb0d226c484d/charset_normalizer-3.5.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd", size = 331815, upload-time = "2026-09-30T04:38:26.216Z" }, + { url = "https://files.pythonhosted.org/packages/7e/84/371eac6b30bdbcbf2d632a1a01809103459216fcaae61b8b8d922c1bfb8a/charset_normalizer-3.5.2-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7", size = 253276, upload-time = "2026-09-30T04:38:28.032Z" }, + { url = "https://files.pythonhosted.org/packages/43/6f/c4fbae58febff71709c51bc7e18fdfa55341dc382704740f9f0cbf03817b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f", size = 241239, upload-time = "2026-09-30T04:38:29.732Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/458c3f42164a07d0c5210798e9e704b39e540a6793b05aba67f3a35243a9/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93", size = 231121, upload-time = "2026-09-30T04:38:31.462Z" }, + { url = "https://files.pythonhosted.org/packages/09/54/ab9e89367076f6331bb6c65c4bf14a5361fa5191cb6561bf534f18504e1b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade", size = 260350, upload-time = "2026-09-30T04:38:33.239Z" }, + { url = "https://files.pythonhosted.org/packages/7c/c1/061431ecc688d9d76602502cb57cc01e691e682c18f1beb45f9673b5bbd2/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0", size = 255430, upload-time = "2026-09-30T04:38:34.865Z" }, + { url = "https://files.pythonhosted.org/packages/8d/1f/20c8949f0676f7ab811abdeb7f4d7f1cbc6e61ff20bef08b44edeb092bc8/charset_normalizer-3.5.2-cp37-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26", size = 250612, upload-time = "2026-09-30T04:38:36.649Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9e/46f2fa4c431fc98c4ae76a8cb5bdca54e0341e3cfc3fcfd8e82740250818/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011", size = 242083, upload-time = "2026-09-30T04:38:38.26Z" }, + { url = "https://files.pythonhosted.org/packages/bd/39/559be29a0c0f086e0bba6922babd38916cc5e0b58ced4de13ee01ea05508/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621", size = 232738, upload-time = "2026-09-30T04:38:39.81Z" }, + { url = "https://files.pythonhosted.org/packages/ff/6c/387b0e4f756a282831c1d9fc6aeb6c51ca4507ca202767c8de15ce9b12e2/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4", size = 260703, upload-time = "2026-09-30T04:38:41.346Z" }, + { url = "https://files.pythonhosted.org/packages/96/92/1fdf015f09ef449f50d3ac4b67c90887c9c318b727daa95cc4f866e6521d/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e", size = 247622, upload-time = "2026-09-30T04:38:42.937Z" }, + { url = "https://files.pythonhosted.org/packages/dc/3c/8e7b8a5671ad5d433669fb2a76f1a0164df2d9b1718b0206bc2a16d840cc/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_s390x.whl", hash = "sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c", size = 257500, upload-time = "2026-09-30T04:38:44.604Z" }, + { url = "https://files.pythonhosted.org/packages/b4/f0/45b579df5cabc1d5d53ea1cc35e8437d3ca768c0acccc7041517cb6fbb32/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0", size = 255100, upload-time = "2026-09-30T04:38:46.289Z" }, + { url = "https://files.pythonhosted.org/packages/31/68/fdec18a343f5fb3f310588dd478b09ac4799e0b187dbade3a8cd776f03ef/charset_normalizer-3.5.2-cp37-abi3-win32.whl", hash = "sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf", size = 174499, upload-time = "2026-09-30T04:38:47.999Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8a/b618149cc5207943a0242068d7a27897f56a62947b5a039085f2a22029f8/charset_normalizer-3.5.2-cp37-abi3-win_amd64.whl", hash = "sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036", size = 200092, upload-time = "2026-09-30T04:38:49.707Z" }, + { url = "https://files.pythonhosted.org/packages/03/cf/4c66866fa9e2b1c78e3c911516d1de497a677b7ac60f1eceda74ce777ca3/charset_normalizer-3.5.2-cp37-abi3-win_arm64.whl", hash = "sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e", size = 294363, upload-time = "2026-09-30T04:38:51.312Z" }, + { url = "https://files.pythonhosted.org/packages/fc/ad/d07d7862a62ffa6d79d68074d14823243dd235a77c45262acbf6adeb28bf/charset_normalizer-3.5.2-py3-none-any.whl", hash = "sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685", size = 68872, upload-time = "2026-09-30T04:39:21.828Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "docker" +version = "7.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "requests" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/88/7f/731ff914b0255d3d065f45fd4e626d4b8c95dbcbaada049f337a6ac16410/docker-7.2.0.tar.gz", hash = "sha256:cebb93773d334f778e023a7ee352a8d6e13ab1bd3b863a4d4a59dec897df43ac", size = 118731, upload-time = "2026-07-09T14:53:46.39Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/23/529140fe1aab80fc6992f93a706deec709140a6397439139a054e1515c45/docker-7.2.0-py3-none-any.whl", hash = "sha256:a3f45fdeb9165e2d25d9a1d02ddf3bc70fb572cf5ebbf9b58558c22caf29b71f", size = 148775, upload-time = "2026-07-09T14:53:45.224Z" }, +] + +[[package]] +name = "idna" +version = "3.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/08/8eea9d4b8302028f3abb2c0813953f7aec26d33b7a8960ed760e65ff29fa/idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44", size = 216463, upload-time = "2026-09-17T14:11:04.752Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz", hash = "sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960", size = 21209, upload-time = "2026-10-06T22:48:38.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/db/7fc19e6f2dc92a966727031389fc2e08b558f0f25eb7403c1119ad4713cd/pluggy-1.7.0.tar.gz", hash = "sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8", size = 123304, upload-time = "2026-10-15T09:50:58.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", hash = "sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", size = 27082, upload-time = "2026-10-15T09:50:56.808Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pywin32" +version = "312" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/ff/32aa7d2ed0ab12b323aaa64f9b75e6ad4f8fd09f9ccfc28c79414d46838d/pywin32-312-cp312-cp312-win32.whl", hash = "sha256:dab4f65ac9c4e48400a2a0530c46c3c579cd5905ecd11b80692373915269208b", size = 6371877, upload-time = "2026-06-04T07:49:28.836Z" }, + { url = "https://files.pythonhosted.org/packages/03/d9/77040d3b43df3f3be32ea289433d660d2727f5ba327bc73be835127d9d60/pywin32-312-cp312-cp312-win_amd64.whl", hash = "sha256:b457f6d628a47e8a7346ce22acb7e1a46a4a78b52e1d17e1af56871bd19a93bc", size = 6914841, upload-time = "2026-06-04T07:49:31.85Z" }, + { url = "https://files.pythonhosted.org/packages/e3/cc/7b1ec671775756020a0ee7f4feeaf3c568f0ab86bd3900088cf986937a92/pywin32-312-cp312-cp312-win_arm64.whl", hash = "sha256:6017c58e12f6809fbb0555b75df144c2922a9ffd18e4b9b5afa863b6c1a9d950", size = 6727901, upload-time = "2026-06-04T07:49:34.244Z" }, + { url = "https://files.pythonhosted.org/packages/2d/41/12fbfd7f36ed2146d8bc9de96c2741296bf0d490b98508496cff322e274c/pywin32-312-cp313-cp313-win32.whl", hash = "sha256:7a27df850933d16a8eabfbaeb73d52b273e2da667f80d70b01a89d1f6828d02c", size = 6370184, upload-time = "2026-06-04T07:49:36.253Z" }, + { url = "https://files.pythonhosted.org/packages/ba/db/36a78e3403099d31d9746d13fdcde5accc43c1155f375a34d15983a479a7/pywin32-312-cp313-cp313-win_amd64.whl", hash = "sha256:c53e878d15a1c44788082bfe712a905433473aa38f86375b7cf8b45e3acbaaf9", size = 6914298, upload-time = "2026-06-04T07:49:38.876Z" }, + { url = "https://files.pythonhosted.org/packages/84/37/c1697194092b76de9ed47ca124323f02c57ffc8a45c06f88a3d5acaf01eb/pywin32-312-cp313-cp313-win_arm64.whl", hash = "sha256:59aba5d5940842075343a5ddc6b11f1cdf0d1567fe745290359dfbcc7c2eb831", size = 6727640, upload-time = "2026-06-04T07:49:41.083Z" }, + { url = "https://files.pythonhosted.org/packages/fc/2b/1f3cded5822fd49c02f40544cbb5f58c7cfd6b1694869fd476cb6170ee97/pywin32-312-cp314-cp314-win32.whl", hash = "sha256:a77a90fbb6881238d2ca9c6fd797b25817f3768fe78d214a90137ff055a75f5b", size = 6468928, upload-time = "2026-06-04T07:49:43.188Z" }, + { url = "https://files.pythonhosted.org/packages/21/82/3bf86d2e2808902013132e1ce905a7da0da53790f3836c64bf44d55e24f3/pywin32-312-cp314-cp314-win_amd64.whl", hash = "sha256:a4dd3a848290ef724347b19f301045831d8e802fa4464f491b98b1e0a081432e", size = 7024157, upload-time = "2026-06-04T07:49:45.34Z" }, + { url = "https://files.pythonhosted.org/packages/a4/0e/73f6d6800b4f27655abd9e9f6aaeaefcddb2b946e4674efa2bab184a7f7b/pywin32-312-cp314-cp314-win_arm64.whl", hash = "sha256:9fce94568364e0155e6dfb781ac5d95903be8baf28670632beab1b523f300daa", size = 6839598, upload-time = "2026-06-04T07:49:47.613Z" }, + { url = "https://files.pythonhosted.org/packages/eb/61/caa39686032d2ebdd04ff0ab5cbe163126c0066d98e00c9018646e42393b/pywin32-312-cp315-cp315-win32.whl", hash = "sha256:5c1fbe4a937a73ae9297384a3da38518cbc694c68ad8a809b2e19acd350f03ed", size = 6471159, upload-time = "2026-06-04T07:49:50.035Z" }, + { url = "https://files.pythonhosted.org/packages/0f/cd/7e1de64a4a6f69c04214169657ccab0d93a670ea50e35eb8f489d7378249/pywin32-312-cp315-cp315-win_amd64.whl", hash = "sha256:c2f03a0f73f804a13c2735b99392b0cd426bb4f2c4d0178e5ac966a0f21618d5", size = 7025293, upload-time = "2026-06-04T07:49:54.857Z" }, + { url = "https://files.pythonhosted.org/packages/23/ed/4532e9388e65fa16b46776ef47ad631a64eda1631884488af707666350ed/pywin32-312-cp315-cp315-win_arm64.whl", hash = "sha256:a8597d28f267b39074aef51fa593530082b39cbe5a074226096857b1fed2dfb9", size = 6840337, upload-time = "2026-06-04T07:49:57.531Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "ruff" +version = "0.17.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e7/e6d5749983b020e6cb1055abec0a6f361a8e94f0184cdcf56d4760995f02/ruff-0.17.1.tar.gz", hash = "sha256:5bb796c5112e9fb9527f2ef2faa0a130d6312d19de92bcfaddd70dbb2d41020f", size = 5323349, upload-time = "2026-10-15T15:23:18.64Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/78/90c46ca6250709f7c37e397b60d06061adee8d0f7d13959ed6a8e49b14fd/ruff-0.17.1-py3-none-linux_armv6l.whl", hash = "sha256:1796c41eafec66f4635125dd1e074a7aa27b8d20c187d484360acf11d247fc4a", size = 10147528, upload-time = "2026-10-15T15:22:33.667Z" }, + { url = "https://files.pythonhosted.org/packages/aa/d2/a2e28bc35b083ae51fa07dfba5280a667c8b01ecaee7802fc8849dcd5c46/ruff-0.17.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:8eadb09a788010be29c895418bdfa513e4e1c65e64b94e5b19e4cd381f6cc7fb", size = 10225974, upload-time = "2026-10-15T15:22:36.328Z" }, + { url = "https://files.pythonhosted.org/packages/8e/71/7a9029a81c8206ee01a98f6cfeeb56e6c095af86857581a37045c46d7e94/ruff-0.17.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:29b454f18ca3eeaf37fbf140841dcf27d36109e29dee9aae8acecb68c26c6f74", size = 9270361, upload-time = "2026-10-15T15:22:38.651Z" }, + { url = "https://files.pythonhosted.org/packages/c0/8c/e075e9c917cb0575297880c96768304e881240171c870cbd8d9b7b54bec2/ruff-0.17.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:872e03dd231744d6c5ec5c8af6be52b71323fc6ee096e4c8b34d9f844ee3b978", size = 9938300, upload-time = "2026-10-15T15:22:41.284Z" }, + { url = "https://files.pythonhosted.org/packages/87/27/0b4bb926a49de2383f161272b4d863571a4c7ecdbf595a6bc58813a13367/ruff-0.17.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4c0d6cb5f8fbac77eec9af49f4fc0eaf856b05545e372b0694d24b4e76b1c640", size = 9948464, upload-time = "2026-10-15T15:22:43.681Z" }, + { url = "https://files.pythonhosted.org/packages/14/27/518d57e0e02aecb8ea4acac6b57b51a53173d25fa734952bfdd7d531e88e/ruff-0.17.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78beced4a3c94dfedbee4b999b95b6a27b35820fdc0afbacc6d37a8da26c1cd1", size = 10638639, upload-time = "2026-10-15T15:22:46.081Z" }, + { url = "https://files.pythonhosted.org/packages/01/82/d89a4e498ade59e6cee52d67f8da569019372212b87468c1fdeae884601c/ruff-0.17.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7feefc63921202894f0e158b0e32f45a5950d70a7fb693ccb9264816c25b1c05", size = 11337238, upload-time = "2026-10-15T15:22:48.894Z" }, + { url = "https://files.pythonhosted.org/packages/a2/7c/3b0fdef9352656317c977086fe1d802902406e4f2a0d8578da90c0e6d1c5/ruff-0.17.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:684154861a819bf1f433669d2a0ce9c3e54677c65b8ebb866cc2a7ec86345785", size = 10896327, upload-time = "2026-10-15T15:22:51.45Z" }, + { url = "https://files.pythonhosted.org/packages/62/86/5566290219708ce2928f3a29c4f843a1185b341ec6111c4631a33c5ceced/ruff-0.17.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6ef98e51de8ef38082f90653ceeb84944bd7673783eea13e1647c5b9b064189", size = 10350811, upload-time = "2026-10-15T15:22:54.401Z" }, + { url = "https://files.pythonhosted.org/packages/ad/d2/ca8462dabb3c413990d49703ba306c30c862195a75448d080fa8fcce9775/ruff-0.17.1-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:8f9aa5b5bd5fa1937803ea609c8f153b8de39bf3e0e149897ae34b0c5b52f467", size = 10569994, upload-time = "2026-10-15T15:22:57.142Z" }, + { url = "https://files.pythonhosted.org/packages/e3/ad/dd1864c1f1db17c8cfb297ce70b352388e6977620f66f7182c52855407df/ruff-0.17.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:80e53b0e5faa3c5667f035daaa032893ae42b7cfe708c886d39ed131e54688ab", size = 10059672, upload-time = "2026-10-15T15:23:00.084Z" }, + { url = "https://files.pythonhosted.org/packages/65/45/3f56cf1d2168c2165cdd5ac6f9668aee66a4cfdd4caf2b2cc5ccb2b1d568/ruff-0.17.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:056c5d5fa0ac790051ae165d7035c7bf956d02de2db557f2e9b31d877a505fca", size = 9947803, upload-time = "2026-10-15T15:23:03.132Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4b/5457cb2be4c1d1ea5e919558131a0f197a4a454a0530d9e30820e8206e4c/ruff-0.17.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:bd8f5a55b10f371642eacd0e8665a42a8b97da15c9a7187c79e697b77b5e0ce4", size = 10351807, upload-time = "2026-10-15T15:23:05.628Z" }, + { url = "https://files.pythonhosted.org/packages/49/fb/db15b4f8fa24a1e36b99e1401f42ef6df16ee299a48670d5d262b771e7d7/ruff-0.17.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:ce05e6f3973611d9550a600e04bb531061e368baba31610eb57c1509a8f21a0a", size = 10765716, upload-time = "2026-10-15T15:23:08.092Z" }, + { url = "https://files.pythonhosted.org/packages/0a/e8/1e186578076aff7a80735f625005fde678412ab609cd323ed182eb7d6b87/ruff-0.17.1-py3-none-win32.whl", hash = "sha256:1de0030a8aa78c65c5fecb84b2e5c390ac684c0452875a02afc7c083d49a496a", size = 9895093, upload-time = "2026-10-15T15:23:10.582Z" }, + { url = "https://files.pythonhosted.org/packages/6a/2e/da3d9589bae8e8b58f3a48bd5bc0b980232fb14a83bf02c3be7660f87c1b/ruff-0.17.1-py3-none-win_amd64.whl", hash = "sha256:5df76f16580fab58d0f6c98a3ac254a4abceb7b2489b8b678335869cff77b3d7", size = 10097955, upload-time = "2026-10-15T15:23:13.309Z" }, + { url = "https://files.pythonhosted.org/packages/df/c9/0f2a4dea06a5e12c1191ee3d96b1e1233797d0d5e18f97a2b909a95a9836/ruff-0.17.1-py3-none-win_arm64.whl", hash = "sha256:42400b9a6ff8515ff27796f07f92e0195be011c86e81e381606f58ea5502fef4", size = 9853542, upload-time = "2026-10-15T15:23:16.05Z" }, +] + +[[package]] +name = "urllib3" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, +] + +[[package]] +name = "valinor-integration" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "docker" }, + { name = "pyyaml" }, + { name = "requests" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "docker", specifier = ">=7.2.0" }, + { name = "pyyaml", specifier = ">=6.0.3" }, + { name = "requests", specifier = ">=2.34.2" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=9.1.1" }, + { name = "ruff", specifier = ">=0.17.1" }, +] From 9588157a8b5cd893dafd22afa4ace7a211d41de5 Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Mon, 2 Nov 2026 09:38:57 -0800 Subject: [PATCH 21/37] Add valinor-cli extract for offline sub-extracts valinor-cli extract --tile LEVEL/TILE_ID copies the given tiles of a source into a new extract tarball with write_extract. The source can be an extract tarball or a tile directory. You can give --tile more than once, or give it a comma-separated list. It uses GraphId::parse_tile. The command prints a one-line summary on stderr, which counts each tile once. - The command writes the extract to a temporary file in the output directory, syncs it, and then renames it to the output path. So the output path is either the complete extract or what it was before the command. The temporary file gets the permissions that File::create gives, which are 0o666 minus the umask. The default of tempfile is 0o600. tempfile is a new dependency of valinor-cli. - The extract subcommand takes a source path directly, not a Valhalla JSON config. So the config path moves from a global argument into the get-edge arguments (valinor-cli get-edge ). The VALHALLA_CONFIG environment variable still works. - get-edge parses its graph ID with the FromStr implementation of GraphId, in place of its own parser. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + valinor-cli/Cargo.toml | 1 + valinor-cli/README.md | 25 ++- valinor-cli/src/commands/extract.rs | 290 +++++++++++++++++++++++++++ valinor-cli/src/commands/get_edge.rs | 35 +--- valinor-cli/src/commands/mod.rs | 6 +- valinor-cli/src/main.rs | 73 ++++++- 7 files changed, 389 insertions(+), 42 deletions(-) create mode 100644 valinor-cli/src/commands/extract.rs diff --git a/Cargo.lock b/Cargo.lock index a4c24cb..1c1792d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2286,6 +2286,7 @@ dependencies = [ "anyhow", "clap", "serde_json", + "tempfile", "tracing", "tracing-subscriber", "valhalla-graphtile", diff --git a/valinor-cli/Cargo.toml b/valinor-cli/Cargo.toml index cc0b362..1961b76 100644 --- a/valinor-cli/Cargo.toml +++ b/valinor-cli/Cargo.toml @@ -12,6 +12,7 @@ rust-version.workspace = true anyhow = { workspace = true } clap = { workspace = true, features = ["derive", "env"] } serde_json = { workspace = true } +tempfile = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter"] } valhalla-graphtile = { workspace = true, features = ["serde"] } diff --git a/valinor-cli/README.md b/valinor-cli/README.md index f620a82..df299af 100644 --- a/valinor-cli/README.md +++ b/valinor-cli/README.md @@ -1,8 +1,8 @@ # Valinor CLI A CLI multi-tool for quickly inspecting Valhalla graphs. -This utility takes a Valhalla JSON configuration as input, -and has subcommands with various functionality. +Most subcommands take a Valhalla JSON configuration as input +(`extract` is the exception; it takes a tile extract tarball or a tile directory directly). This is primarily meant as a low-level debugging tool for Valinor developers. If this looks useful to you, it probably is; otherwise, it's probably not. @@ -12,9 +12,26 @@ If this looks useful to you, it probably is; otherwise, it's probably not. As an example, you can get edge info like this (works with either the integer or slash separated form): ```shell -RUST_LOG=info cargo run --package valinor-cli -- ~/valhalla-docker/valhalla/valhalla.json get-edge 901881911882 +RUST_LOG=info cargo run --package valinor-cli -- get-edge ~/valhalla-docker/valhalla/valhalla.json 901881911882 ``` Then you'll get some JSON output on your terminal with the details. -Run the `help` subcommand for more details (everything is automatically documented via clap). \ No newline at end of file +## Example: writing an offline sub-extract + +You can write a smaller, fully spec-compliant extract +which contains only some tiles of a source (each tile is `level/tile_id`). +The source can be a tile extract tarball or a tile directory: + +```shell +cargo run --package valinor-cli -- extract valhalla-graphtile/fixtures/andorra-tiles.tar andorra-sub.tar --tile 0/3015,1/47701,2/763926 +cargo run --package valinor-cli -- extract valhalla-graphtile/fixtures/andorra-tiles andorra-sub.tar --tile 0/3015,1/47701,2/763926 +``` + +A one-line summary (tile count and output size) is printed on stderr. +The output is interchangeable with `valhalla_build_extract` output. +The command writes to a temporary file in the output directory, +and renames it to the output path only when the extract is complete. +So the output path never contains a partial extract. + +Run the `help` subcommand for more details (everything is automatically documented via clap). diff --git a/valinor-cli/src/commands/extract.rs b/valinor-cli/src/commands/extract.rs new file mode 100644 index 0000000..4ab9a4a --- /dev/null +++ b/valinor-cli/src/commands/extract.rs @@ -0,0 +1,290 @@ +//! The `extract` subcommand: writes an offline sub-extract of a tile extract or tile directory. + +use std::{ + collections::HashSet, + io::BufWriter, + num::NonZeroUsize, + path::{Path, PathBuf}, +}; + +#[cfg(unix)] +use std::{fs::Permissions, os::unix::fs::PermissionsExt}; + +use anyhow::Context; +use clap::Args; +use tempfile::NamedTempFile; +use valhalla_graphtile::GraphId; +use valhalla_graphtile::extract::write_extract; +use valhalla_graphtile::tile_provider::{ + DirectoryGraphTileProvider, GraphTileProvider, TarballTileProvider, +}; + +/// Write an offline sub-extract of a tile extract tarball or a tile directory +/// +/// The output is an extract tarball in the format of `valhalla_build_extract`, +/// with the given tiles copied byte for byte. +#[derive(Args, Debug)] +pub(crate) struct ExtractArgs { + /// Path to the source: a tile extract tarball (e.g. tiles.tar) + /// or a tile directory (e.g. the `tile_dir` of a Valhalla config) + source: PathBuf, + + /// Path to write the output extract tarball to + output: PathBuf, + + /// A tile to copy, as level/tile_id (e.g. `--tile 2/763926`) + /// + /// Give the option once for each tile, + /// or give a comma-separated list (e.g. `--tile 0/3015,1/47701`). + #[arg( + long = "tile", + value_name = "LEVEL/TILE_ID", + required = true, + value_delimiter = ',', + value_parser = GraphId::parse_tile + )] + pub(crate) tiles: Vec, +} + +/// A successful extract, as reported on stderr. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct ExtractSummary { + tile_count: usize, + bytes_written: u64, +} + +/// Writes the sub-extract from the tile directory or the tarball at the source path. +fn write_sub_extract(args: &ExtractArgs) -> anyhow::Result { + let bytes_written = if args.source.is_dir() { + // The writer reads each tile once, so a cache is of no use. + let provider = DirectoryGraphTileProvider::new(args.source.clone(), NonZeroUsize::MIN); + write_atomically(&provider, &args.tiles, &args.output)? + } else { + let provider = TarballTileProvider::new_readonly(&args.source).with_context(|| { + format!( + "Failed to open source tile extract at {}", + args.source.display() + ) + })?; + write_atomically(&provider, &args.tiles, &args.output)? + }; + + // The extract contains each tile once, even if the arguments name it more than once. + let tile_count = args.tiles.iter().collect::>().len(); + Ok(ExtractSummary { + tile_count, + bytes_written, + }) +} + +/// Writes the extract to a temporary file next to `output`, and then renames it to `output`. +/// +/// If the write fails, `output` keeps its previous contents. +fn write_atomically( + provider: &P, + tiles: &[GraphId], + output: &Path, +) -> anyhow::Result { + // The rename is atomic only within one file system. + let output_dir = output + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + + let mut builder = tempfile::Builder::new(); + builder.prefix(".valinor-extract-").suffix(".tar.partial"); + // Use the permissions of `File::create`, not the owner-only default of `tempfile`. + #[cfg(unix)] + builder.permissions(Permissions::from_mode(0o666)); + let temp_file: NamedTempFile = builder.tempfile_in(output_dir).with_context(|| { + format!( + "Failed to create a temporary file in {}", + output_dir.display() + ) + })?; + + let mut dest = BufWriter::new(temp_file); + let bytes_written = write_extract(provider, tiles.iter().copied(), &mut dest) + .context("Failed to write the extract")?; + let temp_file = dest + .into_inner() + .map_err(|e| e.into_error()) + .context("Failed to flush the output extract")?; + // Make sure that the data is on disk before the rename makes the file visible. + temp_file + .as_file() + .sync_all() + .context("Failed to sync the output extract to disk")?; + temp_file.persist(output).with_context(|| { + format!( + "Failed to move the output extract into place at {}", + output.display() + ) + })?; + + Ok(bytes_written) +} + +/// Runs the `extract` subcommand, and prints a one-line summary on stderr. +pub(crate) fn run(args: &ExtractArgs) -> anyhow::Result<()> { + let summary = write_sub_extract(args)?; + let noun = if summary.tile_count == 1 { + "tile" + } else { + "tiles" + }; + eprintln!( + "Extracted {} {noun} ({} bytes) to {}", + summary.tile_count, + summary.bytes_written, + args.output.display() + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + /// Path to a fixture in the sibling `valhalla-graphtile` crate. + fn fixture(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("valhalla-graphtile") + .join("fixtures") + .join(name) + } + + fn tile(level: u8, tile_id: u64) -> GraphId { + GraphId::try_from_components(level, tile_id, 0).expect("valid tile ID in test") + } + + // Tragically, memory maps are not supported by Miri! + #[cfg(not(miri))] + #[test] + fn extract_round_trips_from_a_tarball_and_a_directory() { + let temp_dir = tempfile::tempdir().expect("a temporary directory is created"); + let tiles = vec![tile(0, 3015), tile(1, 47_701), tile(2, 763_926)]; + // The arguments name one tile twice, but the extract contains it once. + let mut tile_args = tiles.clone(); + tile_args.push(tile(1, 47_701)); + + let mut outputs = Vec::new(); + for (source, name) in [ + (fixture("andorra-tiles.tar"), "from-tarball.tar"), + (fixture("andorra-tiles"), "from-directory.tar"), + ] { + let output = temp_dir.path().join(name); + let args = ExtractArgs { + source, + output: output.clone(), + tiles: tile_args.clone(), + }; + + let summary = write_sub_extract(&args).expect("the extract writes cleanly"); + assert_eq!( + summary.tile_count, + tiles.len(), + "The summary must count each tile once" + ); + + let on_disk = fs::metadata(&output).expect("the output exists").len(); + assert_eq!( + summary.bytes_written, on_disk, + "The summary must give the size of the output file" + ); + + let extract = TarballTileProvider::new_readonly(&output) + .expect("the generated extract opens cleanly"); + let mut extracted: Vec = extract.tile_ids().copied().collect(); + extracted.sort_unstable(); + let mut expected = tiles.clone(); + expected.sort_unstable(); + assert_eq!( + extracted, expected, + "The extract must contain each requested tile once" + ); + + outputs.push(fs::read(&output).expect("the output is readable")); + } + + assert_eq!( + outputs[0], outputs[1], + "The tarball and directory fixtures must give the same extract" + ); + + let mut names: Vec<_> = fs::read_dir(temp_dir.path()) + .expect("the directory is readable") + .map(|entry| entry.expect("the entry is readable").file_name()) + .collect(); + names.sort_unstable(); + assert_eq!( + names, + ["from-directory.tar", "from-tarball.tar"], + "No temporary files must remain after a successful extract" + ); + } + + #[cfg(not(miri))] + #[test] + fn failed_extract_keeps_the_previous_output() { + let temp_dir = tempfile::tempdir().expect("a temporary directory is created"); + let output = temp_dir.path().join("andorra-sub.tar"); + fs::write(&output, b"previous contents").expect("the previous output is written"); + + let args = ExtractArgs { + source: fixture("andorra-tiles.tar"), + output: output.clone(), + // The fixture has no tile 2/0. + tiles: vec![tile(0, 3015), tile(2, 0)], + }; + + let error = write_sub_extract(&args).expect_err("a missing tile fails the extract"); + assert!( + format!("{error:#}").contains("GraphId 2/0/0 is not in the provider"), + "The error must name the missing tile: {error:#}" + ); + assert_eq!( + fs::read(&output).expect("the previous output still exists"), + b"previous contents", + "A failed extract must not change the previous output" + ); + assert_eq!( + fs::read_dir(temp_dir.path()) + .expect("the directory is readable") + .count(), + 1, + "The temporary file must be removed after a failed extract" + ); + } + + #[cfg(unix)] + #[cfg(not(miri))] + #[test] + fn output_has_the_permissions_of_a_new_file() { + let temp_dir = tempfile::tempdir().expect("a temporary directory is created"); + let output = temp_dir.path().join("andorra-sub.tar"); + let reference = temp_dir.path().join("reference"); + fs::File::create(&reference).expect("the reference file is created"); + + let args = ExtractArgs { + source: fixture("andorra-tiles.tar"), + output: output.clone(), + tiles: vec![tile(0, 3015)], + }; + write_sub_extract(&args).expect("the extract writes cleanly"); + + let mode = |path: &Path| { + fs::metadata(path) + .expect("the file exists") + .permissions() + .mode() + }; + assert_eq!( + mode(&output), + mode(&reference), + "The output must have the permissions of a file from File::create" + ); + } +} diff --git a/valinor-cli/src/commands/get_edge.rs b/valinor-cli/src/commands/get_edge.rs index b201ca4..211cef9 100644 --- a/valinor-cli/src/commands/get_edge.rs +++ b/valinor-cli/src/commands/get_edge.rs @@ -3,7 +3,6 @@ use std::{ fs, path::{Path, PathBuf}, - str::FromStr, }; use anyhow::{Context, anyhow}; @@ -20,28 +19,12 @@ use valhalla_graphtile::{ /// Pretty-print information about a directed edge (incl. live traffic, if available) #[derive(Args, Debug)] pub(crate) struct GetEdgeArgs { - /// Graph ID (u64) or slash-form level/tile/index - graph_id: String, -} - -fn parse_graph_id(input: &str) -> anyhow::Result { - // Try to parse as a pure integer. - if let Ok(id) = input.parse::() { - return GraphId::try_from_id(id).map_err(|e| anyhow!(e)); - } - - // Try slash-separated level/tile/index - let parts: Vec<&str> = input.split('/').collect(); - if parts.len() == 3 { - let level = u8::from_str(parts[0]).context("invalid level in graph id")?; - let tile_id = u64::from_str(parts[1]).context("invalid tile id in graph id")?; - let index = u64::from_str(parts[2]).context("invalid index in graph id")?; - return GraphId::try_from_components(level, tile_id, index).map_err(|e| anyhow!(e)); - } + /// Path to valhalla.json + #[arg(env)] + valhalla_config: PathBuf, - Err(anyhow!( - "Unrecognized graph id format. Use a u64 integer or level/tile/index" - )) + /// Graph ID (u64) or slash-form level/tile_id/index + pub(crate) graph_id: GraphId, } #[derive(Debug, Clone)] @@ -107,10 +90,10 @@ fn pretty_print_edge_info( Ok(()) } -/// Runs the `get-edge` subcommand with the given Valhalla config. -pub(crate) fn run(valhalla_config: &Path, args: &GetEdgeArgs) -> anyhow::Result<()> { - let gid = parse_graph_id(&args.graph_id)?; - let sources = parse_valhalla_data_paths(valhalla_config)?; +/// Runs the `get-edge` subcommand. +pub(crate) fn run(args: &GetEdgeArgs) -> anyhow::Result<()> { + let gid = args.graph_id; + let sources = parse_valhalla_data_paths(&args.valhalla_config)?; let traffic_extract = if let Some(path) = sources.traffic_extract { info!(path = path.to_str(), "Using traffic extract"); Some(TrafficTileProvider::new_readonly(path)?) diff --git a/valinor-cli/src/commands/mod.rs b/valinor-cli/src/commands/mod.rs index 24c714d..f23f125 100644 --- a/valinor-cli/src/commands/mod.rs +++ b/valinor-cli/src/commands/mod.rs @@ -1,6 +1,4 @@ -//! The subcommands of the CLI. -//! -//! Each subcommand has its own module, -//! which contains its clap arguments and a `run` function. +//! The subcommands of the CLI, with one module for each subcommand. +pub(crate) mod extract; pub(crate) mod get_edge; diff --git a/valinor-cli/src/main.rs b/valinor-cli/src/main.rs index 02a1b68..7ed9e99 100644 --- a/valinor-cli/src/main.rs +++ b/valinor-cli/src/main.rs @@ -1,21 +1,15 @@ mod commands; -use std::path::PathBuf; - use clap::{Parser, Subcommand}; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::{EnvFilter, Layer}; -use commands::get_edge; +use commands::{extract, get_edge}; #[derive(Parser, Debug)] #[command(name = "valinor-cli", author, version, about, long_about = None)] struct Cli { - /// Path to valhalla.json - #[arg(env)] - valhalla_config: PathBuf, - /// Subcommand/tool to run #[command(subcommand)] command: Commands, @@ -24,6 +18,7 @@ struct Cli { #[derive(Subcommand, Debug)] enum Commands { GetEdge(get_edge::GetEdgeArgs), + Extract(extract::ExtractArgs), } fn main() -> anyhow::Result<()> { @@ -35,7 +30,8 @@ fn main() -> anyhow::Result<()> { let cli = Cli::parse(); match cli.command { - Commands::GetEdge(args) => get_edge::run(&cli.valhalla_config, &args), + Commands::GetEdge(args) => get_edge::run(&args), + Commands::Extract(args) => extract::run(&args), } } @@ -43,9 +39,70 @@ fn main() -> anyhow::Result<()> { mod tests { use super::*; use clap::CommandFactory; + use valhalla_graphtile::GraphId; + + fn tile(level: u8, tile_index: u64) -> GraphId { + GraphId::try_from_components(level, tile_index, 0).expect("valid tile ID in test") + } #[test] fn clap_derive_is_consistent() { Cli::command().debug_assert(); } + + #[test] + fn extract_tiles_accept_repeated_and_comma_separated_options() { + let cli = Cli::try_parse_from([ + "valinor-cli", + "extract", + "in.tar", + "out.tar", + "--tile", + "0/3015,1/47701", + "--tile", + "2/763926", + ]) + .expect("the arguments parse"); + let Commands::Extract(args) = cli.command else { + panic!("expected the extract subcommand"); + }; + assert_eq!( + args.tiles, + vec![tile(0, 3015), tile(1, 47_701), tile(2, 763_926)] + ); + } + + #[test] + fn extract_tiles_reject_malformed_values() { + for input in ["2", "2/763926/7", "2/abc", "8/0"] { + let error = Cli::try_parse_from([ + "valinor-cli", + "extract", + "in.tar", + "out.tar", + "--tile", + input, + ]) + .expect_err("a malformed tile is rejected"); + assert!(error.to_string().contains(input), "{input}: {error}"); + } + } + + #[test] + fn get_edge_accepts_both_graph_id_forms() { + let expected = GraphId::try_from_components(2, 762_485, 42).expect("valid graph ID"); + for input in [expected.value().to_string(), "2/762485/42".to_string()] { + let cli = Cli::try_parse_from(["valinor-cli", "get-edge", "valhalla.json", &input]) + .expect("the arguments parse"); + let Commands::GetEdge(args) = cli.command else { + panic!("expected the get-edge subcommand"); + }; + assert_eq!(args.graph_id, expected); + } + } + + #[test] + fn extract_tiles_are_required() { + assert!(Cli::try_parse_from(["valinor-cli", "extract", "in.tar", "out.tar"]).is_err()); + } } From 8db81fd2202f14f7035fc024e52cba7622cb7af6 Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Mon, 2 Nov 2026 22:05:46 -0800 Subject: [PATCH 22/37] Read and write the test tile so the writer tests run in Miri On Linux, fs::copy uses the copy_file_range system call, and Miri does not support it. Two writer tests copied a tile with fs::copy, so they failed in the Miri jobs. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/extract/writer.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/valhalla-graphtile/src/extract/writer.rs b/valhalla-graphtile/src/extract/writer.rs index 2b2c959..f1497b2 100644 --- a/valhalla-graphtile/src/extract/writer.rs +++ b/valhalla-graphtile/src/extract/writer.rs @@ -601,11 +601,10 @@ mod tests { let tile_path = base.path().join(&relative_path); std::fs::create_dir_all(tile_path.parent().expect("the tile path has a parent")) .expect("Unable to create the tile directory"); - std::fs::copy( - test_fixtures::andorra_tiles_dir().join(&relative_path), - &tile_path, - ) - .expect("Unable to copy the tile"); + // Miri does not support the `copy_file_range` call that `fs::copy` uses on Linux. + let bytes = std::fs::read(test_fixtures::andorra_tiles_dir().join(&relative_path)) + .expect("Unable to read the tile"); + std::fs::write(&tile_path, bytes).expect("Unable to write the tile"); (base, tile_path) } From 62d309be29d254168aaa6290ad0ac5bd192c0743 Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Tue, 3 Nov 2026 19:17:37 -0800 Subject: [PATCH 23/37] Gate the test items that only tarball tests use under Miri Miri does not support memory maps, so the tests that read a tarball do not run in Miri. Their imports and fixture paths gave unused warnings in the Miri jobs. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/test_fixtures.rs | 3 +++ valhalla-graphtile/src/tile_provider/tarball.rs | 2 ++ 2 files changed, 5 insertions(+) diff --git a/valhalla-graphtile/src/test_fixtures.rs b/valhalla-graphtile/src/test_fixtures.rs index d620b2a..bc1e96e 100644 --- a/valhalla-graphtile/src/test_fixtures.rs +++ b/valhalla-graphtile/src/test_fixtures.rs @@ -1,6 +1,7 @@ //! Paths to the test fixtures in the `fixtures` directory of this crate. //! //! All fixtures are made from the same OpenStreetMap extract of Andorra. +//! Miri does not support memory maps, so the tarball fixtures do not exist under Miri. use std::path::PathBuf; @@ -17,6 +18,7 @@ pub(crate) fn andorra_tiles_dir() -> PathBuf { /// The tiles of [`andorra_tiles_dir`] in an extract tarball, /// as `valhalla_build_extract` writes it. +#[cfg(not(miri))] pub(crate) fn andorra_tiles_tar() -> PathBuf { fixture("andorra-tiles.tar") } @@ -28,6 +30,7 @@ pub(crate) fn andorra_tiles_with_traffic_dir() -> PathBuf { } /// A live traffic tarball for the Andorra tiles. +#[cfg(not(miri))] pub(crate) fn andorra_traffic_tar() -> PathBuf { fixture("andorra-traffic.tar") } diff --git a/valhalla-graphtile/src/tile_provider/tarball.rs b/valhalla-graphtile/src/tile_provider/tarball.rs index aa5ac3a..862c49d 100644 --- a/valhalla-graphtile/src/tile_provider/tarball.rs +++ b/valhalla-graphtile/src/tile_provider/tarball.rs @@ -651,8 +651,10 @@ pub fn serialize_index_bin(entries: &[TileIndexBinEntry]) -> Result<&[u8], Inval #[cfg(test)] mod test { use super::*; + #[cfg(not(miri))] use crate::graph_tile::handles::GraphTileHandle; use crate::graph_tile::{GraphTile, GraphTileView}; + #[cfg(not(miri))] use crate::test_fixtures; use crate::tile_provider::DirectoryGraphTileProvider; use proptest::collection::vec; From 86c50a70703e3d87419f90e7b2133cd0cc5174b8 Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Tue, 3 Nov 2026 19:19:01 -0800 Subject: [PATCH 24/37] Move the writer tests that Miri cannot run to one module Five writer tests had their own cfg(not(miri)) gate, but the imports and the Andorra tile list that they use did not. So those items gave unused warnings in the Miri jobs. One gated module now holds them all. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/extract/writer.rs | 306 +++++++++++------------ 1 file changed, 153 insertions(+), 153 deletions(-) diff --git a/valhalla-graphtile/src/extract/writer.rs b/valhalla-graphtile/src/extract/writer.rs index f1497b2..b15807e 100644 --- a/valhalla-graphtile/src/extract/writer.rs +++ b/valhalla-graphtile/src/extract/writer.rs @@ -240,13 +240,9 @@ mod tests { use crate::graph_tile::GraphTileView; use crate::graph_tile::handles::OwnedGraphTileHandle; use crate::test_fixtures; - use crate::tile_provider::{DirectoryGraphTileProvider, TarballTileProvider, parse_index_bin}; + use crate::tile_provider::{DirectoryGraphTileProvider, parse_index_bin}; use geo::{CoordFloat, Point}; use num_traits::FromPrimitive; - use proptest::collection::vec; - use proptest::prelude::ProptestConfig; - use proptest::sample::select; - use proptest::{prop_assert, prop_assert_eq, proptest}; use std::collections::{HashMap, HashSet}; use std::io::Read; use std::num::NonZeroUsize; @@ -254,17 +250,6 @@ mod tests { use std::sync::Arc; use tar::Archive; - /// The tiles of the Andorra fixtures, in archive order. - const ANDORRA_TILES: [(u8, u64); 7] = [ - (0, 3015), - (1, 47_701), - (2, 762_485), - (2, 762_486), - (2, 763_925), - (2, 763_926), - (2, 763_927), - ]; - /// The smallest Andorra tile, which keeps the tests that run under Miri fast. const SMALL_TILE: (u8, u64) = (2, 763_927); @@ -272,10 +257,6 @@ mod tests { GraphId::try_from_components(level, tile_id, 0).expect("valid tile ID in test") } - fn andorra_tiles() -> Vec { - ANDORRA_TILES.into_iter().map(tile).collect() - } - fn directory_provider(base: &Path) -> DirectoryGraphTileProvider { DirectoryGraphTileProvider::new(base.to_path_buf(), NonZeroUsize::MIN) } @@ -673,154 +654,173 @@ mod tests { ); } - // Miri is too slow to decode all of the Andorra tiles. + // Miri is too slow to decode all of the Andorra tiles, and it does not support memory maps. #[cfg(not(miri))] - #[test] - fn test_write_all_andorra_tiles() { - let provider = TestProvider::new(); - let mut archive_bytes = Vec::new(); - let written = write_extract(&provider, andorra_tiles(), &mut archive_bytes) - .expect("the extract writes cleanly"); - - assert_eq!( - written, - archive_bytes.len() as u64, - "The return value must be the number of bytes written" - ); - check_archive( - &archive_bytes, - &andorra_tiles(), - &test_fixtures::andorra_tiles_dir(), - ); - } + mod all_andorra_tiles { + use super::*; + use crate::tile_provider::TarballTileProvider; + use proptest::collection::vec; + use proptest::prelude::ProptestConfig; + use proptest::sample::select; + use proptest::{prop_assert, prop_assert_eq, proptest}; + + /// The tiles of the Andorra fixtures, in archive order. + const ANDORRA_TILES: [(u8, u64); 7] = [ + (0, 3015), + (1, 47_701), + (2, 762_485), + (2, 762_486), + (2, 763_925), + (2, 763_926), + (2, 763_927), + ]; - // Miri is too slow to decode all of the Andorra tiles. - #[cfg(not(miri))] - #[test] - fn test_order_and_duplicates_do_not_change_output() { - let provider = TestProvider::new(); - let mut in_order = Vec::new(); - write_extract(&provider, andorra_tiles(), &mut in_order).expect("first write succeeds"); - - // Reverse order, with every tile twice: once as a base ID and once with a feature index. - let scrambled = andorra_tiles().into_iter().rev().flat_map(|graph_id| { - let with_feature_index = graph_id - .with_feature_index(42) - .expect("valid feature index"); - [graph_id, with_feature_index] - }); - let mut from_scrambled = Vec::new(); - write_extract(&provider, scrambled, &mut from_scrambled).expect("second write succeeds"); + fn andorra_tiles() -> Vec { + ANDORRA_TILES.into_iter().map(tile).collect() + } - assert!( - in_order == from_scrambled, - "The order of the tile IDs, duplicates, and feature indexes must not change the output" - ); - } + #[test] + fn test_write_all_andorra_tiles() { + let provider = TestProvider::new(); + let mut archive_bytes = Vec::new(); + let written = write_extract(&provider, andorra_tiles(), &mut archive_bytes) + .expect("the extract writes cleanly"); - // Tragically, memory maps are not supported by Miri! - #[cfg(not(miri))] - #[test] - fn test_andorra_round_trip() { - let source_provider = TarballTileProvider::new_readonly(test_fixtures::andorra_tiles_tar()) - .expect("Unable to init the source tile provider"); - - // Out of archive order, and with one nonzero feature index. - let tile_ids = [ - tile((2, 763_926)), - tile((0, 3015)), - GraphId::try_from_components(1, 47_701, 42).expect("valid graph ID in test"), - tile((2, 762_485)), - ]; - let excluded = tile((2, 763_925)); - assert!( - source_provider.tile_size(excluded).is_ok(), - "The excluded tile must be in the source" - ); + assert_eq!( + written, + archive_bytes.len() as u64, + "The return value must be the number of bytes written" + ); + check_archive( + &archive_bytes, + &andorra_tiles(), + &test_fixtures::andorra_tiles_dir(), + ); + } - let mut archive_bytes = Vec::new(); - write_extract(&source_provider, tile_ids, &mut archive_bytes) - .expect("the extract writes cleanly"); + #[test] + fn test_order_and_duplicates_do_not_change_output() { + let provider = TestProvider::new(); + let mut in_order = Vec::new(); + write_extract(&provider, andorra_tiles(), &mut in_order).expect("first write succeeds"); + + // Reverse order, with every tile twice: once as a base ID and once with a feature index. + let scrambled = andorra_tiles().into_iter().rev().flat_map(|graph_id| { + let with_feature_index = graph_id + .with_feature_index(42) + .expect("valid feature index"); + [graph_id, with_feature_index] + }); + let mut from_scrambled = Vec::new(); + write_extract(&provider, scrambled, &mut from_scrambled) + .expect("second write succeeds"); - let mut archive_file = tempfile::NamedTempFile::new().expect("a temporary file is created"); - archive_file - .write_all(&archive_bytes) - .expect("the extract is written to disk"); - let extract_provider = TarballTileProvider::new_readonly(archive_file.path()) - .expect("the generated extract opens cleanly"); - - for graph_id in tile_ids { - let extract_tile = extract_provider - .get_handle_for_tile_containing(graph_id) - .expect("the tile is in the extract"); - let source_tile = source_provider - .get_handle_for_tile_containing(graph_id) - .expect("the tile is in the source"); assert!( - extract_tile.tile_bytes() == source_tile.tile_bytes(), - "Tile {graph_id} must have the same bytes in the extract and in the source" + in_order == from_scrambled, + "The order of the tile IDs, duplicates, and feature indexes must not change the output" ); } - assert!( - matches!( - extract_provider.tile_size(excluded), - Err(GraphTileProviderError::TileDoesNotExist) - ), - "A tile that was not requested must not be in the extract" - ); - } - - // Tragically, memory maps are not supported by Miri! - #[cfg(not(miri))] - #[test] - fn test_directory_and_tarball_providers_write_the_same_extract() { - // The two fixtures contain the same tiles. - let tarball = TarballTileProvider::new_readonly(test_fixtures::andorra_tiles_tar()) - .expect("Unable to init the tarball tile provider"); - let directory = directory_provider(&test_fixtures::andorra_tiles_dir()); - - let mut from_tarball = Vec::new(); - write_extract(&tarball, andorra_tiles(), &mut from_tarball) - .expect("the extract writes cleanly from the tarball"); - let mut from_directory = Vec::new(); - write_extract(&directory, andorra_tiles(), &mut from_directory) - .expect("the extract writes cleanly from the directory"); - assert!( - from_tarball == from_directory, - "The tarball and the directory of the same tiles must give the same extract" - ); - } - - // Miri is too slow to decode the Andorra tiles in many cases. - #[cfg(not(miri))] - proptest! { - #![proptest_config(ProptestConfig::with_cases(32))] #[test] - fn test_archive_matches_independent_reader( - // Andorra tiles in any order, with duplicates and feature indexes. - tile_ids in vec((select(&ANDORRA_TILES[..]), 0u64..4), 1..12) - ) { - let tile_ids: Vec = tile_ids - .into_iter() - .map(|(tile_key, feature_index)| { - tile(tile_key) - .with_feature_index(feature_index) - .expect("valid feature index") - }) - .collect(); - let mut expected_tiles: Vec = - tile_ids.iter().map(GraphId::tile_base_id).collect(); - expected_tiles.sort_unstable_by_key(|graph_id| (graph_id.level(), graph_id.tile_id())); - expected_tiles.dedup(); + fn test_andorra_round_trip() { + let source_provider = + TarballTileProvider::new_readonly(test_fixtures::andorra_tiles_tar()) + .expect("Unable to init the source tile provider"); + + // Out of archive order, and with one nonzero feature index. + let tile_ids = [ + tile((2, 763_926)), + tile((0, 3015)), + GraphId::try_from_components(1, 47_701, 42).expect("valid graph ID in test"), + tile((2, 762_485)), + ]; + let excluded = tile((2, 763_925)); + assert!( + source_provider.tile_size(excluded).is_ok(), + "The excluded tile must be in the source" + ); let mut archive_bytes = Vec::new(); - let written = write_extract(&TestProvider::new(), tile_ids, &mut archive_bytes) + write_extract(&source_provider, tile_ids, &mut archive_bytes) .expect("the extract writes cleanly"); - prop_assert_eq!(written, archive_bytes.len() as u64); - prop_assert!(!expected_tiles.is_empty()); - check_archive(&archive_bytes, &expected_tiles, &test_fixtures::andorra_tiles_dir()); + let mut archive_file = + tempfile::NamedTempFile::new().expect("a temporary file is created"); + archive_file + .write_all(&archive_bytes) + .expect("the extract is written to disk"); + let extract_provider = TarballTileProvider::new_readonly(archive_file.path()) + .expect("the generated extract opens cleanly"); + + for graph_id in tile_ids { + let extract_tile = extract_provider + .get_handle_for_tile_containing(graph_id) + .expect("the tile is in the extract"); + let source_tile = source_provider + .get_handle_for_tile_containing(graph_id) + .expect("the tile is in the source"); + assert!( + extract_tile.tile_bytes() == source_tile.tile_bytes(), + "Tile {graph_id} must have the same bytes in the extract and in the source" + ); + } + assert!( + matches!( + extract_provider.tile_size(excluded), + Err(GraphTileProviderError::TileDoesNotExist) + ), + "A tile that was not requested must not be in the extract" + ); + } + + #[test] + fn test_directory_and_tarball_providers_write_the_same_extract() { + // The two fixtures contain the same tiles. + let tarball = TarballTileProvider::new_readonly(test_fixtures::andorra_tiles_tar()) + .expect("Unable to init the tarball tile provider"); + let directory = directory_provider(&test_fixtures::andorra_tiles_dir()); + + let mut from_tarball = Vec::new(); + write_extract(&tarball, andorra_tiles(), &mut from_tarball) + .expect("the extract writes cleanly from the tarball"); + let mut from_directory = Vec::new(); + write_extract(&directory, andorra_tiles(), &mut from_directory) + .expect("the extract writes cleanly from the directory"); + assert!( + from_tarball == from_directory, + "The tarball and the directory of the same tiles must give the same extract" + ); + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(32))] + + #[test] + fn test_archive_matches_independent_reader( + // Andorra tiles in any order, with duplicates and feature indexes. + tile_ids in vec((select(&ANDORRA_TILES[..]), 0u64..4), 1..12) + ) { + let tile_ids: Vec = tile_ids + .into_iter() + .map(|(tile_key, feature_index)| { + tile(tile_key) + .with_feature_index(feature_index) + .expect("valid feature index") + }) + .collect(); + let mut expected_tiles: Vec = + tile_ids.iter().map(GraphId::tile_base_id).collect(); + expected_tiles.sort_unstable_by_key(|graph_id| (graph_id.level(), graph_id.tile_id())); + expected_tiles.dedup(); + + let mut archive_bytes = Vec::new(); + let written = write_extract(&TestProvider::new(), tile_ids, &mut archive_bytes) + .expect("the extract writes cleanly"); + + prop_assert_eq!(written, archive_bytes.len() as u64); + prop_assert!(!expected_tiles.is_empty()); + check_archive(&archive_bytes, &expected_tiles, &test_fixtures::andorra_tiles_dir()); + } } } } From 48adbfe180505762bac31a6af4ecf057949f96ff Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Tue, 3 Nov 2026 19:30:19 -0800 Subject: [PATCH 25/37] Reword the doc comments of the provider methods Split the long sentences and use active verbs. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/graph_tile/handles.rs | 6 +++--- valhalla-graphtile/src/test_fixtures.rs | 2 +- valhalla-graphtile/src/tile_provider/mod.rs | 4 ++-- valhalla-graphtile/src/tile_provider/tarball.rs | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/valhalla-graphtile/src/graph_tile/handles.rs b/valhalla-graphtile/src/graph_tile/handles.rs index cb6b488..47b24ba 100644 --- a/valhalla-graphtile/src/graph_tile/handles.rs +++ b/valhalla-graphtile/src/graph_tile/handles.rs @@ -22,7 +22,7 @@ pub trait GraphTileHandle { /// Returns the bytes of the tile, exactly as the handle got them. /// - /// The view from [`GraphTileHandle::tile_view`] was decoded from these bytes, + /// The handle decoded the view of [`GraphTileHandle::tile_view`] from these bytes, /// so they are a valid graph tile. fn tile_bytes(&self) -> &[u8]; } @@ -317,8 +317,8 @@ impl GraphTileHandle for MmapGraphTileHandle { #[inline] fn tile_bytes(&self) -> &[u8] { - // SAFETY: The decoded view borrows this same range of the memory map, - // so the caller of `MmapGraphTileHandle::try_from` already accepted + // SAFETY: The decoded view borrows this same range of the memory map. + // The caller of `MmapGraphTileHandle::try_from` already accepted // the conditions of `MmapTilePointer::as_tile_bytes` for the life of the handle. unsafe { self.borrow_owner().as_tile_bytes() } } diff --git a/valhalla-graphtile/src/test_fixtures.rs b/valhalla-graphtile/src/test_fixtures.rs index bc1e96e..40872a6 100644 --- a/valhalla-graphtile/src/test_fixtures.rs +++ b/valhalla-graphtile/src/test_fixtures.rs @@ -1,6 +1,6 @@ //! Paths to the test fixtures in the `fixtures` directory of this crate. //! -//! All fixtures are made from the same OpenStreetMap extract of Andorra. +//! All fixtures come from the same OpenStreetMap extract of Andorra. //! Miri does not support memory maps, so the tarball fixtures do not exist under Miri. use std::path::PathBuf; diff --git a/valhalla-graphtile/src/tile_provider/mod.rs b/valhalla-graphtile/src/tile_provider/mod.rs index f2f3995..cc502ca 100644 --- a/valhalla-graphtile/src/tile_provider/mod.rs +++ b/valhalla-graphtile/src/tile_provider/mod.rs @@ -121,8 +121,8 @@ pub trait GraphTileProvider { /// /// # Performance /// - /// Implementations SHOULD get the size from an index or from file metadata, - /// so that a caller can get the sizes of many tiles before it reads any of them. + /// Implementations SHOULD get the size from an index or from file metadata. + /// A caller can then get the sizes of many tiles before it reads any of them. /// /// # Errors /// diff --git a/valhalla-graphtile/src/tile_provider/tarball.rs b/valhalla-graphtile/src/tile_provider/tarball.rs index 862c49d..c56869b 100644 --- a/valhalla-graphtile/src/tile_provider/tarball.rs +++ b/valhalla-graphtile/src/tile_provider/tarball.rs @@ -152,7 +152,7 @@ impl TarballTileProvider { }; // Tile pointers slice the map without a bounds check, - // so the map must have the length that the index ranges were checked against. + // so the length of the map must equal the length that the index check used. if mmap.len() as u64 != archive_len { return Err(GraphTileProviderError::InvalidTarball(format!( "the archive changed size while it was opened ({archive_len} bytes, then {} bytes)", @@ -939,7 +939,7 @@ mod test { proptest! { #[test] fn test_index_bin_roundtrip( - // Levels are 3 bits, tile IDs are 22 bits, and offsets are counted in 512-byte blocks. + // Levels are 3 bits, tile IDs are 22 bits, and offsets count 512-byte blocks. // The offset bound keeps the end of every tile inside a u64. raw_entries in vec( (0u8..=7, 0u64..1 << 22, 1u64..=(u64::MAX - u64::from(u32::MAX)) / 512, 0u32..=u32::MAX), From 3747b45412da76d774b2d20d6b84bb2dccca282a Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Tue, 3 Nov 2026 19:30:19 -0800 Subject: [PATCH 26/37] Reword the doc comment of GraphId::parse_tile Split the long sentence and use active verbs. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/graph_id.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/valhalla-graphtile/src/graph_id.rs b/valhalla-graphtile/src/graph_id.rs index 5147c74..bd55d20 100644 --- a/valhalla-graphtile/src/graph_id.rs +++ b/valhalla-graphtile/src/graph_id.rs @@ -237,13 +237,13 @@ impl GraphId { /// Parses the base graph ID of a tile from `level/tile_id` (e.g. `2/762485`). /// - /// Whitespace around each component is ignored. + /// The parser ignores whitespace around each component. /// /// # Errors /// /// Returns an error if `input` does not have two `/`-separated components, - /// if a component is not a non-negative integer, - /// or if the level or the tile ID is out of range. + /// or if a component is not a non-negative integer. + /// It also returns an error if the level or the tile ID is out of range. pub fn parse_tile(input: &str) -> Result { let [level, tile_id] = parse_components(input, "level/tile_id", ["level", "tile ID"])?; components_to_graph_id(input, level, tile_id, 0) From ec4ccc94b23b2e8237126746f1beb6ce29be20c5 Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Tue, 3 Nov 2026 19:30:20 -0800 Subject: [PATCH 27/37] Reword the comments of the extract writer Remove the semicolons from the test comments and use active verbs. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/extract/writer.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/valhalla-graphtile/src/extract/writer.rs b/valhalla-graphtile/src/extract/writer.rs index b15807e..34d56cf 100644 --- a/valhalla-graphtile/src/extract/writer.rs +++ b/valhalla-graphtile/src/extract/writer.rs @@ -36,10 +36,10 @@ pub enum ExtractWriteError { /// A requested tile is not in the provider. #[error("tile {0} is not in the provider")] MissingTile(GraphId), - /// A requested graph ID cannot be converted into a tar entry path. + /// A requested graph ID has no tar entry path. #[error("invalid graph ID: {0}")] InvalidGraphId(#[from] InvalidGraphIdError), - /// A tile cannot be indexed. + /// A tile has no valid `index.bin` entry. #[error("invalid index.bin: {0}")] InvalidIndexBin(#[from] InvalidIndexBinError), /// The length of a tile differs from the size that the provider reported, @@ -84,7 +84,7 @@ impl ExtractWriteError { /// The writer copies each tile verbatim. /// The tiles are written in the standard order, /// so a given input set of tile IDs always produces the same output bytes. -/// Each tile is written once, even if `tile_ids` contains more than one graph ID in it. +/// The writer writes each tile once, even if `tile_ids` contains more than one graph ID in it. /// /// For a [`DirectoryGraphTileProvider`](crate::tile_provider::DirectoryGraphTileProvider), /// the tile files must not change during the write, @@ -429,11 +429,11 @@ mod tests { // The index entry (a header and 5 × 16 bytes) fills blocks 0 and 1. // Each payload starts one block after its header. let expected_offsets = [ - 3 * B, // header at block 2; payload is 1 block - 5 * B, // header at block 4; payload is padded to 1 block - 7 * B, // header at block 6; payload is padded to 2 blocks - 10 * B, // header at block 9; payload is 2 blocks - 13 * B, // header at block 12; payload is padded to 1 block + 3 * B, // header at block 2, payload is 1 block + 5 * B, // header at block 4, payload is padded to 1 block + 7 * B, // header at block 6, payload is padded to 2 blocks + 10 * B, // header at block 9, payload is 2 blocks + 13 * B, // header at block 12, payload is padded to 1 block ]; assert_eq!( From 9487e6d2d0a1c09c78a9910046ea3359162f2440 Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Tue, 3 Nov 2026 19:30:20 -0800 Subject: [PATCH 28/37] Reword the comments of the integration harness Split the long sentences, use active verbs, and remove two comments that repeat the code. Co-Authored-By: Claude Opus 5.5 --- scripts/integration/src/valinor_integration/cli.py | 1 - .../integration/src/valinor_integration/extract.py | 2 +- .../integration/src/valinor_integration/runner.py | 4 ++-- .../integration/src/valinor_integration/settings.py | 1 - .../integration/src/valinor_integration/valhalla.py | 13 +++++++------ 5 files changed, 10 insertions(+), 11 deletions(-) diff --git a/scripts/integration/src/valinor_integration/cli.py b/scripts/integration/src/valinor_integration/cli.py index b12cb7f..3021189 100644 --- a/scripts/integration/src/valinor_integration/cli.py +++ b/scripts/integration/src/valinor_integration/cli.py @@ -75,7 +75,6 @@ def main(argv: list[str] | None = None) -> int: # Remove the containers on SIGTERM (for example, a cancelled CI job), as on Ctrl-C. signal.signal(signal.SIGTERM, _interrupt) argv = sys.argv[1:] if argv is None else argv - # Make `run` the default command. if not argv or argv[0] not in (*COMMANDS, "-h", "--help"): argv = ["run", *argv] args = _parser().parse_args(argv) diff --git a/scripts/integration/src/valinor_integration/extract.py b/scripts/integration/src/valinor_integration/extract.py index 2b552a8..4557313 100644 --- a/scripts/integration/src/valinor_integration/extract.py +++ b/scripts/integration/src/valinor_integration/extract.py @@ -16,7 +16,7 @@ def read_tile_list(path: Path) -> list[str]: """Reads a tile list: one ``level/tile_id`` on each line. - ``#`` starts a comment, and blank lines are ignored. + ``#`` starts a comment, and the reader skips blank lines. ``valinor-cli`` checks the form of each tile. """ tiles = [] diff --git a/scripts/integration/src/valinor_integration/runner.py b/scripts/integration/src/valinor_integration/runner.py index d612a4e..ba01a17 100644 --- a/scripts/integration/src/valinor_integration/runner.py +++ b/scripts/integration/src/valinor_integration/runner.py @@ -45,7 +45,7 @@ def run(settings: Settings, inputs: Inputs) -> bool: """Serves both extracts, sends every request to both, and compares the responses. Returns ``True`` if every check passes. - The responses, the logs, and the config are saved in the work directory. + The work directory keeps the responses, the logs, and the config. """ collection = _load_checked_collection(inputs.collection) client, config = _prepare(settings, inputs) @@ -66,7 +66,7 @@ def run(settings: Settings, inputs: Inputs) -> bool: for name, server in servers.items(): responses[name] = _send_all(server, collection, responses_dir / name, failures) - # Capture the logs before the containers are removed. + # Capture the logs before the exit removes the containers. for name, server in servers.items(): logs = server.logs() log_file = inputs.workdir / f"valhalla-{name}.log" diff --git a/scripts/integration/src/valinor_integration/settings.py b/scripts/integration/src/valinor_integration/settings.py index 895943f..f9badd3 100644 --- a/scripts/integration/src/valinor_integration/settings.py +++ b/scripts/integration/src/valinor_integration/settings.py @@ -17,7 +17,6 @@ LATEST_TAG = "latest" DEFAULT_SERVE_PORT = 8002 -# This file is scripts/integration/src/valinor_integration/settings.py. HARNESS_DIR = Path(__file__).resolve().parents[2] REPO_ROOT = HARNESS_DIR.parents[1] diff --git a/scripts/integration/src/valinor_integration/valhalla.py b/scripts/integration/src/valinor_integration/valhalla.py index 4f8a7ff..3ebd7e2 100644 --- a/scripts/integration/src/valinor_integration/valhalla.py +++ b/scripts/integration/src/valinor_integration/valhalla.py @@ -37,8 +37,8 @@ def connect() -> docker.DockerClient: """Connects to the Docker daemon. - If the default connection fails, this uses the endpoint of the current Docker CLI context, - which docker-py does not read (for example, the socket of colima). + If the default connection fails, this uses the current Docker CLI context + (for example, the socket of colima). docker-py does not read that context. """ try: client = docker.from_env() @@ -110,8 +110,8 @@ def build_config(client: docker.DockerClient, image: str) -> dict[str, Any]: class ValhallaServer: """A ``valhalla_service`` container which serves one extract. - Use it as a context manager: the container starts on entry, - and is removed on exit (also after an error or Ctrl-C). + Use it as a context manager: entry starts the container, + and exit removes it (also after an error or Ctrl-C). Call ``logs`` before the exit to keep the logs. """ @@ -161,8 +161,9 @@ def _start(self) -> None: command=[CONTAINER_CONFIG, "1"], ports={f"{SERVICE_PORT}/tcp": self.host_port}, ) - # Copy the data in place of a bind mount, which fails for a directory - # that the Docker VM does not share (such as /tmp with colima). + # Copy the data in place of a bind mount. + # A bind mount fails for a directory that the Docker VM does not share + # (such as /tmp with colima). with tempfile.TemporaryFile() as archive: self._write_data_archive(archive) archive.seek(0) From 5e09de8ad4c7bc7955eb6801b7b119ffca830b04 Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Tue, 3 Nov 2026 20:17:41 -0800 Subject: [PATCH 29/37] Get tile bytes from the graph tile view GraphTileView keeps the slice that it decoded, and tile_bytes returns that slice with no copy. Decoding fails when bytes remain after the last section, so the slice is always exactly one tile. A caller of with_tile_containing can now get the bytes of a tile from the view. GraphTileProvider::TileHandle does not need a GraphTileHandle bound, and the memory-map handle does not need a second unsafe call to as_tile_bytes. GraphTileHandle is the same as on main. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/graph_tile.rs | 31 ++++++++++++ valhalla-graphtile/src/graph_tile/builder.rs | 1 + valhalla-graphtile/src/graph_tile/handles.rs | 50 ------------------- .../src/tile_provider/directory.rs | 11 ++-- valhalla-graphtile/src/tile_provider/mod.rs | 5 +- .../src/tile_provider/tarball.rs | 17 +++---- 6 files changed, 47 insertions(+), 68 deletions(-) diff --git a/valhalla-graphtile/src/graph_tile.rs b/valhalla-graphtile/src/graph_tile.rs index 7f1d375..e7199b6 100644 --- a/valhalla-graphtile/src/graph_tile.rs +++ b/valhalla-graphtile/src/graph_tile.rs @@ -460,6 +460,8 @@ pub struct GraphNode<'a> { /// Cloning this struct is cheap, since the internal types are all references. #[derive(Clone)] pub struct GraphTileView<'a> { + /// All bytes of the tile, from the start of the header to the end of the last section. + tile_bytes: &'a [u8], /// Header with various metadata about the tile and internal sizes. header: &'a GraphTileHeader, /// The list of nodes in the graph tile. @@ -491,6 +493,16 @@ pub struct GraphTileView<'a> { // TODO: Operator one stops(?) } +impl<'a> GraphTileView<'a> { + /// Returns the bytes that this view decoded, with no copy. + /// + /// The slice is exactly one full tile, with no trailing bytes. + #[inline] + pub fn tile_bytes(&self) -> &'a [u8] { + self.tile_bytes + } +} + impl GraphTile for GraphTileView<'_> { #[inline] fn graph_id(&self) -> GraphId { @@ -667,6 +679,7 @@ impl<'a> TryFrom<&'a [u8]> for GraphTileView<'a> { type Error = GraphTileDecodingError; fn try_from(bytes: &'a [u8]) -> Result { + let tile_bytes = bytes; let (header, bytes) = GraphTileHeader::ref_from_prefix(bytes).map_err(|e| { GraphTileDecodingError::CastError { field: "header".to_string(), @@ -884,6 +897,7 @@ impl<'a> TryFrom<&'a [u8]> for GraphTileView<'a> { if bytes.is_empty() { Ok(Self { + tile_bytes, header, nodes, transitions, @@ -985,6 +999,23 @@ mod tests { use enumset::{EnumSet, enum_set}; use rand::{RngExt, rng}; + #[test] + fn test_tile_bytes_is_the_decoded_slice() { + for tile in [ + &*TEST_GRAPH_TILE_L0, + &*TEST_GRAPH_TILE_L2, + &*TEST_GRAPH_TILE_WITH_FLOW, + ] { + let decoded = tile.borrow_owner().as_slice(); + let tile_bytes = tile.borrow_dependent().tile_bytes(); + assert!( + core::ptr::eq(tile_bytes, decoded), + "The view of {} must return the slice that it decoded, with no copy", + tile.graph_id() + ); + } + } + #[test] fn test_get_opp_edge_index() { let tile = &*TEST_GRAPH_TILE_L0; diff --git a/valhalla-graphtile/src/graph_tile/builder.rs b/valhalla-graphtile/src/graph_tile/builder.rs index 348fd20..6df6647 100644 --- a/valhalla-graphtile/src/graph_tile/builder.rs +++ b/valhalla-graphtile/src/graph_tile/builder.rs @@ -129,6 +129,7 @@ fn build_speed_profile_dedup_map(memory: &[I16]) -> HashMap<[i16; COEFFICIEN impl<'a> From<&'a OwnedGraphTileHandle> for GraphTileBuilder<'a> { fn from(value: &'a OwnedGraphTileHandle) -> Self { let GraphTileView { + tile_bytes: _, header, nodes, transitions, diff --git a/valhalla-graphtile/src/graph_tile/handles.rs b/valhalla-graphtile/src/graph_tile/handles.rs index 47b24ba..54855f2 100644 --- a/valhalla-graphtile/src/graph_tile/handles.rs +++ b/valhalla-graphtile/src/graph_tile/handles.rs @@ -15,31 +15,8 @@ use memmap2::MmapRaw; use self_cell::self_cell; use std::sync::Arc; -/// An owned handle to a decoded graph tile, which keeps the bytes that it decoded. pub trait GraphTileHandle { - /// Returns a view of the decoded tile. fn tile_view(&'_ self) -> GraphTileView<'_>; - - /// Returns the bytes of the tile, exactly as the handle got them. - /// - /// The handle decoded the view of [`GraphTileHandle::tile_view`] from these bytes, - /// so they are a valid graph tile. - fn tile_bytes(&self) -> &[u8]; -} - -impl GraphTileHandle for Arc -where - T: GraphTileHandle, -{ - #[inline] - fn tile_view(&'_ self) -> GraphTileView<'_> { - (**self).tile_view() - } - - #[inline] - fn tile_bytes(&self) -> &[u8] { - (**self).tile_bytes() - } } self_cell! { @@ -61,18 +38,6 @@ impl TryFrom> for OwnedGraphTileHandle { } } -impl GraphTileHandle for OwnedGraphTileHandle { - #[inline] - fn tile_view(&'_ self) -> GraphTileView<'_> { - self.borrow_dependent().clone() - } - - #[inline] - fn tile_bytes(&self) -> &[u8] { - self.borrow_owner() - } -} - impl GraphTile for OwnedGraphTileHandle { #[inline] fn graph_id(&self) -> GraphId { @@ -309,21 +274,6 @@ impl MmapGraphTileHandle { } } -impl GraphTileHandle for MmapGraphTileHandle { - #[inline] - fn tile_view(&'_ self) -> GraphTileView<'_> { - self.borrow_dependent().clone() - } - - #[inline] - fn tile_bytes(&self) -> &[u8] { - // SAFETY: The decoded view borrows this same range of the memory map. - // The caller of `MmapGraphTileHandle::try_from` already accepted - // the conditions of `MmapTilePointer::as_tile_bytes` for the life of the handle. - unsafe { self.borrow_owner().as_tile_bytes() } - } -} - /// An encapsulating type that can be used to get the actual bytes for a tile. /// /// This provides a single type that can "own" the backing bytes for tile memory, diff --git a/valhalla-graphtile/src/tile_provider/directory.rs b/valhalla-graphtile/src/tile_provider/directory.rs index 9ea4b3b..50ad14b 100644 --- a/valhalla-graphtile/src/tile_provider/directory.rs +++ b/valhalla-graphtile/src/tile_provider/directory.rs @@ -180,7 +180,7 @@ impl GraphTileProvider for DirectoryGraphTileProvider { /// /// The size is the size of the file at the time of the call. /// If the file changes after the provider caches the tile, - /// the size and the cached handle do not agree. + /// the size and the cached tile do not agree. fn tile_size(&self, graph_id: GraphId) -> Result { let base_graph_id = graph_id.tile_base_id(); let size = match std::fs::metadata(self.path_for_graph_id(base_graph_id)?) { @@ -224,7 +224,6 @@ mod test { use super::DirectoryGraphTileProvider; use crate::GraphId; use crate::graph_tile::GraphTile; - use crate::graph_tile::handles::GraphTileHandle; use crate::test_fixtures; use crate::tile_hierarchy::STANDARD_LEVELS; use crate::tile_provider::{GraphTileProvider, GraphTileProviderError}; @@ -456,7 +455,7 @@ mod test { } #[test] - fn test_tile_size_matches_handle_bytes() { + fn test_tile_size_matches_view_bytes() { let provider = DirectoryGraphTileProvider::new(test_fixtures::andorra_tiles_dir(), NonZeroUsize::MIN); // The feature index is not zero, so the provider must use the base ID of the tile. @@ -465,12 +464,12 @@ mod test { let size = provider .tile_size(graph_id) .expect("Unable to get the tile size"); - let tile = provider - .get_handle_for_tile_containing(graph_id) + let tile_bytes_len = provider + .with_tile_containing(graph_id, |tile| tile.tile_bytes().len()) .expect("Unable to get tile"); assert_eq!( usize::try_from(size).expect("The size fits in a usize"), - tile.tile_bytes().len(), + tile_bytes_len, "The tile size must be the length of the tile bytes" ); } diff --git a/valhalla-graphtile/src/tile_provider/mod.rs b/valhalla-graphtile/src/tile_provider/mod.rs index cc502ca..ef3dda0 100644 --- a/valhalla-graphtile/src/tile_provider/mod.rs +++ b/valhalla-graphtile/src/tile_provider/mod.rs @@ -19,7 +19,6 @@ mod tarball; mod traffic; use crate::graph_id::InvalidGraphIdError; -use crate::graph_tile::handles::GraphTileHandle; use crate::graph_tile::{ DirectedEdge, GraphTile, GraphTileDecodingError, GraphTileView, LookupError, NodeInfo, OpposingEdgeIndex, @@ -69,7 +68,7 @@ pub trait GraphTileProvider { /// The tile handle type for owned handles. /// /// Code using the `GraphTileProvider` may rely on the handles being cheaply cloneable. - type TileHandle: GraphTile + GraphTileHandle + Clone; + type TileHandle: GraphTile + Clone; /// Gets the tile containing the given graph ID, /// and does some work in a closure which takes the reference as a parameter. @@ -117,7 +116,7 @@ pub trait GraphTileProvider { /// without reading or decoding the tile. /// /// While the tile data does not change, the size is the length of - /// [`GraphTileHandle::tile_bytes`] for the handle of the same tile. + /// [`GraphTileView::tile_bytes`] for the same tile. /// /// # Performance /// diff --git a/valhalla-graphtile/src/tile_provider/tarball.rs b/valhalla-graphtile/src/tile_provider/tarball.rs index c56869b..891894e 100644 --- a/valhalla-graphtile/src/tile_provider/tarball.rs +++ b/valhalla-graphtile/src/tile_provider/tarball.rs @@ -651,8 +651,6 @@ pub fn serialize_index_bin(entries: &[TileIndexBinEntry]) -> Result<&[u8], Inval #[cfg(test)] mod test { use super::*; - #[cfg(not(miri))] - use crate::graph_tile::handles::GraphTileHandle; use crate::graph_tile::{GraphTile, GraphTileView}; #[cfg(not(miri))] use crate::test_fixtures; @@ -1043,30 +1041,31 @@ mod test { }; assert_eq!(directory_tile.borrow_owner(), immutable_tarball_tile_bytes); - let tarball_handle = tarball_provider - .get_handle_for_tile_containing(*graph_id) - .expect("Unable to get tile"); for (provider_name, tile_bytes, tile_size) in [ ( "directory", - directory_tile.tile_bytes(), + directory_provider + .with_tile_containing(*graph_id, |tile| tile.tile_bytes().to_vec()), directory_provider.tile_size(*graph_id), ), ( "tarball", - tarball_handle.tile_bytes(), + tarball_provider + .with_tile_containing(*graph_id, |tile| tile.tile_bytes().to_vec()), tarball_provider.tile_size(*graph_id), ), ( "immutable tarball", - immutable_tarball_tile_pointer.tile_bytes(), + immutable_tarball_provider + .with_tile_containing(*graph_id, |tile| tile.tile_bytes().to_vec()), immutable_tarball_provider.tile_size(*graph_id), ), ] { + let tile_bytes = tile_bytes.expect("Unable to get tile"); assert_eq!( tile_bytes, directory_tile.borrow_owner().as_slice(), - "The {provider_name} handle must give the bytes of the tile file for {graph_id}" + "The {provider_name} view must give the bytes of the tile file for {graph_id}" ); let tile_size = tile_size.expect("Unable to get the tile size"); assert_eq!( From fecea8fa3726c65244a98091811ba4cfc11b57e0 Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Tue, 3 Nov 2026 20:17:41 -0800 Subject: [PATCH 30/37] Reword two doc comments in the tile providers The tarball path functions in test_fixtures are not compiled under Miri. The files are still there. Split the check_range doc comment at a semantic line break. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/test_fixtures.rs | 2 +- valhalla-graphtile/src/tile_provider/tarball.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/valhalla-graphtile/src/test_fixtures.rs b/valhalla-graphtile/src/test_fixtures.rs index 40872a6..44387e5 100644 --- a/valhalla-graphtile/src/test_fixtures.rs +++ b/valhalla-graphtile/src/test_fixtures.rs @@ -1,7 +1,7 @@ //! Paths to the test fixtures in the `fixtures` directory of this crate. //! //! All fixtures come from the same OpenStreetMap extract of Andorra. -//! Miri does not support memory maps, so the tarball fixtures do not exist under Miri. +//! Miri does not support memory maps, so the tarball paths are not compiled under Miri. use std::path::PathBuf; diff --git a/valhalla-graphtile/src/tile_provider/tarball.rs b/valhalla-graphtile/src/tile_provider/tarball.rs index 891894e..41f9212 100644 --- a/valhalla-graphtile/src/tile_provider/tarball.rs +++ b/valhalla-graphtile/src/tile_provider/tarball.rs @@ -580,7 +580,8 @@ fn check_offset(graph_id: GraphId, offset: u64) -> Result<(), InvalidIndexBinErr Ok(()) } -/// Makes sure that the tile at `offset` with `size` bytes lies inside an archive of `archive_len` bytes. +/// Makes sure that the tile at `offset` with `size` bytes +/// lies inside an archive of `archive_len` bytes. fn check_range( graph_id: GraphId, offset: u64, From f25708b531d6ff10c2ae43a529463efebbd74fe3 Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Tue, 3 Nov 2026 21:09:47 -0800 Subject: [PATCH 31/37] Keep GraphId::file_path and drop GraphId::relative_path The extract writer can give the tar crate a PathBuf from file_path. On Windows, tar changes each backslash in an entry path to a slash, so a separate string path is not necessary. file_path keeps its code, but it gets the tiling system with STANDARD_LEVELS.get. For levels 4 to 7, it returns InvalidGraphIdError::Level and does not panic on an index out of bounds. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/graph_id.rs | 75 ++++++++++++------------------ 1 file changed, 31 insertions(+), 44 deletions(-) diff --git a/valhalla-graphtile/src/graph_id.rs b/valhalla-graphtile/src/graph_id.rs index bd55d20..62e02f8 100644 --- a/valhalla-graphtile/src/graph_id.rs +++ b/valhalla-graphtile/src/graph_id.rs @@ -249,17 +249,14 @@ impl GraphId { components_to_graph_id(input, level, tile_id, 0) } - /// Returns the relative path of the tile file for this graph ID, - /// with `/` separators on every platform (e.g. `2/000/762/485.gph`). - /// - /// Valhalla uses this path in tile directories and extract tarballs. - /// The feature index is not part of the path. + /// Constructs a relative path for the given tile. /// /// # Errors /// /// This will fail if the tile is invalid for this level of tiling. /// TODO: It seems like we could do this check at tile creation time? - pub fn relative_path(&self, extension: &str) -> Result { + pub fn file_path(&self, extension: &str) -> Result { + // This is quite ugly and prone to failure; this is a pretty literal C++ translation let level_number = self.level(); let level = if level_number == TRANSIT_LEVEL.level { &*TRANSIT_LEVEL @@ -275,31 +272,25 @@ impl GraphId { return Err(InvalidGraphIdError::GraphTileId); } - // The width of the largest tile ID of the level, rounded up to a whole number of groups. - let n_digits = (max_id.max(1).ilog10() + 1).next_multiple_of(3) as usize; - let padded_id = format!("{tile_id:0>n_digits$}"); - - let mut path = level_number.to_string(); - // The padded ID is ASCII, and its length is a multiple of 3. - for group_start in (0..n_digits).step_by(3) { - path.push('/'); - path.push_str(&padded_id[group_start..group_start + 3]); - } - if !extension.is_empty() { - path.push('.'); - path.push_str(extension); - } - - Ok(path) - } + let l = max_id.max(1).ilog10() + 1; + let rem = l % 3; + let n_digits = if rem == 0 { l } else { l + (3 - rem) }; + debug_assert!(n_digits % 3 == 0); + + // This may not be as efficient as the Valhalla implementation, but it's more readable. + // Format tile_id with leading zeros to match max_length + let padded_id = format!("{:0>width$}", tile_id, width = n_digits as usize); + let tile_id_chars: Vec<_> = padded_id.chars().collect(); + // Create the final path using groups of threes + let tile_id_component = tile_id_chars + .rchunks(3) + .fold(PathBuf::new(), |acc, chunk| { + PathBuf::from(chunk.iter().collect::()).join(acc) + }) + .with_extension(extension); - /// Returns [`GraphId::relative_path`] as a [`PathBuf`], with the separators of the platform. - /// - /// # Errors - /// - /// This will fail if the tile is invalid for this level of tiling. - pub fn file_path(&self, extension: &str) -> Result { - Ok(self.relative_path(extension)?.split('/').collect()) + // Build and return the final string + Ok(PathBuf::from(self.level().to_string()).join(tile_id_component)) } } @@ -549,7 +540,7 @@ mod tests { } #[test] - fn test_relative_path() { + fn test_file_path() { // Levels 0 and 1 pad the tile ID to 6 digits, and level 2 pads it to 9. for (graph_id, expected) in [ (tile(0, 3015), "0/003/015.gph"), @@ -559,7 +550,6 @@ mod tests { (tile(2, 762_485), "2/000/762/485.gph"), (tile(2, 1_036_799), "2/001/036/799.gph"), ] { - assert_eq!(graph_id.relative_path("gph").as_deref(), Ok(expected)); assert_eq!( graph_id.file_path("gph"), Ok(expected.split('/').collect::()) @@ -568,32 +558,29 @@ mod tests { let with_index = GraphId::try_from_components(2, 762_485, 42).unwrap(); assert_eq!( - with_index.relative_path("gph").as_deref(), - Ok("2/000/762/485.gph") + with_index.file_path("gph"), + Ok(["2", "000", "762", "485.gph"].iter().collect::()) ); assert_eq!( - tile(1, 47_701).relative_path("").as_deref(), - Ok("1/047/701") + tile(1, 47_701).file_path(""), + Ok(["1", "047", "701"].iter().collect::()) ); } #[test] - fn test_relative_path_invalid() { + fn test_file_path_invalid() { // Level 0 has 45 × 90 tiles, so its largest tile ID is 4049. assert_eq!( - tile(0, 4049).relative_path("gph").as_deref(), - Ok("0/004/049.gph") + tile(0, 4049).file_path("gph"), + Ok(["0", "004", "049.gph"].iter().collect::()) ); assert_eq!( - tile(0, 4050).relative_path("gph"), + tile(0, 4050).file_path("gph"), Err(InvalidGraphIdError::GraphTileId) ); // Levels 4 to 7 fit in the bit field, but they have no tiling system. - assert_eq!( - tile(5, 0).relative_path("gph"), - Err(InvalidGraphIdError::Level) - ); + assert_eq!(tile(5, 0).file_path("gph"), Err(InvalidGraphIdError::Level)); } #[test] From a47aaff31932aca0fc1a107268bfb7b20ce01d9a Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Tue, 3 Nov 2026 21:09:47 -0800 Subject: [PATCH 32/37] Say which range check GraphId::parse_tile does parse_tile checks that the level and the tile ID fit in a graph ID. It does not check the tiling grid of the level. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/graph_id.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/valhalla-graphtile/src/graph_id.rs b/valhalla-graphtile/src/graph_id.rs index 62e02f8..c859f96 100644 --- a/valhalla-graphtile/src/graph_id.rs +++ b/valhalla-graphtile/src/graph_id.rs @@ -243,7 +243,7 @@ impl GraphId { /// /// Returns an error if `input` does not have two `/`-separated components, /// or if a component is not a non-negative integer. - /// It also returns an error if the level or the tile ID is out of range. + /// It also returns an error if the level or the tile ID does not fit in a graph ID. pub fn parse_tile(input: &str) -> Result { let [level, tile_id] = parse_components(input, "level/tile_id", ["level", "tile ID"])?; components_to_graph_id(input, level, tile_id, 0) From ebdaccca177ee9b2cee317a9b4dd1c1ad363f82a Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Tue, 3 Nov 2026 21:26:37 -0800 Subject: [PATCH 33/37] Reword the comments of the extract writer The write_extract doc says that the extract is deterministic and drops the details of how the writer copies and orders the tiles. The test helper doc says that it reads the raw header path. Remove the comment on the end-of-archive writes, because the module doc lists the marker. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/extract/writer.rs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/valhalla-graphtile/src/extract/writer.rs b/valhalla-graphtile/src/extract/writer.rs index 313ebc5..75a2c8e 100644 --- a/valhalla-graphtile/src/extract/writer.rs +++ b/valhalla-graphtile/src/extract/writer.rs @@ -78,13 +78,8 @@ impl ExtractWriteError { } } -/// Writes an extract with the tiles that contain `tile_ids` from `provider` into `dest`, -/// and returns the number of bytes written. -/// -/// The writer copies each tile verbatim. -/// The tiles are written in the standard order, -/// so a given input set of tile IDs always produces the same output bytes. -/// The writer writes each tile once, even if `tile_ids` contains more than one graph ID in it. +/// Writes a deterministic extract with the tiles that contain `tile_ids` from `provider` +/// into `dest`, and returns the number of bytes written. /// /// For a [`DirectoryGraphTileProvider`](crate::tile_provider::DirectoryGraphTileProvider), /// the tile files must not change during the write, @@ -165,7 +160,6 @@ where .map_err(|source| ExtractWriteError::from_provider(graph_id, source))??; } - // Write the end of archive marker dest.write_all(&ZERO_BLOCK)?; dest.write_all(&ZERO_BLOCK)?; dest.flush()?; @@ -342,7 +336,7 @@ mod tests { } } - /// Returns the path and payload offset of each entry, as the `tar` crate reads them. + /// Returns the raw header path and the payload offset of each entry. fn read_entries(archive_bytes: &[u8]) -> Vec<(String, u64)> { Archive::new(archive_bytes) .entries() From 5b1d05f47685fcc4a1e80cb134af02bccc0c7b36 Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Tue, 3 Nov 2026 21:36:52 -0800 Subject: [PATCH 34/37] Find the smallest Andorra tile from the fixtures test_fixtures::smallest_andorra_tile walks the tile directory, takes the smallest tile file, and reads its graph ID from the tile header. The writer tests use it and do not hardcode the tile ID. check_archive compares the raw tar entry names with `/`-joined paths, so the writer tests do not need a literal tile name. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/extract/writer.rs | 59 ++++++++++++------------ valhalla-graphtile/src/test_fixtures.rs | 40 +++++++++++++++- 2 files changed, 69 insertions(+), 30 deletions(-) diff --git a/valhalla-graphtile/src/extract/writer.rs b/valhalla-graphtile/src/extract/writer.rs index 75a2c8e..52a1238 100644 --- a/valhalla-graphtile/src/extract/writer.rs +++ b/valhalla-graphtile/src/extract/writer.rs @@ -249,9 +249,6 @@ mod tests { use std::sync::Arc; use tar::Archive; - /// The smallest Andorra tile, which keeps the tests that run under Miri fast. - const SMALL_TILE: (u8, u64) = (2, 763_927); - fn tile((level, tile_id): (u8, u64)) -> GraphId { GraphId::try_from_components(level, tile_id, 0).expect("valid tile ID in test") } @@ -377,6 +374,7 @@ mod tests { let entries = read_entries(archive_bytes); let mut expected_paths = vec![INDEX_BIN_FILE_NAME.to_string()]; + // Tar entry names use `/` on every platform. expected_paths.extend(expected_tiles.iter().map(|graph_id| { let file_path = graph_id.file_path(TILE_FILE_EXTENSION).expect("valid tile"); let components: Vec<&str> = file_path @@ -505,7 +503,7 @@ mod tests { let mut dest = Vec::new(); assert!( matches!( - write_extract(&TestProvider::new(), [tile(SMALL_TILE), missing], &mut dest), + write_extract(&TestProvider::new(), [test_fixtures::smallest_andorra_tile(), missing], &mut dest), Err(ExtractWriteError::MissingTile(graph_id)) if graph_id == missing ), "A tile that is not in the provider must be a MissingTile error" @@ -516,16 +514,18 @@ mod tests { #[test] fn test_write_reports_size_errors_before_write() { let mut provider = TestProvider::new(); - provider.size_failures.insert(tile(SMALL_TILE)); + provider + .size_failures + .insert(test_fixtures::smallest_andorra_tile()); let mut dest = Vec::new(); assert!( matches!( - write_extract(&provider, [tile(SMALL_TILE)], &mut dest), + write_extract(&provider, [test_fixtures::smallest_andorra_tile()], &mut dest), Err(ExtractWriteError::Provider { graph_id, source: GraphTileProviderError::TileFetchError(_), - }) if graph_id == tile(SMALL_TILE) + }) if graph_id == test_fixtures::smallest_andorra_tile() ), "A failure to get a tile size must be a Provider error" ); @@ -538,15 +538,17 @@ mod tests { #[test] fn test_write_reports_read_errors() { let mut provider = TestProvider::new(); - provider.read_failures.insert(tile(SMALL_TILE)); + provider + .read_failures + .insert(test_fixtures::smallest_andorra_tile()); assert!( matches!( - write_extract(&provider, [tile(SMALL_TILE)], Vec::new()), + write_extract(&provider, [test_fixtures::smallest_andorra_tile()], Vec::new()), Err(ExtractWriteError::Provider { graph_id, source: GraphTileProviderError::TileFetchError(_), - }) if graph_id == tile(SMALL_TILE) + }) if graph_id == test_fixtures::smallest_andorra_tile() ), "A failure to read a tile must be a Provider error" ); @@ -556,20 +558,20 @@ mod tests { fn test_write_rejects_size_mismatch() { let mut provider = TestProvider::new(); let real_size = provider - .tile_size(tile(SMALL_TILE)) + .tile_size(test_fixtures::smallest_andorra_tile()) .expect("the tile has a size"); provider .reported_sizes - .insert(tile(SMALL_TILE), real_size + 1); + .insert(test_fixtures::smallest_andorra_tile(), real_size + 1); assert!( matches!( - write_extract(&provider, [tile(SMALL_TILE)], Vec::new()), + write_extract(&provider, [test_fixtures::smallest_andorra_tile()], Vec::new()), Err(ExtractWriteError::SizeMismatch { graph_id, expected, actual, - }) if graph_id == tile(SMALL_TILE) + }) if graph_id == test_fixtures::smallest_andorra_tile() && expected == real_size + 1 && actual == real_size as usize ), @@ -580,7 +582,7 @@ mod tests { /// Copies the small Andorra tile into a new tile directory. fn copy_small_tile() -> (tempfile::TempDir, std::path::PathBuf) { let base = tempfile::tempdir().expect("Unable to create a temporary directory"); - let relative_path = tile(SMALL_TILE) + let relative_path = test_fixtures::smallest_andorra_tile() .file_path(TILE_FILE_EXTENSION) .expect("valid tile"); let tile_path = base.path().join(&relative_path); @@ -598,7 +600,9 @@ mod tests { let (base, tile_path) = copy_small_tile(); let provider = directory_provider(base.path()); let cached_len = provider - .with_tile_containing(tile(SMALL_TILE), |tile| tile.tile_bytes().len()) + .with_tile_containing(test_fixtures::smallest_andorra_tile(), |tile| { + tile.tile_bytes().len() + }) .expect("Unable to get tile"); // The file grows, but the cache of the provider keeps the old bytes. @@ -610,7 +614,7 @@ mod tests { assert!( matches!( - write_extract(&provider, [tile(SMALL_TILE)], Vec::new()), + write_extract(&provider, [test_fixtures::smallest_andorra_tile()], Vec::new()), Err(ExtractWriteError::SizeMismatch { expected, actual, @@ -628,11 +632,11 @@ mod tests { assert!( matches!( - write_extract(&directory_provider(base.path()), [tile(SMALL_TILE)], Vec::new()), + write_extract(&directory_provider(base.path()), [test_fixtures::smallest_andorra_tile()], Vec::new()), Err(ExtractWriteError::Provider { graph_id, source: GraphTileProviderError::DecodingError(_), - }) if graph_id == tile(SMALL_TILE) + }) if graph_id == test_fixtures::smallest_andorra_tile() ), "An empty tile file must fail to decode" ); @@ -642,8 +646,12 @@ mod tests { fn test_write_small_tile() { let provider = TestProvider::new(); let mut archive_bytes = Vec::new(); - let written = write_extract(&provider, [tile(SMALL_TILE)], &mut archive_bytes) - .expect("the extract writes cleanly"); + let written = write_extract( + &provider, + [test_fixtures::smallest_andorra_tile()], + &mut archive_bytes, + ) + .expect("the extract writes cleanly"); assert_eq!( written, @@ -652,16 +660,9 @@ mod tests { ); check_archive( &archive_bytes, - &[tile(SMALL_TILE)], + &[test_fixtures::smallest_andorra_tile()], &test_fixtures::andorra_tiles_dir(), ); - - // Tar entry names use `/` on every platform. - let entry_paths: Vec = read_entries(&archive_bytes) - .into_iter() - .map(|(path, _)| path) - .collect(); - assert_eq!(entry_paths, ["index.bin", "2/000/763/927.gph"]); } // Miri is too slow to decode all of the Andorra tiles, and it does not support memory maps. diff --git a/valhalla-graphtile/src/test_fixtures.rs b/valhalla-graphtile/src/test_fixtures.rs index 44387e5..1b69025 100644 --- a/valhalla-graphtile/src/test_fixtures.rs +++ b/valhalla-graphtile/src/test_fixtures.rs @@ -1,9 +1,15 @@ -//! Paths to the test fixtures in the `fixtures` directory of this crate. +//! The test fixtures in the `fixtures` directory of this crate. //! //! All fixtures come from the same OpenStreetMap extract of Andorra. //! Miri does not support memory maps, so the tarball paths are not compiled under Miri. +use crate::GraphId; +use crate::graph_tile::GraphTile; +use crate::graph_tile::handles::OwnedGraphTileHandle; +use crate::tile_provider::TILE_FILE_EXTENSION; use std::path::PathBuf; +use std::sync::LazyLock; +use walkdir::WalkDir; fn fixture(name: &str) -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -16,6 +22,38 @@ pub(crate) fn andorra_tiles_dir() -> PathBuf { fixture("andorra-tiles") } +/// The base graph ID of the smallest tile in [`andorra_tiles_dir`]. +/// +/// Tests that run under Miri use this tile to stay fast. +pub(crate) fn smallest_andorra_tile() -> GraphId { + static SMALLEST: LazyLock = LazyLock::new(|| { + let path = WalkDir::new(andorra_tiles_dir()) + .into_iter() + .map(|entry| entry.expect("Unable to read the fixture directory")) + .filter(|entry| { + entry.file_type().is_file() + && entry + .path() + .extension() + .is_some_and(|extension| extension == TILE_FILE_EXTENSION) + }) + .min_by_key(|entry| { + let len = entry + .metadata() + .expect("Unable to read the tile metadata") + .len(); + (len, entry.path().to_path_buf()) + }) + .expect("The fixture directory has tiles") + .into_path(); + let bytes = std::fs::read(path).expect("Unable to read the tile"); + OwnedGraphTileHandle::try_from(bytes) + .expect("Unable to decode the tile") + .graph_id() + }); + *SMALLEST +} + /// The tiles of [`andorra_tiles_dir`] in an extract tarball, /// as `valhalla_build_extract` writes it. #[cfg(not(miri))] From f1d8633a51f6e0b94dde71f212e774acb59090b3 Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Wed, 4 Nov 2026 17:27:11 -0800 Subject: [PATCH 35/37] Remove graph ID parsing from the library The CLI is the only user of these parsers, so they move back to the CLI. The library keeps only GraphId::file_path, which the extract writer uses. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/graph_id.rs | 205 ----------------------------- valhalla-graphtile/src/lib.rs | 2 +- 2 files changed, 1 insertion(+), 206 deletions(-) diff --git a/valhalla-graphtile/src/graph_id.rs b/valhalla-graphtile/src/graph_id.rs index c859f96..9966707 100644 --- a/valhalla-graphtile/src/graph_id.rs +++ b/valhalla-graphtile/src/graph_id.rs @@ -1,5 +1,4 @@ use crate::tile_hierarchy::{STANDARD_LEVELS, TRANSIT_LEVEL}; -use core::str::FromStr; #[cfg(feature = "serde")] use serde::{Serialize, Serializer}; use std::fmt::{Display, Formatter}; @@ -38,31 +37,6 @@ pub enum InvalidGraphIdError { InvalidGraphId, } -/// The reasons why a string is not a valid graph ID or tile. -#[derive(Debug, Error, PartialEq)] -pub enum ParseGraphIdError { - /// The input does not have the expected number of `/`-separated components. - #[error("'{input}' is not in the {expected} form")] - Format { - input: String, - /// The expected form (e.g. `level/tile_id`). - expected: &'static str, - }, - /// A component of the input is not a non-negative integer. - #[error("the {component} in '{input}' is not a valid number")] - Component { - input: String, - /// The name of the component (e.g. `level`). - component: &'static str, - }, - /// The components are numbers, but they do not make a valid graph ID. - #[error("'{input}' is not a valid graph ID: {source}")] - Invalid { - input: String, - source: InvalidGraphIdError, - }, -} - /// An Identifier of a node or an edge within the tiled, hierarchical graph. /// It packs a hierarchy level, tile ID, and an identifier within /// the tile/level into a 64-bit integer. @@ -235,20 +209,6 @@ impl GraphId { GraphId(U64::::new(self.value() & 0x01ff_ffff)) } - /// Parses the base graph ID of a tile from `level/tile_id` (e.g. `2/762485`). - /// - /// The parser ignores whitespace around each component. - /// - /// # Errors - /// - /// Returns an error if `input` does not have two `/`-separated components, - /// or if a component is not a non-negative integer. - /// It also returns an error if the level or the tile ID does not fit in a graph ID. - pub fn parse_tile(input: &str) -> Result { - let [level, tile_id] = parse_components(input, "level/tile_id", ["level", "tile ID"])?; - components_to_graph_id(input, level, tile_id, 0) - } - /// Constructs a relative path for the given tile. /// /// # Errors @@ -294,75 +254,6 @@ impl GraphId { } } -impl FromStr for GraphId { - type Err = ParseGraphIdError; - - /// Parses a graph ID from its packed `u64` value (e.g. `16889572344463360`) - /// or from the `level/tile_id/index` form (e.g. `2/762485/42`). - /// - /// To parse a whole tile in the `level/tile_id` form, use [`GraphId::parse_tile`]. - fn from_str(input: &str) -> Result { - if let Ok(id) = input.trim().parse::() { - return GraphId::try_from_id(id).map_err(|source| ParseGraphIdError::Invalid { - input: input.to_string(), - source, - }); - } - - let [level, tile_id, index] = parse_components( - input, - "level/tile_id/index", - ["level", "tile ID", "feature index"], - )?; - components_to_graph_id(input, level, tile_id, index) - } -} - -/// Splits `input` into exactly `N` `/`-separated integer components. -fn parse_components( - input: &str, - expected: &'static str, - names: [&'static str; N], -) -> Result<[u64; N], ParseGraphIdError> { - let format_error = || ParseGraphIdError::Format { - input: input.to_string(), - expected, - }; - - let mut parts = input.split('/'); - let mut values = [0; N]; - for (value, component) in values.iter_mut().zip(names) { - let part = parts.next().ok_or_else(format_error)?; - *value = part - .trim() - .parse() - .map_err(|_| ParseGraphIdError::Component { - input: input.to_string(), - component, - })?; - } - if parts.next().is_some() { - return Err(format_error()); - } - - Ok(values) -} - -/// Converts parsed components into a graph ID, and reports range errors against `input`. -fn components_to_graph_id( - input: &str, - level: u64, - tile_id: u64, - index: u64, -) -> Result { - let invalid = |source| ParseGraphIdError::Invalid { - input: input.to_string(), - source, - }; - let level = u8::try_from(level).map_err(|_| invalid(InvalidGraphIdError::Level))?; - GraphId::try_from_components(level, tile_id, index).map_err(invalid) -} - impl Ord for GraphId { #[inline] fn cmp(&self, other: &Self) -> std::cmp::Ordering { @@ -582,100 +473,4 @@ mod tests { // Levels 4 to 7 fit in the bit field, but they have no tiling system. assert_eq!(tile(5, 0).file_path("gph"), Err(InvalidGraphIdError::Level)); } - - #[test] - fn test_parse_tile() { - assert_eq!(GraphId::parse_tile("2/763926"), Ok(tile(2, 763_926))); - assert_eq!(GraphId::parse_tile(" 0 / 3015 "), Ok(tile(0, 3015))); - } - - #[test] - fn test_parse_tile_invalid() { - let format = |input: &str| ParseGraphIdError::Format { - input: input.to_string(), - expected: "level/tile_id", - }; - assert_eq!(GraphId::parse_tile("2"), Err(format("2"))); - assert_eq!(GraphId::parse_tile("2/763926/7"), Err(format("2/763926/7"))); - assert_eq!( - GraphId::parse_tile("2/"), - Err(ParseGraphIdError::Component { - input: "2/".to_string(), - component: "tile ID", - }) - ); - assert_eq!( - GraphId::parse_tile("x/763926"), - Err(ParseGraphIdError::Component { - input: "x/763926".to_string(), - component: "level", - }) - ); - assert_eq!( - GraphId::parse_tile("-1/0"), - Err(ParseGraphIdError::Component { - input: "-1/0".to_string(), - component: "level", - }) - ); - // Levels have 3 bits, and tile IDs have 22 bits. - for (input, source) in [ - ("8/0", InvalidGraphIdError::Level), - ("256/0", InvalidGraphIdError::Level), - ("0/4194304", InvalidGraphIdError::GraphTileId), - ] { - assert_eq!( - GraphId::parse_tile(input), - Err(ParseGraphIdError::Invalid { - input: input.to_string(), - source, - }) - ); - } - } - - #[test] - fn test_from_str() { - let expected = GraphId::try_from_components(2, 762_485, 42).unwrap(); - assert_eq!("2/762485/42".parse(), Ok(expected)); - assert_eq!(expected.value().to_string().parse(), Ok(expected)); - assert_eq!( - " 16889572344463360 " - .parse::() - .map(|id| id.value()), - Ok(16_889_572_344_463_360) - ); - } - - #[test] - fn test_from_str_invalid() { - assert_eq!( - "2/762485".parse::(), - Err(ParseGraphIdError::Format { - input: "2/762485".to_string(), - expected: "level/tile_id/index", - }) - ); - assert_eq!( - "2/762485/x".parse::(), - Err(ParseGraphIdError::Component { - input: "2/762485/x".to_string(), - component: "feature index", - }) - ); - assert_eq!( - INVALID_GRAPH_ID.to_string().parse::(), - Err(ParseGraphIdError::Invalid { - input: INVALID_GRAPH_ID.to_string(), - source: InvalidGraphIdError::InvalidGraphId, - }) - ); - assert_eq!( - "0/0/2097152".parse::(), - Err(ParseGraphIdError::Invalid { - input: "0/0/2097152".to_string(), - source: InvalidGraphIdError::TileIndex, - }) - ); - } } diff --git a/valhalla-graphtile/src/lib.rs b/valhalla-graphtile/src/lib.rs index 0db98cd..e3cbab7 100644 --- a/valhalla-graphtile/src/lib.rs +++ b/valhalla-graphtile/src/lib.rs @@ -19,7 +19,7 @@ use serde::{Deserialize, Serialize}; // The implementations are sufficiently complex that we want to have lots of files, // But many of those only have one or two useful definitions to re-export, // so this flattens things for better ergonomics. -pub use graph_id::{GraphId, InvalidGraphIdError, ParseGraphIdError}; +pub use graph_id::{GraphId, InvalidGraphIdError}; /// Road class; broad hierarchies of relative (and sometimes locally specific) importance. /// From 8cc384ba2a9cf6cf254a8c07abaca9366d4492f6 Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Wed, 4 Nov 2026 17:28:16 -0800 Subject: [PATCH 36/37] Parse graph IDs and tiles in the CLI The get-edge and extract commands now parse their arguments with CLI functions and do not use the parsers in the library. Co-Authored-By: Claude Opus 5.5 --- valinor-cli/src/commands/extract.rs | 14 ++++++++++++-- valinor-cli/src/commands/get_edge.rs | 19 +++++++++++++++++++ valinor-cli/src/main.rs | 20 +++++++++++++++++++- 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/valinor-cli/src/commands/extract.rs b/valinor-cli/src/commands/extract.rs index 4ab9a4a..e223b9e 100644 --- a/valinor-cli/src/commands/extract.rs +++ b/valinor-cli/src/commands/extract.rs @@ -10,7 +10,7 @@ use std::{ #[cfg(unix)] use std::{fs::Permissions, os::unix::fs::PermissionsExt}; -use anyhow::Context; +use anyhow::{Context, anyhow}; use clap::Args; use tempfile::NamedTempFile; use valhalla_graphtile::GraphId; @@ -41,11 +41,21 @@ pub(crate) struct ExtractArgs { value_name = "LEVEL/TILE_ID", required = true, value_delimiter = ',', - value_parser = GraphId::parse_tile + value_parser = parse_tile )] pub(crate) tiles: Vec, } +/// Parses the base graph ID of a tile from the `level/tile_id` form. +fn parse_tile(input: &str) -> anyhow::Result { + let (level, tile_id) = input + .split_once('/') + .ok_or_else(|| anyhow!("expected the level/tile_id form"))?; + let level = level.parse().context("invalid level")?; + let tile_id = tile_id.parse().context("invalid tile id")?; + Ok(GraphId::try_from_components(level, tile_id, 0)?) +} + /// A successful extract, as reported on stderr. #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct ExtractSummary { diff --git a/valinor-cli/src/commands/get_edge.rs b/valinor-cli/src/commands/get_edge.rs index 211cef9..22d53fd 100644 --- a/valinor-cli/src/commands/get_edge.rs +++ b/valinor-cli/src/commands/get_edge.rs @@ -24,9 +24,28 @@ pub(crate) struct GetEdgeArgs { valhalla_config: PathBuf, /// Graph ID (u64) or slash-form level/tile_id/index + #[arg(value_parser = parse_graph_id)] pub(crate) graph_id: GraphId, } +/// Parses a graph ID from its packed `u64` value or from the `level/tile_id/index` form. +fn parse_graph_id(input: &str) -> anyhow::Result { + if let Ok(id) = input.parse::() { + return Ok(GraphId::try_from_id(id)?); + } + + let parts: Vec<&str> = input.split('/').collect(); + let [level, tile_id, index] = parts[..] else { + return Err(anyhow!( + "Unrecognized graph id format. Use a u64 integer or level/tile_id/index" + )); + }; + let level = level.parse().context("invalid level in graph id")?; + let tile_id = tile_id.parse().context("invalid tile id in graph id")?; + let index = index.parse().context("invalid index in graph id")?; + Ok(GraphId::try_from_components(level, tile_id, index)?) +} + #[derive(Debug, Clone)] struct DataSources { routing_graph: Option, diff --git a/valinor-cli/src/main.rs b/valinor-cli/src/main.rs index 7ed9e99..62b0957 100644 --- a/valinor-cli/src/main.rs +++ b/valinor-cli/src/main.rs @@ -74,7 +74,15 @@ mod tests { #[test] fn extract_tiles_reject_malformed_values() { - for input in ["2", "2/763926/7", "2/abc", "8/0"] { + for input in [ + "2", + "2/763926/7", + "2/abc", + "x/0", + "8/0", + "256/0", + "0/4194304", + ] { let error = Cli::try_parse_from([ "valinor-cli", "extract", @@ -101,6 +109,16 @@ mod tests { } } + #[test] + fn get_edge_rejects_malformed_graph_ids() { + // The last value has a level of 8, which does not fit in the 3-bit level field. + for input in ["2/762485", "2/762485/42/0", "2/762485/x", "x/0/0", "8/0/0"] { + let error = Cli::try_parse_from(["valinor-cli", "get-edge", "valhalla.json", input]) + .expect_err("a malformed graph ID is rejected"); + assert!(error.to_string().contains(input), "{input}: {error}"); + } + } + #[test] fn extract_tiles_are_required() { assert!(Cli::try_parse_from(["valinor-cli", "extract", "in.tar", "out.tar"]).is_err()); From 708c61a18e829b63acd53b50a73a67623df2832e Mon Sep 17 00:00:00 2001 From: Jacob Fielding Date: Wed, 4 Nov 2026 17:38:31 -0800 Subject: [PATCH 37/37] Test the tile ID boundary of each level in GraphId::file_path The test of valid paths repeated the test that is already on main, so it is removed. Co-Authored-By: Claude Opus 5.5 --- valhalla-graphtile/src/graph_id.rs | 61 +++++++++++------------------- 1 file changed, 22 insertions(+), 39 deletions(-) diff --git a/valhalla-graphtile/src/graph_id.rs b/valhalla-graphtile/src/graph_id.rs index 9966707..6ef358f 100644 --- a/valhalla-graphtile/src/graph_id.rs +++ b/valhalla-graphtile/src/graph_id.rs @@ -426,51 +426,34 @@ mod tests { ); } - fn tile(level: u8, tile_id: u64) -> GraphId { - GraphId::try_from_components(level, tile_id, 0).expect("valid tile ID in test") - } - #[test] - fn test_file_path() { - // Levels 0 and 1 pad the tile ID to 6 digits, and level 2 pads it to 9. - for (graph_id, expected) in [ - (tile(0, 3015), "0/003/015.gph"), - (tile(0, 49), "0/000/049.gph"), - (tile(1, 47_701), "1/047/701.gph"), - (tile(1, 0), "1/000/000.gph"), - (tile(2, 762_485), "2/000/762/485.gph"), - (tile(2, 1_036_799), "2/001/036/799.gph"), + fn test_file_path_invalid() { + // Each level has a 4°, 1°, or 0.25° grid of 4050, 64800, or 1036800 tiles. + // Level 2 has one more directory, because its tile IDs have more digits. + for (level, max_tile_id, path) in [ + (0, 4049, "0/004/049.gph"), + (1, 64_799, "1/064/799.gph"), + (2, 1_036_799, "2/001/036/799.gph"), ] { assert_eq!( - graph_id.file_path("gph"), - Ok(expected.split('/').collect::()) + GraphId::try_from_components(level, max_tile_id, 0) + .unwrap() + .file_path("gph"), + Ok(path.into()) + ); + assert_eq!( + GraphId::try_from_components(level, max_tile_id + 1, 0) + .unwrap() + .file_path("gph"), + Err(InvalidGraphIdError::GraphTileId) ); } - - let with_index = GraphId::try_from_components(2, 762_485, 42).unwrap(); - assert_eq!( - with_index.file_path("gph"), - Ok(["2", "000", "762", "485.gph"].iter().collect::()) - ); - - assert_eq!( - tile(1, 47_701).file_path(""), - Ok(["1", "047", "701"].iter().collect::()) - ); - } - - #[test] - fn test_file_path_invalid() { - // Level 0 has 45 × 90 tiles, so its largest tile ID is 4049. - assert_eq!( - tile(0, 4049).file_path("gph"), - Ok(["0", "004", "049.gph"].iter().collect::()) - ); + // Levels 4 to 7 fit in the bit field, but they have no tiling system. assert_eq!( - tile(0, 4050).file_path("gph"), - Err(InvalidGraphIdError::GraphTileId) + GraphId::try_from_components(5, 0, 0) + .unwrap() + .file_path("gph"), + Err(InvalidGraphIdError::Level) ); - // Levels 4 to 7 fit in the bit field, but they have no tiling system. - assert_eq!(tile(5, 0).file_path("gph"), Err(InvalidGraphIdError::Level)); } }