diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000..befadfb --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,50 @@ +# Integration tests of offline extracts with C++ Valhalla. See scripts/integration/README.md. +name: Integration + +on: + pull_request: + branches: [ "main" ] + paths: + - "valhalla-graphtile/src/extract/**" + # The tar layout and the index.bin code are in the tile providers. + - "valhalla-graphtile/src/tile_provider/**" + - "valinor-cli/**" + - "scripts/integration/**" + - ".github/workflows/integration.yml" + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + # The Valhalla release under test. Without this pin, the harness uses `latest`. + # When you change it, examine the differences in the responses and logs before you merge. + VALHALLA_VERSION: "3.8.2" + +jobs: + integration: + # Linux only: the job tests the Docker image, not the platform. + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - uses: astral-sh/setup-uv@v10.3.0 + + - name: Check the harness + working-directory: scripts/integration + run: | + uv run --locked ruff check + uv run --locked ruff format --check + uv run --locked pytest + + - name: Run the integration tests with C++ Valhalla + run: uv run --locked --project scripts/integration valinor-integration + + - name: Upload harness artifacts on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: integration-workdir + path: | + target/integration/responses + target/integration/valhalla-*.log + target/integration/valhalla.json diff --git a/Cargo.lock b/Cargo.lock index 21f89d8..9f2fb6d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2425,6 +2425,7 @@ dependencies = [ "self_cell", "serde", "tar", + "tempfile", "thiserror 2.0.21", "tracing", "trig-const", @@ -2478,6 +2479,7 @@ dependencies = [ "anyhow", "clap", "serde_json", + "tempfile", "tracing", "tracing-subscriber", "valhalla-graphtile", diff --git a/Cargo.toml b/Cargo.toml index f6cba1d..e8755c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,6 +54,7 @@ serde = { version = "1.0.229", features = ["derive"] } serde_with = "3.24.0" serde_json = "1.0.154" tar = "0.4.46" +tempfile = "3.27.0" thiserror = "2.0.21" tokio = { version = "1.54.1", features = ["macros", "rt-multi-thread", "fs", "io-util", "sync"] } tracing = "0.1.44" diff --git a/scripts/integration/.gitignore b/scripts/integration/.gitignore new file mode 100644 index 0000000..5831ca4 --- /dev/null +++ b/scripts/integration/.gitignore @@ -0,0 +1,3 @@ +.venv/ +__pycache__/ +.pytest_cache/ diff --git a/scripts/integration/README.md b/scripts/integration/README.md new file mode 100644 index 0000000..709c9a1 --- /dev/null +++ b/scripts/integration/README.md @@ -0,0 +1,94 @@ +# Integration tests with C++ Valhalla + +This harness makes sure that a sub-extract from `valinor-cli extract` +works in stock C++ Valhalla the same way as the full extract. + +The harness does these steps: + +1. It writes a sub-extract of the full extract with `valinor-cli extract`. + The tiles are in [`sub-extract-tiles.txt`](sub-extract-tiles.txt). + The sub-extract must be a proper subset of the full extract, + or the comparison would prove nothing. +2. It starts two `valhalla_service` containers with docker-py: + one for the full extract and one for the sub-extract. + The image's own `valhalla_build_config` makes the config. + The harness sets `mjolnir.shortcut_caching` to `false`, + because Valhalla stops at startup with a partial extract when it is `true`. +3. It sends each request of the [Bruno collection](bruno/) to both servers. + It saves each response, and it compares the two responses of each request + (see [`checks.py`](src/valinor_integration/checks.py) for the fields that it compares). +4. It makes sure that the logs of both servers contain no `[ERROR]` lines. + +The default full extract is the Andorra fixture +(`valhalla-graphtile/fixtures/andorra-tiles.tar`). + +## Requirements + +- [uv](https://docs.astral.sh/uv/) +- Docker (Docker Desktop, colima, or a Docker daemon on Linux) +- cargo, if you do not set `VALINOR_CLI` + +## Run the test + +From the root of the repository: + +```bash +uv run --project scripts/integration valinor-integration +``` + +The command prints `integration: PASS`, or a list of failures with a diff for each difference. +The work directory (`target/integration` by default) contains: + +- `sub-extract.tar`: the sub-extract. +- `valhalla.json`: the Valhalla config. +- `responses/full/` and `responses/sub/`: the raw response of each request. +- `valhalla-full.log` and `valhalla-sub.log`: the logs of the two servers. + +Use `--help` for the options (a different full extract, work directory, tile list, or collection). + +## Send requests by hand with Bruno + +```bash +uv run --project scripts/integration valinor-integration serve +``` + +This serves the full extract at `http://localhost:8002` +and the sub-extract at `http://localhost:8003` until you push Ctrl-C. +Open `scripts/integration/bruno` in [Bruno](https://www.usebruno.com/), +and select the `full` or the `sub` environment. + +## Environment variables + +All variables are optional. + +| Variable | Default | Description | +| --- | --- | --- | +| `VALHALLA_VERSION` | `latest` | The tag of the Valhalla image, such as `3.8.2`. CI sets it in [`integration.yml`](../../.github/workflows/integration.yml). If it is not set, the harness uses `latest` and logs a warning. | +| `VALHALLA_IMAGE` | `ghcr.io/valhalla/valhalla` | The image repository. | +| `VALINOR_CLI` | (none) | The path of a `valinor-cli` binary. If it is not set, the harness runs `cargo run --release -p valinor-cli`. | +| `VALHALLA_PORT` | `8002` | The port of the full extract in `serve` mode. The sub-extract uses the next port. If you change it, change the Bruno environments too. | + +## Add a request + +1. Add the request to the Bruno collection. + You can do this in the Bruno app, or you can copy a file in `bruno/route/`. + Keep all coordinates far inside the tiles of the sub-extract, + so that the request never needs a tile which the sub-extract does not contain. +2. If the request uses a new endpoint, add a check for the endpoint to + [`checks.py`](src/valinor_integration/checks.py). + The harness refuses a request for an endpoint that has no check. + +The harness reads a subset of the Bruno YAML format: +the method, the URL (with `{{baseUrl}}`), headers, and a JSON body. +It refuses requests with Bruno scripts or assertions, +because it does not run them; put such checks in `checks.py`. + +## Unit tests of the harness + +```bash +uv run --project scripts/integration pytest +``` + +```bash +uv run --project scripts/integration ruff check +``` diff --git a/scripts/integration/bruno/environments/full.yml b/scripts/integration/bruno/environments/full.yml new file mode 100644 index 0000000..bab1789 --- /dev/null +++ b/scripts/integration/bruno/environments/full.yml @@ -0,0 +1,4 @@ +name: full +variables: + - name: baseUrl + value: http://localhost:8002 diff --git a/scripts/integration/bruno/environments/sub.yml b/scripts/integration/bruno/environments/sub.yml new file mode 100644 index 0000000..3ec4348 --- /dev/null +++ b/scripts/integration/bruno/environments/sub.yml @@ -0,0 +1,4 @@ +name: sub +variables: + - name: baseUrl + value: http://localhost:8003 diff --git a/scripts/integration/bruno/locate/andorra-la-vella-center.yml b/scripts/integration/bruno/locate/andorra-la-vella-center.yml new file mode 100644 index 0000000..5746419 --- /dev/null +++ b/scripts/integration/bruno/locate/andorra-la-vella-center.yml @@ -0,0 +1,28 @@ +info: + name: andorra-la-vella-center + type: http + seq: 1 + +http: + method: POST + url: "{{baseUrl}}/locate" + body: + type: json + data: |- + { + "locations": [ + { + "lat": 42.5063, + "lon": 1.5218 + } + ], + "costing": "auto", + "verbose": true + } + auth: inherit + +settings: + encodeUrl: true + timeout: 0 + followRedirects: true + maxRedirects: 5 diff --git a/scripts/integration/bruno/locate/folder.yml b/scripts/integration/bruno/locate/folder.yml new file mode 100644 index 0000000..9ee0e40 --- /dev/null +++ b/scripts/integration/bruno/locate/folder.yml @@ -0,0 +1,8 @@ +info: + name: locate + type: folder + seq: 2 + +request: + auth: inherit + diff --git a/scripts/integration/bruno/opencollection.yml b/scripts/integration/bruno/opencollection.yml new file mode 100644 index 0000000..2246956 --- /dev/null +++ b/scripts/integration/bruno/opencollection.yml @@ -0,0 +1,10 @@ +opencollection: 1.0.0 + +info: + name: Valinor integration +bundled: false +extensions: + bruno: + ignore: + - node_modules + - .git diff --git a/scripts/integration/bruno/route/andorra-la-vella-center-pedestrian.yml b/scripts/integration/bruno/route/andorra-la-vella-center-pedestrian.yml new file mode 100644 index 0000000..591ffe9 --- /dev/null +++ b/scripts/integration/bruno/route/andorra-la-vella-center-pedestrian.yml @@ -0,0 +1,32 @@ +info: + name: andorra-la-vella-center-pedestrian + type: http + seq: 5 + +http: + method: POST + url: "{{baseUrl}}/route" + body: + type: json + data: |- + { + "locations": [ + { + "lat": 42.5063, + "lon": 1.5218 + }, + { + "lat": 42.5103, + "lon": 1.5336 + } + ], + "costing": "pedestrian", + "units": "kilometers" + } + auth: inherit + +settings: + encodeUrl: true + timeout: 0 + followRedirects: true + maxRedirects: 5 diff --git a/scripts/integration/bruno/route/andorra-la-vella-to-encamp-auto.yml b/scripts/integration/bruno/route/andorra-la-vella-to-encamp-auto.yml new file mode 100644 index 0000000..31c3dfa --- /dev/null +++ b/scripts/integration/bruno/route/andorra-la-vella-to-encamp-auto.yml @@ -0,0 +1,32 @@ +info: + name: andorra-la-vella-to-encamp-auto + type: http + seq: 1 + +http: + method: POST + url: "{{baseUrl}}/route" + body: + type: json + data: |- + { + "locations": [ + { + "lat": 42.5063, + "lon": 1.5218 + }, + { + "lat": 42.536, + "lon": 1.5836 + } + ], + "costing": "auto", + "units": "kilometers" + } + auth: inherit + +settings: + encodeUrl: true + timeout: 0 + followRedirects: true + maxRedirects: 5 diff --git a/scripts/integration/bruno/route/andorra-la-vella-to-la-massana-auto.yml b/scripts/integration/bruno/route/andorra-la-vella-to-la-massana-auto.yml new file mode 100644 index 0000000..11ad474 --- /dev/null +++ b/scripts/integration/bruno/route/andorra-la-vella-to-la-massana-auto.yml @@ -0,0 +1,32 @@ +info: + name: andorra-la-vella-to-la-massana-auto + type: http + seq: 2 + +http: + method: POST + url: "{{baseUrl}}/route" + body: + type: json + data: |- + { + "locations": [ + { + "lat": 42.5078, + "lon": 1.5211 + }, + { + "lat": 42.5449, + "lon": 1.5148 + } + ], + "costing": "auto", + "units": "kilometers" + } + auth: inherit + +settings: + encodeUrl: true + timeout: 0 + followRedirects: true + maxRedirects: 5 diff --git a/scripts/integration/bruno/route/encamp-to-canillo-auto.yml b/scripts/integration/bruno/route/encamp-to-canillo-auto.yml new file mode 100644 index 0000000..179452e --- /dev/null +++ b/scripts/integration/bruno/route/encamp-to-canillo-auto.yml @@ -0,0 +1,32 @@ +info: + name: encamp-to-canillo-auto + type: http + seq: 3 + +http: + method: POST + url: "{{baseUrl}}/route" + body: + type: json + data: |- + { + "locations": [ + { + "lat": 42.536, + "lon": 1.5836 + }, + { + "lat": 42.5676, + "lon": 1.5976 + } + ], + "costing": "auto", + "units": "kilometers" + } + auth: inherit + +settings: + encodeUrl: true + timeout: 0 + followRedirects: true + maxRedirects: 5 diff --git a/scripts/integration/bruno/route/folder.yml b/scripts/integration/bruno/route/folder.yml new file mode 100644 index 0000000..c3c95fb --- /dev/null +++ b/scripts/integration/bruno/route/folder.yml @@ -0,0 +1,8 @@ +info: + name: route + type: folder + seq: 1 + +request: + auth: inherit + diff --git a/scripts/integration/bruno/route/ordino-to-la-massana-bicycle.yml b/scripts/integration/bruno/route/ordino-to-la-massana-bicycle.yml new file mode 100644 index 0000000..32baceb --- /dev/null +++ b/scripts/integration/bruno/route/ordino-to-la-massana-bicycle.yml @@ -0,0 +1,32 @@ +info: + name: ordino-to-la-massana-bicycle + type: http + seq: 4 + +http: + method: POST + url: "{{baseUrl}}/route" + body: + type: json + data: |- + { + "locations": [ + { + "lat": 42.5562, + "lon": 1.5332 + }, + { + "lat": 42.5449, + "lon": 1.5148 + } + ], + "costing": "bicycle", + "units": "kilometers" + } + auth: inherit + +settings: + encodeUrl: true + timeout: 0 + followRedirects: true + maxRedirects: 5 diff --git a/scripts/integration/pyproject.toml b/scripts/integration/pyproject.toml new file mode 100644 index 0000000..b196414 --- /dev/null +++ b/scripts/integration/pyproject.toml @@ -0,0 +1,33 @@ +[project] +name = "valinor-integration" +version = "0.1.0" +description = "Integration tests of Valinor tile extracts with stock C++ Valhalla" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "docker>=7.2.0", + "pyyaml>=6.0.3", + "requests>=2.34.2", +] + +[project.scripts] +valinor-integration = "valinor_integration.cli:main" + +[build-system] +requires = ["uv_build>=0.11.29,<0.12"] +build-backend = "uv_build" + +[dependency-groups] +dev = [ + "pytest>=9.1.1", + "ruff>=0.17.1", +] + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.ruff] +line-length = 100 + +[tool.ruff.lint] +select = ["E", "F", "I", "B", "UP", "SIM", "RUF"] diff --git a/scripts/integration/src/valinor_integration/__init__.py b/scripts/integration/src/valinor_integration/__init__.py new file mode 100644 index 0000000..07d3792 --- /dev/null +++ b/scripts/integration/src/valinor_integration/__init__.py @@ -0,0 +1 @@ +"""Integration tests of Valinor tile extracts with stock C++ Valhalla.""" diff --git a/scripts/integration/src/valinor_integration/__main__.py b/scripts/integration/src/valinor_integration/__main__.py new file mode 100644 index 0000000..18ac038 --- /dev/null +++ b/scripts/integration/src/valinor_integration/__main__.py @@ -0,0 +1,5 @@ +import sys + +from valinor_integration.cli import main + +sys.exit(main()) diff --git a/scripts/integration/src/valinor_integration/checks.py b/scripts/integration/src/valinor_integration/checks.py new file mode 100644 index 0000000..9e18c56 --- /dev/null +++ b/scripts/integration/src/valinor_integration/checks.py @@ -0,0 +1,120 @@ +"""The checks of the responses, for each Valhalla endpoint. + +A check has ``validate``, which finds problems in one response, +and ``project``, which keeps only the fields that the comparison uses. +To test a new endpoint, add a request to the Bruno collection and a check here. +""" + +from __future__ import annotations + +import difflib +import json +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Any + +# Per-maneuver fields included in the compared projection. +MANEUVER_FIELDS = ( + "type", + "instruction", + "verbal_pre_transition_instruction", + "street_names", + "length", + "time", + "begin_shape_index", + "end_shape_index", + "travel_mode", + "travel_type", +) + + +@dataclass(frozen=True) +class Check: + validate: Callable[[Any], list[str]] + project: Callable[[Any], Any] + + +def _validate_route(response: Any) -> list[str]: + trip = response.get("trip") if isinstance(response, dict) else None + if not isinstance(trip, dict) or not trip.get("legs"): + return ["the response contains no trip legs"] + return [] + + +def _project_route(response: Any) -> Any: + """Projects a /route response onto its status, units, shapes, summaries, and maneuvers.""" + trip = response["trip"] + return { + "status": trip.get("status"), + "units": trip.get("units"), + "summary": { + "length": trip["summary"]["length"], + "time": trip["summary"]["time"], + }, + "legs": [ + { + "shape": leg["shape"], + "summary": { + "length": leg["summary"]["length"], + "time": leg["summary"]["time"], + }, + "maneuvers": [ + {name: maneuver.get(name) for name in MANEUVER_FIELDS} + for maneuver in leg["maneuvers"] + ], + } + for leg in trip["legs"] + ], + } + + +def _validate_locate(response: Any) -> list[str]: + if not isinstance(response, list) or not response: + return ["the response is not a non-empty list of locations"] + if sum(len(location.get("edges") or []) for location in response) == 0: + return ["the response contains no edges"] + return [] + + +def _project_locate(response: Any) -> Any: + """Projects a /locate response onto the edges found at each location.""" + return [ + sorted( + ( + { + "edge_id": edge.get("edge_id"), + # With `verbose`, the OSM way ID is in the edge info. + "way_id": (edge.get("edge_info") or {}).get("way_id"), + "correlated_lat": edge.get("correlated_lat"), + "correlated_lon": edge.get("correlated_lon"), + "percent_along": edge.get("percent_along"), + } + for edge in location.get("edges") or [] + ), + key=lambda edge: json.dumps(edge, sort_keys=True), + ) + for location in response + ] + + +#: The checks, by the path of the endpoint. +CHECKS: Mapping[str, Check] = { + "/route": Check(validate=_validate_route, project=_project_route), + "/locate": Check(validate=_validate_locate, project=_project_locate), +} + + +def diff_projections( + name: str, expected: Any, actual: Any, expected_label: str, actual_label: str +) -> str: + """A unified diff of two projections, or an empty string if they are equal.""" + if expected == actual: + return "" + return "".join( + difflib.unified_diff( + json.dumps(expected, indent=2, sort_keys=True).splitlines(keepends=True), + json.dumps(actual, indent=2, sort_keys=True).splitlines(keepends=True), + fromfile=f"{expected_label}/{name}", + tofile=f"{actual_label}/{name}", + ) + ) diff --git a/scripts/integration/src/valinor_integration/cli.py b/scripts/integration/src/valinor_integration/cli.py new file mode 100644 index 0000000..3021189 --- /dev/null +++ b/scripts/integration/src/valinor_integration/cli.py @@ -0,0 +1,108 @@ +"""Integration tests of Valinor tile extracts with stock C++ Valhalla. + +``valinor-integration run`` (the default) compares a sub-extract with the full extract. +``valinor-integration serve`` serves both extracts until Ctrl-C. +See README.md for the environment variables. +""" + +from __future__ import annotations + +import argparse +import logging +import signal +import sys +from pathlib import Path + +from valinor_integration import runner +from valinor_integration.errors import HarnessError +from valinor_integration.settings import ( + DEFAULT_COLLECTION, + DEFAULT_FULL_EXTRACT, + DEFAULT_TILE_LIST, + DEFAULT_WORKDIR, + Settings, +) + +log = logging.getLogger("valinor_integration") + +COMMANDS = ("run", "serve") + + +def _parser() -> argparse.ArgumentParser: + common = argparse.ArgumentParser(add_help=False) + common.add_argument( + "--extract", + type=Path, + default=DEFAULT_FULL_EXTRACT, + help="the full extract tarball (default: the Andorra fixture)", + ) + common.add_argument( + "--workdir", + type=Path, + default=DEFAULT_WORKDIR, + help="the directory for the sub-extract, responses, and logs (default: target/integration)", + ) + common.add_argument( + "--tiles", + type=Path, + default=DEFAULT_TILE_LIST, + help="the tile list of the sub-extract (default: sub-extract-tiles.txt)", + ) + common.add_argument( + "--collection", + type=Path, + default=DEFAULT_COLLECTION, + help="the Bruno collection of requests (default: bruno/)", + ) + + parser = argparse.ArgumentParser( + prog="valinor-integration", + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + subparsers = parser.add_subparsers(dest="command", required=True) + subparsers.add_parser("run", parents=[common], help="run the integration test (the default)") + subparsers.add_parser("serve", parents=[common], help="serve both extracts until Ctrl-C") + return parser + + +def _interrupt(signum: int, frame: object) -> None: + raise KeyboardInterrupt + + +def main(argv: list[str] | None = None) -> int: + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") + # Remove the containers on SIGTERM (for example, a cancelled CI job), as on Ctrl-C. + signal.signal(signal.SIGTERM, _interrupt) + argv = sys.argv[1:] if argv is None else argv + if not argv or argv[0] not in (*COMMANDS, "-h", "--help"): + argv = ["run", *argv] + args = _parser().parse_args(argv) + inputs = runner.Inputs( + full_extract=args.extract.resolve(), + workdir=args.workdir.resolve(), + tile_list=args.tiles.resolve(), + collection=args.collection.resolve(), + ) + + try: + settings = Settings.from_env() + log.info("Valhalla image: %s", settings.image_ref) + if args.command == "serve": + runner.serve(settings, inputs) + return 0 + if runner.run(settings, inputs): + log.info("integration: PASS") + return 0 + log.error("integration: FAIL (responses and logs are in %s)", inputs.workdir) + return 1 + except HarnessError as e: + log.error("%s", e) + return 1 + except KeyboardInterrupt: + log.info("stopped") + return 130 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/integration/src/valinor_integration/collection.py b/scripts/integration/src/valinor_integration/collection.py new file mode 100644 index 0000000..81b14c5 --- /dev/null +++ b/scripts/integration/src/valinor_integration/collection.py @@ -0,0 +1,169 @@ +"""Reads the requests of a Bruno collection in the OpenCollection YAML format. + +The harness reads an HTTP method, a URL with ``{{variable}}`` placeholders, headers, +and a JSON body. It refuses a request that uses more, such as scripts or assertions, +so that it never sends a different request than Bruno. +""" + +from __future__ import annotations + +import json +import re +from collections.abc import Mapping +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit + +import yaml + +from valinor_integration.errors import HarnessError + +COLLECTION_FILE = "opencollection.yml" +FOLDER_FILE = "folder.yml" +ENVIRONMENTS_DIR = "environments" + +# The keys of a request file that the harness reads, or ignores (`settings` and `docs`). +_REQUEST_KEYS = {"info", "http", "settings", "docs"} +_HTTP_KEYS = {"method", "url", "body", "headers", "auth"} +_VARIABLE = re.compile(r"\{\{\s*([A-Za-z0-9_.-]+)\s*\}\}") + + +@dataclass(frozen=True) +class BrunoRequest: + """One HTTP request of the collection.""" + + #: The request name, from ``info.name``. + name: str + #: The path of the request file, relative to the collection root. + path: Path + seq: int + method: str + #: The URL, with ``{{variable}}`` placeholders. + url: str + headers: Mapping[str, str] = field(default_factory=dict) + #: The parsed JSON body, or ``None`` if the request has no body. + body: Any = None + + @property + def id(self) -> str: + """A unique ID: the path of the request file without the extension.""" + return self.path.with_suffix("").as_posix() + + def resolve_url(self, variables: Mapping[str, str]) -> str: + """Replaces the ``{{variable}}`` placeholders of the URL.""" + + def replace(match: re.Match[str]) -> str: + name = match.group(1) + if name not in variables: + raise HarnessError(f"{self.path}: the URL uses the unknown variable {{{{{name}}}}}") + return variables[name] + + return _VARIABLE.sub(replace, self.url) + + def endpoint(self, variables: Mapping[str, str]) -> str: + """The path of the resolved URL (for example, ``/route``).""" + return urlsplit(self.resolve_url(variables)).path + + +def load_collection(root: Path) -> list[BrunoRequest]: + """Reads every request of the collection at ``root``, in the order of the Bruno app.""" + if not (root / COLLECTION_FILE).is_file(): + raise HarnessError(f"{root} is not a Bruno collection: it has no {COLLECTION_FILE}") + + requests = [] + for path in sorted(root.rglob("*.yml")): + relative = path.relative_to(root) + if relative.name in (COLLECTION_FILE, FOLDER_FILE) or relative.parts[0] == ENVIRONMENTS_DIR: + continue + requests.append(parse_request(relative, path.read_text(encoding="utf-8"))) + + if not requests: + raise HarnessError(f"the Bruno collection at {root} contains no requests") + ids = [request.id for request in requests] + if len(set(ids)) != len(ids): + raise HarnessError(f"the Bruno collection at {root} contains duplicate request IDs") + + requests.sort(key=lambda request: (request.path.parent.as_posix(), request.seq, request.name)) + return requests + + +def parse_request(path: Path, text: str) -> BrunoRequest: + """Parses one request file. ``path`` is only used in error messages and the request ID.""" + + def fail(message: str) -> HarnessError: + return HarnessError(f"{path}: {message}") + + try: + document = yaml.safe_load(text) + except yaml.YAMLError as e: + raise fail(f"invalid YAML: {e}") from None + if not isinstance(document, dict): + raise fail("a request file must be a YAML mapping") + + unsupported = set(document) - _REQUEST_KEYS + if unsupported: + raise fail( + f"unsupported sections {sorted(unsupported)}; the harness does not run Bruno " + "scripts or assertions (put checks in valinor_integration.checks)" + ) + + info = document.get("info") or {} + if info.get("type") != "http": + raise fail(f"unsupported request type {info.get('type')!r}; expected 'http'") + name = info.get("name") + if not isinstance(name, str) or not name: + raise fail("info.name must be a non-empty string") + seq = info.get("seq", 0) + if not isinstance(seq, int): + raise fail("info.seq must be an integer") + + http = document.get("http") or {} + unsupported = set(http) - _HTTP_KEYS + if unsupported: + raise fail(f"unsupported http fields {sorted(unsupported)}") + method = http.get("method") + url = http.get("url") + if not isinstance(method, str) or not isinstance(url, str): + raise fail("http.method and http.url must be strings") + if http.get("auth", "none") not in ("none", "inherit"): + raise fail("authentication is not supported") + + return BrunoRequest( + name=name, + path=path, + seq=seq, + method=method.upper(), + url=url, + headers=_parse_headers(http.get("headers"), fail), + body=_parse_body(http.get("body"), fail), + ) + + +def _parse_headers(headers: Any, fail: Any) -> dict[str, str]: + if headers is None: + return {} + if not isinstance(headers, list): + raise fail("http.headers must be a list") + parsed = {} + for header in headers: + if not isinstance(header, dict) or not isinstance(header.get("name"), str): + raise fail("each header must have a name") + if header.get("disabled", False): + continue + parsed[header["name"]] = str(header.get("value", "")) + return parsed + + +def _parse_body(body: Any, fail: Any) -> Any: + if body is None: + return None + if not isinstance(body, dict) or body.get("type") != "json": + raise fail("only JSON bodies (http.body.type: json) are supported") + data = body.get("data") + if not isinstance(data, str): + raise fail("http.body.data must be a string of JSON") + try: + return json.loads(data) + except json.JSONDecodeError as e: + raise fail(f"http.body.data is not valid JSON: {e}") from None diff --git a/scripts/integration/src/valinor_integration/errors.py b/scripts/integration/src/valinor_integration/errors.py new file mode 100644 index 0000000..5ff783b --- /dev/null +++ b/scripts/integration/src/valinor_integration/errors.py @@ -0,0 +1,5 @@ +"""The error type of the harness.""" + + +class HarnessError(Exception): + """A failure that the harness reports with a message, and no traceback.""" diff --git a/scripts/integration/src/valinor_integration/extract.py b/scripts/integration/src/valinor_integration/extract.py new file mode 100644 index 0000000..4557313 --- /dev/null +++ b/scripts/integration/src/valinor_integration/extract.py @@ -0,0 +1,97 @@ +"""Writes the sub-extract with ``valinor-cli extract``, and checks it against the full extract.""" + +from __future__ import annotations + +import logging +import subprocess +import tarfile +from pathlib import Path + +from valinor_integration.errors import HarnessError +from valinor_integration.settings import REPO_ROOT, Settings + +log = logging.getLogger(__name__) + + +def read_tile_list(path: Path) -> list[str]: + """Reads a tile list: one ``level/tile_id`` on each line. + + ``#`` starts a comment, and the reader skips blank lines. + ``valinor-cli`` checks the form of each tile. + """ + tiles = [] + for line in path.read_text(encoding="utf-8").splitlines(): + tile = line.split("#", 1)[0].strip() + if tile: + tiles.append(tile.replace(" ", "")) + if not tiles: + raise HarnessError(f"the tile list contains no tiles: {path}") + return tiles + + +def valinor_cli_command(settings: Settings) -> list[str]: + """The command which runs ``valinor-cli``.""" + if settings.valinor_cli is not None: + return [str(settings.valinor_cli)] + return [ + "cargo", + "run", + "--quiet", + "--release", + "--manifest-path", + str(REPO_ROOT / "Cargo.toml"), + "--package", + "valinor-cli", + "--", + ] + + +def write_sub_extract( + settings: Settings, full_extract: Path, tiles: list[str], output: Path +) -> None: + """Writes a sub-extract of ``full_extract`` which contains ``tiles``.""" + command = [ + *valinor_cli_command(settings), + "extract", + str(full_extract), + str(output), + "--tile", + ",".join(tiles), + ] + log.info("writing the sub-extract: %s", " ".join(command)) + try: + subprocess.run(command, check=True) + except FileNotFoundError as e: + raise HarnessError(f"cannot run {command[0]}: {e}") from None + except subprocess.CalledProcessError as e: + raise HarnessError(f"valinor-cli extract failed with exit status {e.returncode}") from None + + +def check_proper_subset(full_extract: Path, sub_extract: Path) -> list[str]: + """Makes sure that the sub-extract is a proper subset of the full extract. + + Returns the entries that are only in the full extract. + """ + full_entries = _entry_names(full_extract) + sub_entries = _entry_names(sub_extract) + + extra = sorted(sub_entries - full_entries) + if extra: + raise HarnessError( + f"the sub-extract contains entries which are not in the full extract: {extra}" + ) + removed = sorted(full_entries - sub_entries) + if not removed: + raise HarnessError( + "the sub-extract contains every entry of the full extract, " + "so the comparison would prove nothing" + ) + return removed + + +def _entry_names(archive: Path) -> set[str]: + try: + with tarfile.open(archive) as tar: + return set(tar.getnames()) + except (OSError, tarfile.TarError) as e: + raise HarnessError(f"cannot read the tar archive {archive}: {e}") from None diff --git a/scripts/integration/src/valinor_integration/runner.py b/scripts/integration/src/valinor_integration/runner.py new file mode 100644 index 0000000..ba01a17 --- /dev/null +++ b/scripts/integration/src/valinor_integration/runner.py @@ -0,0 +1,201 @@ +"""The two modes of the harness: ``run`` (the test) and ``serve`` (for manual requests).""" + +from __future__ import annotations + +import json +import logging +import shutil +import time +from contextlib import ExitStack +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import requests + +from valinor_integration import valhalla +from valinor_integration.checks import CHECKS, diff_projections +from valinor_integration.collection import BrunoRequest, load_collection +from valinor_integration.errors import HarnessError +from valinor_integration.extract import check_proper_subset, read_tile_list, write_sub_extract +from valinor_integration.settings import Settings + +log = logging.getLogger(__name__) + +#: The names of the two extracts. +FULL = "full" +SUB = "sub" + + +@dataclass(frozen=True) +class Inputs: + """The files of a harness run.""" + + full_extract: Path + workdir: Path + tile_list: Path + collection: Path + + @property + def sub_extract(self) -> Path: + return self.workdir / "sub-extract.tar" + + +def run(settings: Settings, inputs: Inputs) -> bool: + """Serves both extracts, sends every request to both, and compares the responses. + + Returns ``True`` if every check passes. + The work directory keeps the responses, the logs, and the config. + """ + collection = _load_checked_collection(inputs.collection) + client, config = _prepare(settings, inputs) + + responses_dir = inputs.workdir / "responses" + shutil.rmtree(responses_dir, ignore_errors=True) + + failures: list[str] = [] + responses: dict[str, dict[str, Any]] = {} + with ExitStack() as stack: + servers = { + name: stack.enter_context( + valhalla.ValhallaServer(client, settings.image_ref, name, extract, config) + ) + for name, extract in ((FULL, inputs.full_extract), (SUB, inputs.sub_extract)) + } + + for name, server in servers.items(): + responses[name] = _send_all(server, collection, responses_dir / name, failures) + + # Capture the logs before the exit removes the containers. + for name, server in servers.items(): + logs = server.logs() + log_file = inputs.workdir / f"valhalla-{name}.log" + log_file.write_text(logs, encoding="utf-8") + error_lines = [line for line in logs.splitlines() if "[ERROR]" in line] + if error_lines: + failures.append( + f"{name}: {len(error_lines)} [ERROR] lines in the valhalla_service log " + f"({log_file}); the first is: {error_lines[0]}" + ) + + for request in collection: + full, sub = responses[FULL].get(request.id), responses[SUB].get(request.id) + if full is None or sub is None: + continue # The send failure is already in `failures`. + check = CHECKS[request.endpoint(_variables(""))] + diff = diff_projections(request.id, check.project(full), check.project(sub), FULL, SUB) + if diff: + failures.append(f"{request.id}: the responses differ:\n{diff}") + else: + log.info("ok: %s is identical for both extracts", request.id) + + for failure in failures: + log.error("FAIL: %s", failure) + return not failures + + +def serve(settings: Settings, inputs: Inputs) -> None: + """Serves both extracts on fixed ports until Ctrl-C or SIGTERM raises ``KeyboardInterrupt``.""" + client, config = _prepare(settings, inputs) + ports = {FULL: settings.serve_port, SUB: settings.serve_port + 1} + + with ExitStack() as stack: + servers = [ + stack.enter_context( + valhalla.ValhallaServer( + client, settings.image_ref, name, extract, config, host_port=ports[name] + ) + ) + for name, extract in ((FULL, inputs.full_extract), (SUB, inputs.sub_extract)) + ] + log.info( + "serving the full extract at %s and the sub-extract at %s; " + "in Bruno, open %s and select the 'full' or 'sub' environment; press Ctrl-C to stop", + servers[0].base_url, + servers[1].base_url, + inputs.collection, + ) + while True: + time.sleep(1) + + +def _load_checked_collection(root: Path) -> list[BrunoRequest]: + """Loads the collection, and makes sure that every request has a check.""" + collection = load_collection(root) + for request in collection: + endpoint = request.endpoint(_variables("")) + if endpoint not in CHECKS: + raise HarnessError( + f"{request.path}: the harness has no check for {endpoint}; " + f"add one to valinor_integration.checks (known: {sorted(CHECKS)})" + ) + log.info("loaded %d requests from %s", len(collection), root) + return collection + + +def _prepare(settings: Settings, inputs: Inputs) -> tuple[Any, dict[str, Any]]: + """Writes the sub-extract, and prepares Docker and the Valhalla config.""" + if not inputs.full_extract.is_file(): + raise HarnessError(f"the full extract is not a file: {inputs.full_extract}") + inputs.workdir.mkdir(parents=True, exist_ok=True) + + tiles = read_tile_list(inputs.tile_list) + inputs.sub_extract.unlink(missing_ok=True) + write_sub_extract(settings, inputs.full_extract, tiles, inputs.sub_extract) + for entry in check_proper_subset(inputs.full_extract, inputs.sub_extract): + log.info("only in the full extract: %s", entry) + + client = valhalla.connect() + valhalla.ensure_image(client, settings) + config = valhalla.build_config(client, settings.image_ref) + (inputs.workdir / "valhalla.json").write_text(json.dumps(config, indent=2), encoding="utf-8") + return client, config + + +def _variables(base_url: str) -> dict[str, str]: + """The variables of the Bruno environments, for one server.""" + return {"baseUrl": base_url} + + +def _send_all( + server: valhalla.ValhallaServer, + collection: list[BrunoRequest], + out_dir: Path, + failures: list[str], +) -> dict[str, Any]: + """Sends every request to one server, and saves and validates each response.""" + variables = _variables(server.base_url) + responses = {} + for request in collection: + label = f"{server.name}: {request.id}" + try: + response = requests.request( + request.method, + request.resolve_url(variables), + json=request.body, + headers=dict(request.headers), + timeout=valhalla.REQUEST_TIMEOUT_SECONDS, + ) + except requests.RequestException as e: + failures.append(f"{label}: the request failed: {e}") + continue + + out_path = out_dir / f"{request.id}.json" + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(response.text, encoding="utf-8") + + if response.status_code != 200: + failures.append(f"{label}: HTTP {response.status_code}: {response.text[:500]}") + continue + try: + body = response.json() + except ValueError: + failures.append(f"{label}: the response is not JSON") + continue + + problems = CHECKS[request.endpoint(variables)].validate(body) + if problems: + failures.extend(f"{label}: {problem}" for problem in problems) + continue + responses[request.id] = body + return responses diff --git a/scripts/integration/src/valinor_integration/settings.py b/scripts/integration/src/valinor_integration/settings.py new file mode 100644 index 0000000..f9badd3 --- /dev/null +++ b/scripts/integration/src/valinor_integration/settings.py @@ -0,0 +1,91 @@ +"""Settings of the harness, from environment variables. + +Every setting has a default, so ``valinor-integration`` runs with no configuration. +See README.md for the variables. +""" + +from __future__ import annotations + +import os +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path + +from valinor_integration.errors import HarnessError + +DEFAULT_VALHALLA_IMAGE = "ghcr.io/valhalla/valhalla" +LATEST_TAG = "latest" +DEFAULT_SERVE_PORT = 8002 + +HARNESS_DIR = Path(__file__).resolve().parents[2] +REPO_ROOT = HARNESS_DIR.parents[1] + +DEFAULT_FULL_EXTRACT = REPO_ROOT / "valhalla-graphtile" / "fixtures" / "andorra-tiles.tar" +DEFAULT_WORKDIR = REPO_ROOT / "target" / "integration" +DEFAULT_TILE_LIST = HARNESS_DIR / "sub-extract-tiles.txt" +DEFAULT_COLLECTION = HARNESS_DIR / "bruno" + + +@dataclass(frozen=True) +class Settings: + """The settings which come from environment variables.""" + + valhalla_image: str + #: The image tag, or ``None`` to use ``latest``. + valhalla_version: str | None + valinor_cli: Path | None + serve_port: int + + @property + def image_tag(self) -> str: + return self.valhalla_version or LATEST_TAG + + @property + def image_ref(self) -> str: + return f"{self.valhalla_image}:{self.image_tag}" + + @property + def is_pinned(self) -> bool: + return self.valhalla_version is not None + + @classmethod + def from_env(cls, env: Mapping[str, str] | None = None) -> Settings: + """Reads the settings from ``env`` (by default, the process environment). + + An empty variable is the same as an unset variable. + """ + env = os.environ if env is None else env + + def get(name: str) -> str | None: + value = env.get(name, "").strip() + return value or None + + version = get("VALHALLA_VERSION") + if version is not None and ("/" in version or ":" in version or "@" in version): + raise HarnessError( + f"VALHALLA_VERSION must be an image tag such as 3.8.2, not {version!r}; " + "set VALHALLA_IMAGE to change the repository" + ) + + cli = get("VALINOR_CLI") + valinor_cli = Path(cli).expanduser() if cli is not None else None + if valinor_cli is not None and not valinor_cli.is_file(): + raise HarnessError(f"VALINOR_CLI is not a file: {valinor_cli}") + + port_value = get("VALHALLA_PORT") + serve_port = DEFAULT_SERVE_PORT + if port_value is not None: + try: + serve_port = int(port_value) + except ValueError: + raise HarnessError(f"VALHALLA_PORT is not a number: {port_value!r}") from None + # serve mode also uses the next port, for the sub-extract. + if not 1 <= serve_port <= 65534: + raise HarnessError(f"VALHALLA_PORT must be from 1 to 65534, not {serve_port}") + + return cls( + valhalla_image=get("VALHALLA_IMAGE") or DEFAULT_VALHALLA_IMAGE, + valhalla_version=version, + valinor_cli=valinor_cli, + serve_port=serve_port, + ) diff --git a/scripts/integration/src/valinor_integration/valhalla.py b/scripts/integration/src/valinor_integration/valhalla.py new file mode 100644 index 0000000..3ebd7e2 --- /dev/null +++ b/scripts/integration/src/valinor_integration/valhalla.py @@ -0,0 +1,254 @@ +"""Runs stock C++ Valhalla in Docker (with docker-py) against a tile extract.""" + +from __future__ import annotations + +import io +import json +import logging +import subprocess +import tarfile +import tempfile +import time +from pathlib import Path +from types import TracebackType +from typing import IO, Any + +import docker +import docker.errors +import requests +from docker.models.containers import Container + +from valinor_integration.errors import HarnessError +from valinor_integration.settings import Settings + +log = logging.getLogger(__name__) + +#: The directory in the container that holds the extract and the config. +CONTAINER_DATA_DIR = "/valinor" +CONTAINER_EXTRACT = f"{CONTAINER_DATA_DIR}/tiles.tar" +CONTAINER_CONFIG = f"{CONTAINER_DATA_DIR}/valhalla.json" +#: The port of valhalla_service in the container. +SERVICE_PORT = 8002 + +READINESS_TIMEOUT_SECONDS = 90 +REQUEST_TIMEOUT_SECONDS = 30 + + +def connect() -> docker.DockerClient: + """Connects to the Docker daemon. + + If the default connection fails, this uses the current Docker CLI context + (for example, the socket of colima). docker-py does not read that context. + """ + try: + client = docker.from_env() + client.ping() + return client + except docker.errors.DockerException as default_error: + try: + endpoint = subprocess.run( + ["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + client = docker.DockerClient(base_url=endpoint) + client.ping() + log.info("connected to Docker through the CLI context endpoint %s", endpoint) + return client + except (OSError, subprocess.CalledProcessError, docker.errors.DockerException): + raise HarnessError( + f"cannot connect to Docker ({default_error}); is the Docker daemon running?" + ) from None + + +def ensure_image(client: docker.DockerClient, settings: Settings) -> None: + """Pulls ``latest`` every time, and a pinned tag only if it is not present.""" + if not settings.is_pinned: + log.warning( + "VALHALLA_VERSION is not set, so the harness uses %s; " + "set VALHALLA_VERSION to pin a release", + settings.image_ref, + ) + else: + try: + client.images.get(settings.image_ref) + return + except docker.errors.ImageNotFound: + pass + + log.info("pulling %s", settings.image_ref) + try: + client.images.pull(settings.valhalla_image, tag=settings.image_tag) + except docker.errors.APIError as e: + raise HarnessError(f"cannot pull {settings.image_ref}: {e}") from None + + +def build_config(client: docker.DockerClient, image: str) -> dict[str, Any]: + """Makes a config for the extract at ``CONTAINER_EXTRACT`` with the image's own tool.""" + try: + output = client.containers.run( + image, + entrypoint="valhalla_build_config", + command=["--mjolnir-tile-extract", CONTAINER_EXTRACT], + remove=True, + stdout=True, + stderr=False, + ) + except docker.errors.DockerException as e: + raise HarnessError(f"valhalla_build_config failed: {e}") from None + config = json.loads(output) + + mjolnir = config["mjolnir"] + mjolnir["tile_extract"] = CONTAINER_EXTRACT + # Do not enable shortcut_caching: Valhalla then asserts at startup that every + # level 0 and level 1 tile exists, so a partial extract fails to start. + mjolnir["shortcut_caching"] = False + return config + + +class ValhallaServer: + """A ``valhalla_service`` container which serves one extract. + + Use it as a context manager: entry starts the container, + and exit removes it (also after an error or Ctrl-C). + Call ``logs`` before the exit to keep the logs. + """ + + def __init__( + self, + client: docker.DockerClient, + image: str, + name: str, + extract: Path, + config: dict[str, Any], + host_port: int | None = None, + ) -> None: + """``host_port`` ``None`` lets Docker choose a free port.""" + self.client = client + self.image = image + self.name = name + self.extract = extract + self.config = config + self.host_port = host_port + self.container_name = f"valinor-integration-{name}" + self._container: Container | None = None + self.base_url = "" + + def __enter__(self) -> ValhallaServer: + try: + self._start() + except BaseException: + self._remove() + raise + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + traceback: TracebackType | None, + ) -> None: + self._remove() + + def _start(self) -> None: + self._remove_stale_container() + log.info("starting valhalla_service for the %s extract", self.name) + self._container = self.client.containers.create( + self.image, + name=self.container_name, + entrypoint="valhalla_service", + command=[CONTAINER_CONFIG, "1"], + ports={f"{SERVICE_PORT}/tcp": self.host_port}, + ) + # Copy the data in place of a bind mount. + # A bind mount fails for a directory that the Docker VM does not share + # (such as /tmp with colima). + with tempfile.TemporaryFile() as archive: + self._write_data_archive(archive) + archive.seek(0) + if not self._container.put_archive("/", archive): + raise HarnessError(f"cannot copy the {self.name} extract into the container") + self._container.start() + + self._container.reload() + bindings = self._container.ports.get(f"{SERVICE_PORT}/tcp") or [] + if not bindings: + raise HarnessError(f"Docker did not publish the port of the {self.name} container") + self.base_url = f"http://localhost:{bindings[0]['HostPort']}" + self._wait_until_ready() + + def _write_data_archive(self, archive: IO[bytes]) -> None: + """Writes a tar archive with the extract and the config, in ``CONTAINER_DATA_DIR``.""" + with tarfile.open(fileobj=archive, mode="w") as tar: + directory = tarfile.TarInfo(CONTAINER_DATA_DIR.lstrip("/")) + directory.type = tarfile.DIRTYPE + directory.mode = 0o755 + tar.addfile(directory) + + tar.add(self.extract, arcname=CONTAINER_EXTRACT.lstrip("/"), filter=_readable) + + config_bytes = json.dumps(self.config, indent=2).encode("utf-8") + config_info = tarfile.TarInfo(CONTAINER_CONFIG.lstrip("/")) + config_info.size = len(config_bytes) + config_info.mode = 0o644 + tar.addfile(config_info, io.BytesIO(config_bytes)) + + def _wait_until_ready(self) -> None: + assert self._container is not None + deadline = time.monotonic() + READINESS_TIMEOUT_SECONDS + while time.monotonic() < deadline: + try: + status = requests.get(f"{self.base_url}/status", timeout=2) + if status.ok: + version = status.json().get("version", "unknown") + log.info( + "the %s extract is served at %s (Valhalla %s)", + self.name, + self.base_url, + version, + ) + return + except (requests.RequestException, ValueError): + pass + self._container.reload() + if self._container.status not in ("created", "running"): + break + time.sleep(0.5) + + raise HarnessError( + f"valhalla_service did not become ready for the {self.name} extract; " + f"logs:\n{self.logs()}" + ) + + def logs(self) -> str: + if self._container is None: + return "" + return self._container.logs().decode("utf-8", errors="replace") + + def _remove_stale_container(self) -> None: + """Removes a container with the same name, left by an earlier run that crashed.""" + try: + self.client.containers.get(self.container_name).remove(force=True) + log.info("removed the stale container %s", self.container_name) + except docker.errors.NotFound: + pass + + def _remove(self) -> None: + if self._container is None: + return + try: + self._container.remove(force=True) + except docker.errors.NotFound: + pass + except docker.errors.APIError as e: + log.warning("cannot remove the container %s: %s", self.container_name, e) + self._container = None + + +def _readable(info: tarfile.TarInfo) -> tarfile.TarInfo: + """Makes a file in the data archive readable by any user of the container.""" + info.mode = 0o644 + info.uid = info.gid = 0 + info.uname = info.gname = "" + return info diff --git a/scripts/integration/sub-extract-tiles.txt b/scripts/integration/sub-extract-tiles.txt new file mode 100644 index 0000000..2cdfe7a --- /dev/null +++ b/scripts/integration/sub-extract-tiles.txt @@ -0,0 +1,14 @@ +# The tiles of the integration sub-extract, one level/tile_id per line. +# valinor-integration gives these to `valinor-cli extract --tile`. +# +# The list contains every tile of the Andorra fixture except the north-east corner +# (2/763927: longitude 1.75 to 2.0, latitude 42.5 to 42.75), +# so the sub-extract is a proper subset of the fixture. +# All coordinates in the Bruno collection (bruno/) are far from the removed tile. +# If you change this list, make sure that the requests stay inside the remaining tiles. +0/3015 +1/47701 +2/762485 +2/762486 +2/763925 +2/763926 diff --git a/scripts/integration/tests/test_checks.py b/scripts/integration/tests/test_checks.py new file mode 100644 index 0000000..39da2c5 --- /dev/null +++ b/scripts/integration/tests/test_checks.py @@ -0,0 +1,71 @@ +import copy + +from valinor_integration.checks import CHECKS, diff_projections + +ROUTE = { + "trip": { + "status": 0, + "units": "kilometers", + "locations": [{"lat": 1.0, "lon": 2.0}], + "summary": {"length": 1.5, "time": 90.0, "min_lat": 1.0}, + "legs": [ + { + "shape": "abc", + "summary": {"length": 1.5, "time": 90.0}, + "maneuvers": [{"type": 1, "instruction": "Go.", "length": 1.5, "extra": "x"}], + } + ], + } +} + +LOCATE = [ + { + "edges": [ + {"edge_id": {"value": 2}, "edge_info": {"way_id": 7}, "percent_along": 0.5}, + {"edge_id": {"value": 1}, "edge_info": {"way_id": 7}, "percent_along": 0.0}, + ] + } +] + + +def test_route_projection_ignores_excluded_fields() -> None: + other = copy.deepcopy(ROUTE) + other["trip"]["locations"] = [] + other["trip"]["summary"]["min_lat"] = 9.0 + other["trip"]["legs"][0]["maneuvers"][0]["extra"] = "y" + project = CHECKS["/route"].project + assert project(ROUTE) == project(other) + + +def test_route_projection_finds_differences() -> None: + other = copy.deepcopy(ROUTE) + other["trip"]["legs"][0]["shape"] = "abd" + project = CHECKS["/route"].project + diff = diff_projections("route/x", project(ROUTE), project(other), "full", "sub") + assert "--- full/route/x" in diff + assert '"shape": "abd"' in diff + + +def test_route_validation() -> None: + validate = CHECKS["/route"].validate + assert validate(ROUTE) == [] + assert validate({"trip": {"legs": []}}) != [] + assert validate([]) != [] + + +def test_locate_projection_is_order_independent() -> None: + reordered = [{"edges": list(reversed(LOCATE[0]["edges"]))}] + project = CHECKS["/locate"].project + assert project(LOCATE) == project(reordered) + assert project(LOCATE)[0][0]["way_id"] == 7 + + +def test_locate_validation_rejects_empty_results() -> None: + validate = CHECKS["/locate"].validate + assert validate(LOCATE) == [] + assert validate([{"edges": []}]) != [] + assert validate([]) != [] + + +def test_equal_projections_have_no_diff() -> None: + assert diff_projections("x", {"a": 1}, {"a": 1}, "full", "sub") == "" diff --git a/scripts/integration/tests/test_cli.py b/scripts/integration/tests/test_cli.py new file mode 100644 index 0000000..98b5f8f --- /dev/null +++ b/scripts/integration/tests/test_cli.py @@ -0,0 +1,36 @@ +from pathlib import Path + +import pytest + +from valinor_integration import cli, runner + + +@pytest.fixture +def calls(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, runner.Inputs]]: + recorded: list[tuple[str, runner.Inputs]] = [] + monkeypatch.setattr( + runner, "run", lambda settings, inputs: recorded.append(("run", inputs)) or True + ) + monkeypatch.setattr( + runner, "serve", lambda settings, inputs: recorded.append(("serve", inputs)) + ) + monkeypatch.delenv("VALINOR_CLI", raising=False) + return recorded + + +def test_run_is_the_default_command(calls: list[tuple[str, runner.Inputs]]) -> None: + assert cli.main([]) == 0 + assert cli.main(["--workdir", "somewhere"]) == 0 + assert [command for command, _ in calls] == ["run", "run"] + assert calls[1][1].workdir == Path("somewhere").resolve() + + +def test_serve_keeps_its_options(calls: list[tuple[str, runner.Inputs]]) -> None: + assert cli.main(["serve", "--workdir", "somewhere"]) == 0 + assert calls == [("serve", calls[0][1])] + assert calls[0][1].workdir == Path("somewhere").resolve() + + +def test_failed_run_exits_with_one(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(runner, "run", lambda settings, inputs: False) + assert cli.main([]) == 1 diff --git a/scripts/integration/tests/test_collection.py b/scripts/integration/tests/test_collection.py new file mode 100644 index 0000000..bcaff1c --- /dev/null +++ b/scripts/integration/tests/test_collection.py @@ -0,0 +1,95 @@ +from pathlib import Path + +import pytest + +from valinor_integration.collection import load_collection, parse_request +from valinor_integration.errors import HarnessError +from valinor_integration.settings import DEFAULT_COLLECTION + +REQUEST = """\ +info: + name: example + type: http + seq: 2 + +http: + method: post + url: "{{baseUrl}}/route" + headers: + - name: X-Enabled + value: "yes" + - name: X-Disabled + value: "no" + disabled: true + body: + type: json + data: |- + {"costing": "auto"} + auth: inherit + +settings: + encodeUrl: true +""" + + +def test_parse_request() -> None: + request = parse_request(Path("route/example.yml"), REQUEST) + assert request.name == "example" + assert request.id == "route/example" + assert request.seq == 2 + assert request.method == "POST" + assert request.headers == {"X-Enabled": "yes"} + assert request.body == {"costing": "auto"} + assert request.resolve_url({"baseUrl": "http://localhost:1"}) == "http://localhost:1/route" + assert request.endpoint({"baseUrl": "http://localhost:1"}) == "/route" + assert request.endpoint({"baseUrl": ""}) == "/route" + + +def test_unknown_variable_is_an_error() -> None: + request = parse_request(Path("route/example.yml"), REQUEST) + with pytest.raises(HarnessError, match="unknown variable"): + request.resolve_url({}) + + +@pytest.mark.parametrize( + ("replace", "by", "message"), + [ + ("settings:", "runtime:\n scripts: []\nsettings:", "unsupported sections"), + (" auth: inherit", " params: []", "unsupported http fields"), + (" auth: inherit", " auth:\n type: bearer", "authentication"), + (" type: json", " type: xml", "only JSON bodies"), + ('{"costing": "auto"}', '{"costing": ', "not valid JSON"), + (" type: http", " type: graphql", "unsupported request type"), + (" name: example", " name: ''", "info.name"), + ], +) +def test_unsupported_requests_are_rejected(replace: str, by: str, message: str) -> None: + assert replace in REQUEST + with pytest.raises(HarnessError, match=message): + parse_request(Path("route/example.yml"), REQUEST.replace(replace, by, 1)) + + +def test_load_collection_order_and_skipped_files(tmp_path: Path) -> None: + (tmp_path / "opencollection.yml").write_text("opencollection: 1.0.0\n") + (tmp_path / "environments").mkdir() + (tmp_path / "environments" / "local.yml").write_text("name: local\n") + (tmp_path / "b").mkdir() + (tmp_path / "b" / "folder.yml").write_text("info:\n type: folder\n") + (tmp_path / "b" / "second.yml").write_text(REQUEST.replace("name: example", "name: second")) + (tmp_path / "b" / "first.yml").write_text( + REQUEST.replace("name: example", "name: first").replace("seq: 2", "seq: 1") + ) + + assert [request.id for request in load_collection(tmp_path)] == ["b/first", "b/second"] + + +def test_load_collection_requires_a_collection_file(tmp_path: Path) -> None: + with pytest.raises(HarnessError, match="not a Bruno collection"): + load_collection(tmp_path) + + +def test_the_repository_collection_loads() -> None: + requests = load_collection(DEFAULT_COLLECTION) + endpoints = {request.endpoint({"baseUrl": ""}) for request in requests} + assert endpoints == {"/route", "/locate"} + assert all(request.body for request in requests) diff --git a/scripts/integration/tests/test_extract.py b/scripts/integration/tests/test_extract.py new file mode 100644 index 0000000..e539262 --- /dev/null +++ b/scripts/integration/tests/test_extract.py @@ -0,0 +1,52 @@ +import tarfile +from pathlib import Path + +import pytest + +from valinor_integration.errors import HarnessError +from valinor_integration.extract import check_proper_subset, read_tile_list +from valinor_integration.settings import DEFAULT_TILE_LIST + + +def _archive(path: Path, names: list[str]) -> Path: + with tarfile.open(path, "w") as tar: + for name in names: + tar.addfile(tarfile.TarInfo(name)) + return path + + +def test_read_tile_list(tmp_path: Path) -> None: + tile_list = tmp_path / "tiles.txt" + tile_list.write_text("# comment\n\n0/3015\n 2 / 762485 # inline comment\n") + assert read_tile_list(tile_list) == ["0/3015", "2/762485"] + + +def test_empty_tile_list_is_an_error(tmp_path: Path) -> None: + tile_list = tmp_path / "tiles.txt" + tile_list.write_text("# only a comment\n") + with pytest.raises(HarnessError, match="no tiles"): + read_tile_list(tile_list) + + +def test_the_repository_tile_list_reads() -> None: + assert len(read_tile_list(DEFAULT_TILE_LIST)) == 6 + + +def test_proper_subset(tmp_path: Path) -> None: + full = _archive(tmp_path / "full.tar", ["index.bin", "0/a.gph", "2/b.gph"]) + sub = _archive(tmp_path / "sub.tar", ["index.bin", "0/a.gph"]) + assert check_proper_subset(full, sub) == ["2/b.gph"] + + +def test_identical_extracts_prove_nothing(tmp_path: Path) -> None: + full = _archive(tmp_path / "full.tar", ["index.bin", "0/a.gph"]) + sub = _archive(tmp_path / "sub.tar", ["index.bin", "0/a.gph"]) + with pytest.raises(HarnessError, match="would prove nothing"): + check_proper_subset(full, sub) + + +def test_extra_entries_are_an_error(tmp_path: Path) -> None: + full = _archive(tmp_path / "full.tar", ["index.bin", "0/a.gph"]) + sub = _archive(tmp_path / "sub.tar", ["index.bin", "1/c.gph"]) + with pytest.raises(HarnessError, match="not in the full extract"): + check_proper_subset(full, sub) diff --git a/scripts/integration/tests/test_settings.py b/scripts/integration/tests/test_settings.py new file mode 100644 index 0000000..06b444d --- /dev/null +++ b/scripts/integration/tests/test_settings.py @@ -0,0 +1,56 @@ +from pathlib import Path + +import pytest + +from valinor_integration.errors import HarnessError +from valinor_integration.settings import DEFAULT_COLLECTION, DEFAULT_FULL_EXTRACT, Settings + + +def test_defaults_use_latest() -> None: + settings = Settings.from_env({}) + assert settings.image_ref == "ghcr.io/valhalla/valhalla:latest" + assert not settings.is_pinned + assert settings.valinor_cli is None + assert settings.serve_port == 8002 + + +def test_empty_values_are_unset() -> None: + assert Settings.from_env({"VALHALLA_VERSION": " ", "VALINOR_CLI": ""}) == Settings.from_env({}) + + +def test_pinned_version_and_image(tmp_path: Path) -> None: + cli = tmp_path / "valinor-cli" + cli.touch() + settings = Settings.from_env( + { + "VALHALLA_VERSION": "3.8.2", + "VALHALLA_IMAGE": "example.com/valhalla", + "VALINOR_CLI": str(cli), + "VALHALLA_PORT": "9000", + } + ) + assert settings.image_ref == "example.com/valhalla:3.8.2" + assert settings.is_pinned + assert settings.valinor_cli == cli + assert settings.serve_port == 9000 + + +@pytest.mark.parametrize( + "env", + [ + {"VALHALLA_VERSION": "ghcr.io/valhalla/valhalla:3.8.2"}, + {"VALHALLA_VERSION": "3.8.2@sha256:abc"}, + {"VALHALLA_PORT": "http"}, + {"VALHALLA_PORT": "65535"}, + {"VALHALLA_PORT": "0"}, + {"VALINOR_CLI": "/does/not/exist"}, + ], +) +def test_invalid_values(env: dict[str, str]) -> None: + with pytest.raises(HarnessError): + Settings.from_env(env) + + +def test_default_paths_exist() -> None: + assert DEFAULT_FULL_EXTRACT.is_file() + assert DEFAULT_COLLECTION.is_dir() diff --git a/scripts/integration/uv.lock b/scripts/integration/uv.lock new file mode 100644 index 0000000..bab11d4 --- /dev/null +++ b/scripts/integration/uv.lock @@ -0,0 +1,370 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" + +[[package]] +name = "certifi" +version = "2026.10.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/33/1c/f41d4e74c28ab327ff3acd36053f7ea506c55872d7a90b0fa71aa3ab0c89/charset_normalizer-3.5.2.tar.gz", hash = "sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef", size = 172659, upload-time = "2026-09-30T04:39:23.398Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/c8/693809898870237d82785a03f3b2b58fe4c9f14669f84a7d4e623c92a59e/charset_normalizer-3.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491", size = 367780, upload-time = "2026-09-30T04:35:30.888Z" }, + { url = "https://files.pythonhosted.org/packages/c9/87/2fea8c13dc24b3ca9c6f803a5b2dfdeae73eb4f9e12c7885ed908ff0433c/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c", size = 246730, upload-time = "2026-09-30T04:35:32.286Z" }, + { url = "https://files.pythonhosted.org/packages/a8/9e/09efac30b937722f46d3110ba30b875b24b2e3a266ed746cc4e376a94d80/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0", size = 237707, upload-time = "2026-09-30T04:35:33.709Z" }, + { url = "https://files.pythonhosted.org/packages/9e/18/70d76670b13686237863a379928d60bd10e021f17d243ab3d7014c4a5f4e/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51", size = 273050, upload-time = "2026-09-30T04:35:35.138Z" }, + { url = "https://files.pythonhosted.org/packages/54/e2/77a8b09d5adc013ed07b95b01b8b8fa5441c4e810e83ee7e4aae2fa4d91a/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5", size = 270345, upload-time = "2026-09-30T04:35:36.502Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c5/38806a25ab5e65fc178f39affeda20858efafede2fce1ffc2556cfc9fe73/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649", size = 257601, upload-time = "2026-09-30T04:35:37.919Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8d/213565184708fdb263ae55e2c04ee1ff748129dd65d48ed0e3502da9c85a/charset_normalizer-3.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e", size = 252222, upload-time = "2026-09-30T04:35:39.544Z" }, + { url = "https://files.pythonhosted.org/packages/7e/24/76d2cefc25472531e4c5c7dfff68865eb1c39b78482f0fdc15b46f047830/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346", size = 248482, upload-time = "2026-09-30T04:35:41.088Z" }, + { url = "https://files.pythonhosted.org/packages/7d/dc/65a801b66ab4c197e22c433ab25e7ac24324ac6f45a2269aca42cce309bf/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1", size = 241206, upload-time = "2026-09-30T04:35:42.59Z" }, + { url = "https://files.pythonhosted.org/packages/a7/95/ca9b5eabde673002c6f1e7ada1b223916fe18f6d661da7aabd4d643718f1/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875", size = 273190, upload-time = "2026-09-30T04:35:44.347Z" }, + { url = "https://files.pythonhosted.org/packages/2d/8b/803b4d2a3f6e1740f63f1e87b04d14b42f3d4fdfe6ed7d4db2d34102b14f/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1", size = 253527, upload-time = "2026-09-30T04:35:45.915Z" }, + { url = "https://files.pythonhosted.org/packages/a9/55/93c0e5dbd085ae0471346026abbe7e0db9ea2d6fea74e51f0b5a46f233a7/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413", size = 271285, upload-time = "2026-09-30T04:35:47.49Z" }, + { url = "https://files.pythonhosted.org/packages/95/69/0dbd0e0b9b16cfa816cdfcb3e2e3854a1f680dc07fb1245ea125e7448060/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869", size = 260010, upload-time = "2026-09-30T04:35:48.996Z" }, + { url = "https://files.pythonhosted.org/packages/58/9d/e7b88e7b1bf403590c3b573277b5e1e488c68c7a6fbacca310a2c324e90c/charset_normalizer-3.5.2-cp312-cp312-win32.whl", hash = "sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e", size = 184125, upload-time = "2026-09-30T04:35:50.777Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e6/e6e083884cbcfd49c64865af05027fe7011be7b2d9179524f099a1b611f3/charset_normalizer-3.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc", size = 207486, upload-time = "2026-09-30T04:35:52.194Z" }, + { url = "https://files.pythonhosted.org/packages/c4/e3/017aea0911ada7405a825c7d937eb3a13009664e2f5b38e8c4bbf2abf894/charset_normalizer-3.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3", size = 196734, upload-time = "2026-09-30T04:35:53.636Z" }, + { url = "https://files.pythonhosted.org/packages/c5/34/68292d68512768591aaff07c59bb53ee31341c87759433a859c4641a50c2/charset_normalizer-3.5.2-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5", size = 225946, upload-time = "2026-09-30T04:35:55.313Z" }, + { url = "https://files.pythonhosted.org/packages/e3/80/bee0b01b90ccd5322ae1d0abb33fab1bd95b7c2eadaf02aeccf22e04ee83/charset_normalizer-3.5.2-cp313-cp313-android_24_x86_64.whl", hash = "sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e", size = 238619, upload-time = "2026-09-30T04:35:56.863Z" }, + { url = "https://files.pythonhosted.org/packages/78/6e/60ce52a85a7fd631ae8482ae6d74521014ca2f255892679484dc04d7ef56/charset_normalizer-3.5.2-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a", size = 206701, upload-time = "2026-09-30T04:35:58.639Z" }, + { url = "https://files.pythonhosted.org/packages/36/8c/71aafad23f971afc84c2b295bc0c560739ce1dac558aad9fec22e39f3639/charset_normalizer-3.5.2-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:d376bbd28b3a8999db1a103b3b388aee6f1ddeb3e51bc2172993efdcd86e064d", size = 210506, upload-time = "2026-09-30T04:36:00.147Z" }, + { url = "https://files.pythonhosted.org/packages/91/da/3c5a7798c046df7d2d68ad653cf5b6c5a8bfee225055a843c6f2f42aac1a/charset_normalizer-3.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6045373d5a89a5ec71afde535db987ca28e76dfa276c2d4c818265b375d4b055", size = 366626, upload-time = "2026-09-30T04:36:01.77Z" }, + { url = "https://files.pythonhosted.org/packages/e1/16/710ac3de2ee354e2bd1a9c94efe45a2d27b5c6ad39b2d6a905be2c094b6c/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:849df64e889b2e17230d58410a03dba311a65b163508fd33679b2b737d4b7858", size = 244624, upload-time = "2026-09-30T04:36:03.389Z" }, + { url = "https://files.pythonhosted.org/packages/d6/39/45c7439f5b63d24f7d5b2a1d760f34af7628782d7144b4cc8ded45c2d4bc/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:15c44f7edfd477b06f517a5cc317fc1707edb9de2c865f43d4b6513907473234", size = 235731, upload-time = "2026-09-30T04:36:04.987Z" }, + { url = "https://files.pythonhosted.org/packages/4d/34/38f3154785ce92e9f56eb226f4d35bdfae6b008480dd055f58837a89c810/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a89012d6d5476ee112d20d998570ed58df2260a852afb1758809cd6900411d21", size = 270352, upload-time = "2026-09-30T04:36:06.412Z" }, + { url = "https://files.pythonhosted.org/packages/04/f3/859f74e7babc977705026b30593b3be04049632a522fb7000f83c033d747/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0c951d5e6dd9c2ff60609476752bee49da4206adde960ebc247766937f72e718", size = 267393, upload-time = "2026-09-30T04:36:07.865Z" }, + { url = "https://files.pythonhosted.org/packages/4b/85/41d27f234b82e47c167a5f6c0f62501dc0c640585ff4aba79e08a390336a/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4", size = 254660, upload-time = "2026-09-30T04:36:09.248Z" }, + { url = "https://files.pythonhosted.org/packages/58/ca/5d1a997587febe5b26d8daffe363b5c1a091cece19828eec6502fd09c5ef/charset_normalizer-3.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a19a731138fc27d5682277d3b9df22855cea1239bce7fcec5f78f42ef2d1f3c3", size = 249996, upload-time = "2026-09-30T04:36:10.73Z" }, + { url = "https://files.pythonhosted.org/packages/b3/1f/d1e78246f7ed60c8c8d606b4ac27f66ce49cc3e95f24893ccbeba9f77302/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:62603db9a7caa0802eaa28c1c46fecd7b3a263a774069c24c3c28c302448721c", size = 247028, upload-time = "2026-09-30T04:36:12.294Z" }, + { url = "https://files.pythonhosted.org/packages/8e/37/eba316edd4f0c4d3a5d945924c4eeeae59abac4056aa815d8a4268f863a2/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:b6856554c4f44d79fc2307d5768854310a8f0096e501c75637542c82292b0429", size = 240562, upload-time = "2026-09-30T04:36:13.887Z" }, + { url = "https://files.pythonhosted.org/packages/c8/8e/aaa037d40ca9ef045977f1a661048b1aa33f223adfce3452fe9be9f79d14/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:1bc0baf5ef96b6ede57d47f4b8fe4d9d84019c3bfcbeb20a41edc6a6ee341f1f", size = 271699, upload-time = "2026-09-30T04:36:15.41Z" }, + { url = "https://files.pythonhosted.org/packages/26/19/1c1c9f75974adf523b87f34b8a2adc5a435cd65916812bcbd0dfa45f9a29/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:56bc200a365efb37383b7852e4cc5898d3b2da5987289b543956cf8cad71018a", size = 252030, upload-time = "2026-09-30T04:36:16.839Z" }, + { url = "https://files.pythonhosted.org/packages/bc/90/0660ef18e18df0a4d2a1a0edff7dfbba42d4e50ef2425557a5bb7051f77b/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:2c9ad19a6cfcd5ea5c0d41161d22f9df1dcc277e9bef2751391334546a314c00", size = 268565, upload-time = "2026-09-30T04:36:18.468Z" }, + { url = "https://files.pythonhosted.org/packages/79/ba/57adc269824e8658f1a0f97a9e514c247445a9632b3419b97e0ba37f16dc/charset_normalizer-3.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e243bd13217235fc7290c621941c3f5cc8b66e4872495be821d7436ba2fb838d", size = 257627, upload-time = "2026-09-30T04:36:19.938Z" }, + { url = "https://files.pythonhosted.org/packages/9a/85/33abd4315c052d3d4f54c92b1ee49bfbc0dc7115a981e462a793b6d2ab87/charset_normalizer-3.5.2-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:a090bb2c68df85450502e3e20d665e3a5af9c65a84d6508ed477badd49166fd3", size = 143407, upload-time = "2026-09-30T04:36:21.376Z" }, + { url = "https://files.pythonhosted.org/packages/4f/de/6435e18d1aaa5d910b896d551411c96af1f42a0c56c29afc2016c61ccc2e/charset_normalizer-3.5.2-cp313-cp313-win32.whl", hash = "sha256:2b7b3bbfb4fe8ef40600792d762fbaa9057559f9d3fad209525b7a22b99e91fd", size = 183207, upload-time = "2026-09-30T04:36:22.776Z" }, + { url = "https://files.pythonhosted.org/packages/9c/76/b8ec57f4e9ee3253541abf95e4a462c0175fe8032dcd070f1f2421240942/charset_normalizer-3.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:78456a747de8dc58360ffa581f30a002baf5aa28cb262536545e91f113ed7639", size = 206489, upload-time = "2026-09-30T04:36:24.306Z" }, + { url = "https://files.pythonhosted.org/packages/3e/60/c647c6ae47480221e875ea5d743ff94946f7416e3c69415ab772928e8d32/charset_normalizer-3.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:11912e4bb14baae7c5d8791aa55ba0a3a03ec6729073307b0f57270abaa713d3", size = 195501, upload-time = "2026-09-30T04:36:25.846Z" }, + { url = "https://files.pythonhosted.org/packages/58/ca/7aa91362a2f77ac8e9e28a9b902a74f7d0e11a851ef0d27a74308da8cd90/charset_normalizer-3.5.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:1afb975bd5d68d5ce9f6b6d44fdf2f7e34b895a35e95708a7a91b20a3b51d187", size = 226010, upload-time = "2026-09-30T04:36:27.669Z" }, + { url = "https://files.pythonhosted.org/packages/a8/cf/ac8878d0322cf88a1aad4c7b147db32ca0bd806eb0060957b2e31486dbe6/charset_normalizer-3.5.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:bbbfc8e28816f19d7c0f1816664980c0a9875d01b27cdf8eedddb639d9e108ad", size = 239266, upload-time = "2026-09-30T04:36:29.434Z" }, + { url = "https://files.pythonhosted.org/packages/c9/6d/9a08d7e0b29b7208e2c6c01dc56c8e0520e7c7beadbbfb024b58fd69c8a5/charset_normalizer-3.5.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7967d08cf06dee78443b874f98c98036f624f3a4e73e11f9f64f5be4d25393cf", size = 206783, upload-time = "2026-09-30T04:36:30.872Z" }, + { url = "https://files.pythonhosted.org/packages/82/44/b0aa350280e6ff5a5492d17cf10460dd39d5ee848f872f7ba2df10607f60/charset_normalizer-3.5.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4c2b5031f63e331e3839b40aed2dd6f191e9c07edbde303e7876846ea1946995", size = 210449, upload-time = "2026-09-30T04:36:32.625Z" }, + { url = "https://files.pythonhosted.org/packages/7c/8a/40db9aa9f5907bb0e6f8b6d64064bf8852fb33d4b813ff9414911df7647c/charset_normalizer-3.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424", size = 367815, upload-time = "2026-09-30T04:36:34.197Z" }, + { url = "https://files.pythonhosted.org/packages/7f/72/9c5e7707b57c8ddfa9ddf7b0b1d009d7fbab9e9e887d5b721060f37e307d/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8d86d6fc60743dc916eb79e2eb1ec4818e21e427731543af40a3021851174a13", size = 247095, upload-time = "2026-09-30T04:36:35.803Z" }, + { url = "https://files.pythonhosted.org/packages/83/09/71e453691e927de4ddf792770cfaab3f49d494e222f66ea5e404bbd5e39c/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7a881931aa470808df94a8c380eed2bbbc76cd9dc622310f99665658c821eb6d", size = 233942, upload-time = "2026-09-30T04:36:37.407Z" }, + { url = "https://files.pythonhosted.org/packages/9f/86/85c84e4da8b27dd409577d9437926ff581c5f9d3c66038dc68c1a526de51/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8024d00c3faf3fc0c16e07a69f4405e8eac7cc0ab15f65fe6cf43827c4cf72b4", size = 273762, upload-time = "2026-09-30T04:36:38.904Z" }, + { url = "https://files.pythonhosted.org/packages/92/08/564955a4b5f2ccb410ab480bbe8c6a18063ff27f2d35458731c4a5335df9/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4d48f2d08b9de5864e2c8744d4461b862fb149a18274abc8b698c45975573438", size = 269514, upload-time = "2026-09-30T04:36:40.469Z" }, + { url = "https://files.pythonhosted.org/packages/18/24/bad3ac4271589df29cf5ce2f5ae490518a5739358052bd0d61209e6fea54/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34276fd796040bf0993ab33a369aa572e6979c7aab225a88893667ad8eac8f7a", size = 255056, upload-time = "2026-09-30T04:36:42.02Z" }, + { url = "https://files.pythonhosted.org/packages/d6/3e/350d89ad49916b86554d6f5f2d03ec1152148f87e5ff735106c6a03b1a36/charset_normalizer-3.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0521c5665880b33d603717defa76c094048900010897909952397feb3039da56", size = 251311, upload-time = "2026-09-30T04:36:43.577Z" }, + { url = "https://files.pythonhosted.org/packages/56/5b/4970a2d154df502e133402906dd04e3ae7cada7b3011283c88d0479a2585/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:eff0ac9dbe711a4aee69bf04a83896aa9b85f19641264053a9f6d48573abb7dd", size = 250190, upload-time = "2026-09-30T04:36:45.185Z" }, + { url = "https://files.pythonhosted.org/packages/88/8c/f1a91bddc8fb47c2889e29ea7ea49a194eb0d9868675d786806519c00d76/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1503bccbeb36d5527790c3930327704c39af22de3112f1b1666a9f3ce15ee204", size = 237098, upload-time = "2026-09-30T04:36:46.689Z" }, + { url = "https://files.pythonhosted.org/packages/24/0e/bb5dace3cc7e79068425386a6589c19b5a2ab5fefc2a46abea6919683332/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:52aa6992700996af31f375de0c6bacd402b0097fe40b53c426b9f51a90ebabc7", size = 274228, upload-time = "2026-09-30T04:36:48.31Z" }, + { url = "https://files.pythonhosted.org/packages/9d/79/b849ad523017ea9f5a45581bbebed91439e0cf42fd2860a6f64e358eb5a6/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e09a3942ecbdee5cce73ea9d42da82b81b72ac1bf031ce069b93b5adf4eac8cd", size = 252401, upload-time = "2026-09-30T04:36:50.091Z" }, + { url = "https://files.pythonhosted.org/packages/89/8c/75469d690cf47200bce8f6cad7655724fc23148e147abfc5ce78b5f65863/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:c7c9ab723cde841fefb34efbad91e87f00a674b1fe1cd0784fde742bf2c154dc", size = 271846, upload-time = "2026-09-30T04:36:51.719Z" }, + { url = "https://files.pythonhosted.org/packages/26/cd/6d52d3c7437cdcf2e310ce9f28f282e733d4ef60ed19105d1819c356255f/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ddc7dacc8ece3a182e7f15cb862d1fd616b46d076cb1ae9dd232b2c38b655874", size = 258151, upload-time = "2026-09-30T04:36:53.234Z" }, + { url = "https://files.pythonhosted.org/packages/f7/4c/070b38bdb5f49a70199fce923ec0726a49536a63ab262abbfcaaf351110b/charset_normalizer-3.5.2-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:ee43c17b173d46a3212baa6ead3ae258eeabdae48c263a01ccf0218c366dd655", size = 143570, upload-time = "2026-09-30T04:36:54.816Z" }, + { url = "https://files.pythonhosted.org/packages/81/84/9ebfc8ed6c8c4fcd8e726ff6bf220cc8deb3966e31dce9be8dd8aa017e64/charset_normalizer-3.5.2-cp314-cp314-win32.whl", hash = "sha256:4f87960d57feabfb618e4e0af6e7371645fa26a277860739d6e5d6e0012c92f0", size = 186412, upload-time = "2026-09-30T04:36:56.643Z" }, + { url = "https://files.pythonhosted.org/packages/d1/78/5ed86f743d4bc350db307e7636419a0a5ee1d91806d30c7f667bd5c80dae/charset_normalizer-3.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:e4e81e09c1578b8df602e3db08b0b3ea0a6947ad612f52bf8dc5ea8d47691f0c", size = 210965, upload-time = "2026-09-30T04:36:58.205Z" }, + { url = "https://files.pythonhosted.org/packages/53/94/a3a7698e9b1a395e1eb99ccd9a324be9347973bff4e72db2a06496d7cd27/charset_normalizer-3.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:80d02b6f04e92601a081dd97b23d3128033098bff5d35d392ddcc0476ea11253", size = 200406, upload-time = "2026-09-30T04:36:59.764Z" }, + { url = "https://files.pythonhosted.org/packages/c1/48/c5dd00d5ef7791f02666de250a5bb6071e29b7e133cf4b835800b6d3bc27/charset_normalizer-3.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:dca9ab98072a5a54ebacebdc45f53e645336b320c667410b061be1ca588ae709", size = 396871, upload-time = "2026-09-30T04:37:01.543Z" }, + { url = "https://files.pythonhosted.org/packages/12/c8/8379554b42e8368161d898476686947a0fdbd3e8865170d7909dcabfdee8/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0aa869112ef88429ae17820d99c3dd9504c9e9c671d3c246f3d7442cb051084", size = 249711, upload-time = "2026-09-30T04:37:03.111Z" }, + { url = "https://files.pythonhosted.org/packages/4a/eb/2ddb1035d17320caa9f41682935123a9a250277b261c3efc86b2d2a21343/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c0afc6800ba57ccc350374c5bd6150419915d95ce93cdbab2d783d75eaf30ecb", size = 231146, upload-time = "2026-09-30T04:37:04.721Z" }, + { url = "https://files.pythonhosted.org/packages/4a/24/2ecb4bde104322cd7859d6594fcfa74649f8d90b3221c9feecbef149875b/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7dcd882da75ef9adf94903b1e3b9419e8aa8fb4c7396822b834b9ef7fb96954f", size = 271552, upload-time = "2026-09-30T04:37:06.295Z" }, + { url = "https://files.pythonhosted.org/packages/3f/98/9d5f6ebc3aee9fef5d30b4aff11fb2ab7a1222b4064f8ef2c7c87cde217a/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2e06a3a98f916dd41d27f3105e02e7a40181c98c94b9158733d03a6f80506c09", size = 269810, upload-time = "2026-09-30T04:37:07.905Z" }, + { url = "https://files.pythonhosted.org/packages/09/e1/a3b06a10461b1b7628853c934c644e03bc28e42767116afb52f19a56519b/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6bd128f206a7752ae1f2ab6c61bf8a24ba28913a10df8b14c2637b973ff97a80", size = 259845, upload-time = "2026-09-30T04:37:09.554Z" }, + { url = "https://files.pythonhosted.org/packages/fd/d3/6f561f74a296cf27d61775a1dc665ad13f3bff6a798810ca05907f37a7c4/charset_normalizer-3.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c8f3d67aeaf55f017982b73683f0e7342ba2f6635a78f69ce89ebb26aa411e5c", size = 252825, upload-time = "2026-09-30T04:37:11.274Z" }, + { url = "https://files.pythonhosted.org/packages/26/9f/69e13ca3b18f43e0eafcd34c04a45b732ae22a43b54a5fc9e119103356eb/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f", size = 252659, upload-time = "2026-09-30T04:37:12.941Z" }, + { url = "https://files.pythonhosted.org/packages/73/a9/ace29806a0dae18939919c76ba526472d83214afa101105fabff2cf30625/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:92888bb3187c5ba50500b00b3b310c9f2c651709d28036077680cb5255450a03", size = 238819, upload-time = "2026-09-30T04:37:14.659Z" }, + { url = "https://files.pythonhosted.org/packages/f8/c1/6116d52a2e3311ec80f21f5fb5e17b27405f10b9608af8f6e69516841a1b/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:d008d90a7f2471519aef0c90dfbe73b3e6e4d5e66ac48e19154c17e89e98b604", size = 272973, upload-time = "2026-09-30T04:37:16.346Z" }, + { url = "https://files.pythonhosted.org/packages/19/aa/9955c7e93bba10a9c7e8f7a5031b7ced66f3a1883a55c00712b8d5850ff3/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:31f3930700408d211f13378ccbe1c40845d8da54bd0681fac3a9b5aae81c7aa8", size = 253692, upload-time = "2026-09-30T04:37:18.212Z" }, + { url = "https://files.pythonhosted.org/packages/bb/33/2a6ae7fdc1b10cb581cef91addd8cdfc5f40d50abb5702309369d5834579/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:2a925889534b3748302dae5dead07cc13480de1dac3aea80a941b729b471ef93", size = 268757, upload-time = "2026-09-30T04:37:19.877Z" }, + { url = "https://files.pythonhosted.org/packages/a2/22/80992720a0282cd39bba1db35868e6b9c22f41281160143a836544bc1d8a/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f5ec61164adcec446f8969a3358ec3f9b26bbda3b9213e5586d219afa8df2915", size = 262220, upload-time = "2026-09-30T04:37:21.583Z" }, + { url = "https://files.pythonhosted.org/packages/92/9f/181fd07e1bffea1d95cd80c84ac537354f50699c22cfc4d3c02b6fc16208/charset_normalizer-3.5.2-cp314-cp314t-win32.whl", hash = "sha256:598a11a2c7ebaa5334bf698bf29568c9c390abac6a154d8170fedecd1cea38c5", size = 197797, upload-time = "2026-09-30T04:37:23.235Z" }, + { url = "https://files.pythonhosted.org/packages/49/1c/25d8415ec1c4f2f41f1680435e4c87cfb378ff2f677d950946f2a45d0632/charset_normalizer-3.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:7fdde2c9fd9e3eca40631e024664cf2584272cc8f96308cbe5fdfc930f51d8bc", size = 222765, upload-time = "2026-09-30T04:37:24.891Z" }, + { url = "https://files.pythonhosted.org/packages/3e/b4/46b48f013dadfc0d0d33b375438e31bdf5a989dc68389c6bf627054d4df9/charset_normalizer-3.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d1befeed746d247c81127bb14de9dc3d30edb6e5976d34f83f86ed262b1d9105", size = 210985, upload-time = "2026-09-30T04:37:26.634Z" }, + { url = "https://files.pythonhosted.org/packages/ca/e9/34e597dee616d0b8ee4b34d29399e85c2204ade174157a48505d42baa4ff/charset_normalizer-3.5.2-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26", size = 367676, upload-time = "2026-09-30T04:37:28.329Z" }, + { url = "https://files.pythonhosted.org/packages/60/9f/a5d1c91c0263745e2cd344c5a4415d787c575501ab1d449f1148ac6b495d/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9409a8bf35cf78353942504b24a57de3d75b708997a1e4bd8db71ac8633ce364", size = 248414, upload-time = "2026-09-30T04:37:30.167Z" }, + { url = "https://files.pythonhosted.org/packages/26/79/e697f77464748a3ee3cf490c83d592459400d4898380d66c38366b03080c/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:498dc3188ca05a68231ac3fdbfc7f57eb67e1343c30e0fea17f8218c1599b253", size = 244775, upload-time = "2026-09-30T04:37:31.964Z" }, + { url = "https://files.pythonhosted.org/packages/ca/87/3d42a42e18ea066e2513936fd678a00696e77878b5ae04528976abdbcb83/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e242bb1c5e76e97dfa9e7f209a71e93a01d7f19ffdd5cfbb2e2d55b4f08f8ab0", size = 273877, upload-time = "2026-09-30T04:37:33.661Z" }, + { url = "https://files.pythonhosted.org/packages/c3/76/8a28136f3938ba9836f84280ce0c4d61ed1cf15a036b2034900c62634162/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:def79fa35ef0cef8d2accec024f4fdc7ead3012ff02f5215c783f39f03ef8cfc", size = 269455, upload-time = "2026-09-30T04:37:35.573Z" }, + { url = "https://files.pythonhosted.org/packages/a0/a1/4fbf5d0f0f1b2a080474c1cf9a2f12c4c6531bb0e8ba591055e846d2b4e9/charset_normalizer-3.5.2-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229", size = 257336, upload-time = "2026-09-30T04:37:37.397Z" }, + { url = "https://files.pythonhosted.org/packages/ba/a2/8b50aa320adb880ad579518e6f718f24944804b42a88b83d267d5d444125/charset_normalizer-3.5.2-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:04851f73ae72b8413dddadb16a49dfee95263553741fd42d546f7d66907e6be5", size = 257584, upload-time = "2026-09-30T04:37:39.522Z" }, + { url = "https://files.pythonhosted.org/packages/a5/57/50e3fed84e175f40349bd0da7a4fce94c87f0378f52d74f511d89e0bdc20/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:183b88127acdb4fabe59d951ab424faf1af7b63cdbb5f776186c1ea2ffcaed98", size = 251854, upload-time = "2026-09-30T04:37:41.23Z" }, + { url = "https://files.pythonhosted.org/packages/d6/54/f7fbb3493c9f49091213b9c2d6dd65800696f1ce1a3f196a4205f50417b1/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:16fa0eccf81304b79c5cd87f9271c3b85dd9dd99245e4422ae9c0dd45e0f99d3", size = 248767, upload-time = "2026-09-30T04:37:42.883Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/b3a6385acc5a1e45b39ae9c90bfb9cf838a09b9dd37ef2740ab4c6b4a2eb/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:7441d755b7ab94f8d4eb3e43ec05482d760842fd263d003a99102d742cd835e2", size = 274479, upload-time = "2026-09-30T04:37:44.658Z" }, + { url = "https://files.pythonhosted.org/packages/89/44/809913e2cfd279e635a9294fdbbfb1b1dc62a8189d473d561f649fce98d8/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:ca403d7e4798f525fdfc78e258820419cbbd0f0ecbab9de7840e3c017cf6b8cf", size = 258310, upload-time = "2026-09-30T04:37:46.529Z" }, + { url = "https://files.pythonhosted.org/packages/af/a2/f28400ab13359d91bd39179df8e149376b9bf36588e739a3a4f9de2b84b2/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:df29a0a7107f7011e77f4eebdddec4c7331e24d787a0b21a46d63bdf7445da95", size = 271100, upload-time = "2026-09-30T04:37:48.399Z" }, + { url = "https://files.pythonhosted.org/packages/e9/89/9bab37955edf0adb3b66f8a3a6617d9f2f487e0d56f295a6a286cb640aa6/charset_normalizer-3.5.2-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:f3c96f633825733f735c5a9cf21d21a257d8e1edf0b1cee0a064b9c424ca0f7d", size = 259936, upload-time = "2026-09-30T04:37:50.023Z" }, + { url = "https://files.pythonhosted.org/packages/23/b5/4459e08d45a679f903d50fea08bc52cfa728cca4d7bd02c757b5e5abda2e/charset_normalizer-3.5.2-cp315-cp315-win32.whl", hash = "sha256:281cb91036248400f4cc957495cccd44c275c2e0c5854f7e45ac5cf7dc193847", size = 186301, upload-time = "2026-09-30T04:37:51.722Z" }, + { url = "https://files.pythonhosted.org/packages/98/e8/55d5fd3935b4bce6da4fe0df61898e8c82653e317e677bd58aceb9c60f13/charset_normalizer-3.5.2-cp315-cp315-win_amd64.whl", hash = "sha256:89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8", size = 210858, upload-time = "2026-09-30T04:37:53.427Z" }, + { url = "https://files.pythonhosted.org/packages/a9/5b/974423c2fd8e524c7a7f64318c1e02240ef954912fa2b4d70344107b9c68/charset_normalizer-3.5.2-cp315-cp315-win_arm64.whl", hash = "sha256:6be488a102b8cf28d0391d8c4ba7748938ae28b78ad901f8585520fca33ead1a", size = 200309, upload-time = "2026-09-30T04:37:55.015Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f9/00ee0195db1013d8f7c416fd770fbeb560bb46eb2e36b054d05cb56f6cfa/charset_normalizer-3.5.2-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:915563965d418f986e7e145accc592eae9e1a1be3566ff98a05d7a9ec42a76e1", size = 395343, upload-time = "2026-09-30T04:37:56.743Z" }, + { url = "https://files.pythonhosted.org/packages/04/3a/c00b50e94c964cf934c7899cd47c97952fc11dad71cc5884b3c61795b09b/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:65cd72beeeca9d3aaea1201e5923859f308f952f9c71de93f06063c79f0f7a3b", size = 250114, upload-time = "2026-09-30T04:37:58.607Z" }, + { url = "https://files.pythonhosted.org/packages/50/27/d102dc880bbcffd0479ab64dfc1fb96777a854355a55e2bda72a71efadcb/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b7fd005a73d9e657273b7a10dc71a9e03c8fb9ee6999798d6918ce095b81ac7f", size = 239997, upload-time = "2026-09-30T04:38:00.511Z" }, + { url = "https://files.pythonhosted.org/packages/a5/4a/bf7ef45794dd293fab5f98a9309817977fbb845b9998f171b8cc5d8437a3/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e54da4baf05720032d527874d40b65fa4d7e5c6c6a43d0c3adbeffcaf275a2b3", size = 272313, upload-time = "2026-09-30T04:38:02.509Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ee/008a2837737991474c5754bb3191010007663860979701990982a502cbaf/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:124fbf1a8ff966d87ae05bb8bd45a71f966055ed8bba320d0c7cf450bc5f4d0e", size = 269986, upload-time = "2026-09-30T04:38:04.435Z" }, + { url = "https://files.pythonhosted.org/packages/93/ad/bd74a283940dc910c5b14f8e4f80a248082bc9c0fcbe1f54530cb6d9cc5e/charset_normalizer-3.5.2-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:28b4f0d66fb834ff90f28209ac7bce77868c45d8c93e26f906709d9b7c2e1af9", size = 260202, upload-time = "2026-09-30T04:38:06.549Z" }, + { url = "https://files.pythonhosted.org/packages/8a/7b/ed341c66f69f688723501fac752be3d63c7159ca0d0d4174fc611e5710bb/charset_normalizer-3.5.2-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:58ca3755ee7ff7f59b57789ec9833c9de9ea275405cdd240eda1f193112e398a", size = 259083, upload-time = "2026-09-30T04:38:08.311Z" }, + { url = "https://files.pythonhosted.org/packages/cc/9d/e41588b777965e5031a43128a1e96173ebb35ac75fc53ec3b517e7c21cd4/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:443eae2bf318abeaf6f15d785138f71fd6de770e99a92158b8b814265e079115", size = 253081, upload-time = "2026-09-30T04:38:10.402Z" }, + { url = "https://files.pythonhosted.org/packages/81/35/b761eb6d8c1eb218b9b42b9b4d5ac902afdc399fb6dac6f9a9aac7bda589/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:58f361dcbab699cf8f42db3f47c8e7fd1036f138c23a5d08de9fde5f425a730c", size = 248712, upload-time = "2026-09-30T04:38:12.317Z" }, + { url = "https://files.pythonhosted.org/packages/4d/2c/147169a041b747759f37405c0a97157e8e92de967968373101ff14915cba/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:1b4cbc7c3491ccb4aa17fcd8165649d01cf39f76de1696da8631b5f71b85401d", size = 274176, upload-time = "2026-09-30T04:38:14.138Z" }, + { url = "https://files.pythonhosted.org/packages/f0/2d/0ff8db0d373ba8538db686db11cd7e8912031490b9e4f383b41912e8d594/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:ba0b1d2620edf869789c3879223f52bf2afc5d31b3cb47cc57b3a12c05e2aa9d", size = 259413, upload-time = "2026-09-30T04:38:15.841Z" }, + { url = "https://files.pythonhosted.org/packages/8a/8e/b4a085fb47c9d3a7e43576a4784fdd8fe23f907514a972de8086edaf7a48/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:5e2b6b57e9733d39f0c9fd3185efa6b8e29652c4cd8fe94180272cf6ed9a78c4", size = 269434, upload-time = "2026-09-30T04:38:17.626Z" }, + { url = "https://files.pythonhosted.org/packages/83/1c/d8d8d7322a7c3eecdf3237a4a419cf41d2eaad8e006ce7dfdd9d4c8fa2eb/charset_normalizer-3.5.2-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:51cf45226a9b588d0d2b4880c62d686934b63ab0bd79ca23ab0e9762eb27441b", size = 262716, upload-time = "2026-09-30T04:38:19.214Z" }, + { url = "https://files.pythonhosted.org/packages/a0/16/0e4c6ba9b44e97a2da150e52d331e8f9c968b21b358fbffa6c856cebcd89/charset_normalizer-3.5.2-cp315-cp315t-win32.whl", hash = "sha256:5fb29fb8cd1a46c27a1bf9613ad5ec2599310d46b4025d9556404a6b6a292800", size = 197159, upload-time = "2026-09-30T04:38:21.037Z" }, + { url = "https://files.pythonhosted.org/packages/be/33/e90bc2b1374f7f36ef106f56620de5a783907e19ca857efe2277e31cac3e/charset_normalizer-3.5.2-cp315-cp315t-win_amd64.whl", hash = "sha256:a192e2c40070d92c3ccf777e3a5c4ff515573cd2bb7ed0c537fdadbbec5bbf21", size = 221322, upload-time = "2026-09-30T04:38:22.886Z" }, + { url = "https://files.pythonhosted.org/packages/66/89/dfa6dcb08c200b7830ab56439e8c1890f2971d51aafbb3937894a2e7fcfc/charset_normalizer-3.5.2-cp315-cp315t-win_arm64.whl", hash = "sha256:749e97e1b32313717a565abbe321bc2190bc8b35f1a67e4cdbc7c56c8d8ffe58", size = 209192, upload-time = "2026-09-30T04:38:24.648Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ab/176fbfd5b64939c55d652366aa5b9ef1d767af207a3aa6ebeb0d226c484d/charset_normalizer-3.5.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd", size = 331815, upload-time = "2026-09-30T04:38:26.216Z" }, + { url = "https://files.pythonhosted.org/packages/7e/84/371eac6b30bdbcbf2d632a1a01809103459216fcaae61b8b8d922c1bfb8a/charset_normalizer-3.5.2-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7", size = 253276, upload-time = "2026-09-30T04:38:28.032Z" }, + { url = "https://files.pythonhosted.org/packages/43/6f/c4fbae58febff71709c51bc7e18fdfa55341dc382704740f9f0cbf03817b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f", size = 241239, upload-time = "2026-09-30T04:38:29.732Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/458c3f42164a07d0c5210798e9e704b39e540a6793b05aba67f3a35243a9/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93", size = 231121, upload-time = "2026-09-30T04:38:31.462Z" }, + { url = "https://files.pythonhosted.org/packages/09/54/ab9e89367076f6331bb6c65c4bf14a5361fa5191cb6561bf534f18504e1b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade", size = 260350, upload-time = "2026-09-30T04:38:33.239Z" }, + { url = "https://files.pythonhosted.org/packages/7c/c1/061431ecc688d9d76602502cb57cc01e691e682c18f1beb45f9673b5bbd2/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0", size = 255430, upload-time = "2026-09-30T04:38:34.865Z" }, + { url = "https://files.pythonhosted.org/packages/8d/1f/20c8949f0676f7ab811abdeb7f4d7f1cbc6e61ff20bef08b44edeb092bc8/charset_normalizer-3.5.2-cp37-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26", size = 250612, upload-time = "2026-09-30T04:38:36.649Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9e/46f2fa4c431fc98c4ae76a8cb5bdca54e0341e3cfc3fcfd8e82740250818/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011", size = 242083, upload-time = "2026-09-30T04:38:38.26Z" }, + { url = "https://files.pythonhosted.org/packages/bd/39/559be29a0c0f086e0bba6922babd38916cc5e0b58ced4de13ee01ea05508/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621", size = 232738, upload-time = "2026-09-30T04:38:39.81Z" }, + { url = "https://files.pythonhosted.org/packages/ff/6c/387b0e4f756a282831c1d9fc6aeb6c51ca4507ca202767c8de15ce9b12e2/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4", size = 260703, upload-time = "2026-09-30T04:38:41.346Z" }, + { url = "https://files.pythonhosted.org/packages/96/92/1fdf015f09ef449f50d3ac4b67c90887c9c318b727daa95cc4f866e6521d/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e", size = 247622, upload-time = "2026-09-30T04:38:42.937Z" }, + { url = "https://files.pythonhosted.org/packages/dc/3c/8e7b8a5671ad5d433669fb2a76f1a0164df2d9b1718b0206bc2a16d840cc/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_s390x.whl", hash = "sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c", size = 257500, upload-time = "2026-09-30T04:38:44.604Z" }, + { url = "https://files.pythonhosted.org/packages/b4/f0/45b579df5cabc1d5d53ea1cc35e8437d3ca768c0acccc7041517cb6fbb32/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0", size = 255100, upload-time = "2026-09-30T04:38:46.289Z" }, + { url = "https://files.pythonhosted.org/packages/31/68/fdec18a343f5fb3f310588dd478b09ac4799e0b187dbade3a8cd776f03ef/charset_normalizer-3.5.2-cp37-abi3-win32.whl", hash = "sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf", size = 174499, upload-time = "2026-09-30T04:38:47.999Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8a/b618149cc5207943a0242068d7a27897f56a62947b5a039085f2a22029f8/charset_normalizer-3.5.2-cp37-abi3-win_amd64.whl", hash = "sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036", size = 200092, upload-time = "2026-09-30T04:38:49.707Z" }, + { url = "https://files.pythonhosted.org/packages/03/cf/4c66866fa9e2b1c78e3c911516d1de497a677b7ac60f1eceda74ce777ca3/charset_normalizer-3.5.2-cp37-abi3-win_arm64.whl", hash = "sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e", size = 294363, upload-time = "2026-09-30T04:38:51.312Z" }, + { url = "https://files.pythonhosted.org/packages/fc/ad/d07d7862a62ffa6d79d68074d14823243dd235a77c45262acbf6adeb28bf/charset_normalizer-3.5.2-py3-none-any.whl", hash = "sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685", size = 68872, upload-time = "2026-09-30T04:39:21.828Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "docker" +version = "7.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "requests" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/88/7f/731ff914b0255d3d065f45fd4e626d4b8c95dbcbaada049f337a6ac16410/docker-7.2.0.tar.gz", hash = "sha256:cebb93773d334f778e023a7ee352a8d6e13ab1bd3b863a4d4a59dec897df43ac", size = 118731, upload-time = "2026-07-09T14:53:46.39Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/23/529140fe1aab80fc6992f93a706deec709140a6397439139a054e1515c45/docker-7.2.0-py3-none-any.whl", hash = "sha256:a3f45fdeb9165e2d25d9a1d02ddf3bc70fb572cf5ebbf9b58558c22caf29b71f", size = 148775, upload-time = "2026-07-09T14:53:45.224Z" }, +] + +[[package]] +name = "idna" +version = "3.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/08/8eea9d4b8302028f3abb2c0813953f7aec26d33b7a8960ed760e65ff29fa/idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44", size = 216463, upload-time = "2026-09-17T14:11:04.752Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz", hash = "sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960", size = 21209, upload-time = "2026-10-06T22:48:38.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/db/7fc19e6f2dc92a966727031389fc2e08b558f0f25eb7403c1119ad4713cd/pluggy-1.7.0.tar.gz", hash = "sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8", size = 123304, upload-time = "2026-10-15T09:50:58.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", hash = "sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", size = 27082, upload-time = "2026-10-15T09:50:56.808Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pywin32" +version = "312" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/ff/32aa7d2ed0ab12b323aaa64f9b75e6ad4f8fd09f9ccfc28c79414d46838d/pywin32-312-cp312-cp312-win32.whl", hash = "sha256:dab4f65ac9c4e48400a2a0530c46c3c579cd5905ecd11b80692373915269208b", size = 6371877, upload-time = "2026-06-04T07:49:28.836Z" }, + { url = "https://files.pythonhosted.org/packages/03/d9/77040d3b43df3f3be32ea289433d660d2727f5ba327bc73be835127d9d60/pywin32-312-cp312-cp312-win_amd64.whl", hash = "sha256:b457f6d628a47e8a7346ce22acb7e1a46a4a78b52e1d17e1af56871bd19a93bc", size = 6914841, upload-time = "2026-06-04T07:49:31.85Z" }, + { url = "https://files.pythonhosted.org/packages/e3/cc/7b1ec671775756020a0ee7f4feeaf3c568f0ab86bd3900088cf986937a92/pywin32-312-cp312-cp312-win_arm64.whl", hash = "sha256:6017c58e12f6809fbb0555b75df144c2922a9ffd18e4b9b5afa863b6c1a9d950", size = 6727901, upload-time = "2026-06-04T07:49:34.244Z" }, + { url = "https://files.pythonhosted.org/packages/2d/41/12fbfd7f36ed2146d8bc9de96c2741296bf0d490b98508496cff322e274c/pywin32-312-cp313-cp313-win32.whl", hash = "sha256:7a27df850933d16a8eabfbaeb73d52b273e2da667f80d70b01a89d1f6828d02c", size = 6370184, upload-time = "2026-06-04T07:49:36.253Z" }, + { url = "https://files.pythonhosted.org/packages/ba/db/36a78e3403099d31d9746d13fdcde5accc43c1155f375a34d15983a479a7/pywin32-312-cp313-cp313-win_amd64.whl", hash = "sha256:c53e878d15a1c44788082bfe712a905433473aa38f86375b7cf8b45e3acbaaf9", size = 6914298, upload-time = "2026-06-04T07:49:38.876Z" }, + { url = "https://files.pythonhosted.org/packages/84/37/c1697194092b76de9ed47ca124323f02c57ffc8a45c06f88a3d5acaf01eb/pywin32-312-cp313-cp313-win_arm64.whl", hash = "sha256:59aba5d5940842075343a5ddc6b11f1cdf0d1567fe745290359dfbcc7c2eb831", size = 6727640, upload-time = "2026-06-04T07:49:41.083Z" }, + { url = "https://files.pythonhosted.org/packages/fc/2b/1f3cded5822fd49c02f40544cbb5f58c7cfd6b1694869fd476cb6170ee97/pywin32-312-cp314-cp314-win32.whl", hash = "sha256:a77a90fbb6881238d2ca9c6fd797b25817f3768fe78d214a90137ff055a75f5b", size = 6468928, upload-time = "2026-06-04T07:49:43.188Z" }, + { url = "https://files.pythonhosted.org/packages/21/82/3bf86d2e2808902013132e1ce905a7da0da53790f3836c64bf44d55e24f3/pywin32-312-cp314-cp314-win_amd64.whl", hash = "sha256:a4dd3a848290ef724347b19f301045831d8e802fa4464f491b98b1e0a081432e", size = 7024157, upload-time = "2026-06-04T07:49:45.34Z" }, + { url = "https://files.pythonhosted.org/packages/a4/0e/73f6d6800b4f27655abd9e9f6aaeaefcddb2b946e4674efa2bab184a7f7b/pywin32-312-cp314-cp314-win_arm64.whl", hash = "sha256:9fce94568364e0155e6dfb781ac5d95903be8baf28670632beab1b523f300daa", size = 6839598, upload-time = "2026-06-04T07:49:47.613Z" }, + { url = "https://files.pythonhosted.org/packages/eb/61/caa39686032d2ebdd04ff0ab5cbe163126c0066d98e00c9018646e42393b/pywin32-312-cp315-cp315-win32.whl", hash = "sha256:5c1fbe4a937a73ae9297384a3da38518cbc694c68ad8a809b2e19acd350f03ed", size = 6471159, upload-time = "2026-06-04T07:49:50.035Z" }, + { url = "https://files.pythonhosted.org/packages/0f/cd/7e1de64a4a6f69c04214169657ccab0d93a670ea50e35eb8f489d7378249/pywin32-312-cp315-cp315-win_amd64.whl", hash = "sha256:c2f03a0f73f804a13c2735b99392b0cd426bb4f2c4d0178e5ac966a0f21618d5", size = 7025293, upload-time = "2026-06-04T07:49:54.857Z" }, + { url = "https://files.pythonhosted.org/packages/23/ed/4532e9388e65fa16b46776ef47ad631a64eda1631884488af707666350ed/pywin32-312-cp315-cp315-win_arm64.whl", hash = "sha256:a8597d28f267b39074aef51fa593530082b39cbe5a074226096857b1fed2dfb9", size = 6840337, upload-time = "2026-06-04T07:49:57.531Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "ruff" +version = "0.17.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e7/e6d5749983b020e6cb1055abec0a6f361a8e94f0184cdcf56d4760995f02/ruff-0.17.1.tar.gz", hash = "sha256:5bb796c5112e9fb9527f2ef2faa0a130d6312d19de92bcfaddd70dbb2d41020f", size = 5323349, upload-time = "2026-10-15T15:23:18.64Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/78/90c46ca6250709f7c37e397b60d06061adee8d0f7d13959ed6a8e49b14fd/ruff-0.17.1-py3-none-linux_armv6l.whl", hash = "sha256:1796c41eafec66f4635125dd1e074a7aa27b8d20c187d484360acf11d247fc4a", size = 10147528, upload-time = "2026-10-15T15:22:33.667Z" }, + { url = "https://files.pythonhosted.org/packages/aa/d2/a2e28bc35b083ae51fa07dfba5280a667c8b01ecaee7802fc8849dcd5c46/ruff-0.17.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:8eadb09a788010be29c895418bdfa513e4e1c65e64b94e5b19e4cd381f6cc7fb", size = 10225974, upload-time = "2026-10-15T15:22:36.328Z" }, + { url = "https://files.pythonhosted.org/packages/8e/71/7a9029a81c8206ee01a98f6cfeeb56e6c095af86857581a37045c46d7e94/ruff-0.17.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:29b454f18ca3eeaf37fbf140841dcf27d36109e29dee9aae8acecb68c26c6f74", size = 9270361, upload-time = "2026-10-15T15:22:38.651Z" }, + { url = "https://files.pythonhosted.org/packages/c0/8c/e075e9c917cb0575297880c96768304e881240171c870cbd8d9b7b54bec2/ruff-0.17.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:872e03dd231744d6c5ec5c8af6be52b71323fc6ee096e4c8b34d9f844ee3b978", size = 9938300, upload-time = "2026-10-15T15:22:41.284Z" }, + { url = "https://files.pythonhosted.org/packages/87/27/0b4bb926a49de2383f161272b4d863571a4c7ecdbf595a6bc58813a13367/ruff-0.17.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4c0d6cb5f8fbac77eec9af49f4fc0eaf856b05545e372b0694d24b4e76b1c640", size = 9948464, upload-time = "2026-10-15T15:22:43.681Z" }, + { url = "https://files.pythonhosted.org/packages/14/27/518d57e0e02aecb8ea4acac6b57b51a53173d25fa734952bfdd7d531e88e/ruff-0.17.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78beced4a3c94dfedbee4b999b95b6a27b35820fdc0afbacc6d37a8da26c1cd1", size = 10638639, upload-time = "2026-10-15T15:22:46.081Z" }, + { url = "https://files.pythonhosted.org/packages/01/82/d89a4e498ade59e6cee52d67f8da569019372212b87468c1fdeae884601c/ruff-0.17.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7feefc63921202894f0e158b0e32f45a5950d70a7fb693ccb9264816c25b1c05", size = 11337238, upload-time = "2026-10-15T15:22:48.894Z" }, + { url = "https://files.pythonhosted.org/packages/a2/7c/3b0fdef9352656317c977086fe1d802902406e4f2a0d8578da90c0e6d1c5/ruff-0.17.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:684154861a819bf1f433669d2a0ce9c3e54677c65b8ebb866cc2a7ec86345785", size = 10896327, upload-time = "2026-10-15T15:22:51.45Z" }, + { url = "https://files.pythonhosted.org/packages/62/86/5566290219708ce2928f3a29c4f843a1185b341ec6111c4631a33c5ceced/ruff-0.17.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6ef98e51de8ef38082f90653ceeb84944bd7673783eea13e1647c5b9b064189", size = 10350811, upload-time = "2026-10-15T15:22:54.401Z" }, + { url = "https://files.pythonhosted.org/packages/ad/d2/ca8462dabb3c413990d49703ba306c30c862195a75448d080fa8fcce9775/ruff-0.17.1-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:8f9aa5b5bd5fa1937803ea609c8f153b8de39bf3e0e149897ae34b0c5b52f467", size = 10569994, upload-time = "2026-10-15T15:22:57.142Z" }, + { url = "https://files.pythonhosted.org/packages/e3/ad/dd1864c1f1db17c8cfb297ce70b352388e6977620f66f7182c52855407df/ruff-0.17.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:80e53b0e5faa3c5667f035daaa032893ae42b7cfe708c886d39ed131e54688ab", size = 10059672, upload-time = "2026-10-15T15:23:00.084Z" }, + { url = "https://files.pythonhosted.org/packages/65/45/3f56cf1d2168c2165cdd5ac6f9668aee66a4cfdd4caf2b2cc5ccb2b1d568/ruff-0.17.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:056c5d5fa0ac790051ae165d7035c7bf956d02de2db557f2e9b31d877a505fca", size = 9947803, upload-time = "2026-10-15T15:23:03.132Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4b/5457cb2be4c1d1ea5e919558131a0f197a4a454a0530d9e30820e8206e4c/ruff-0.17.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:bd8f5a55b10f371642eacd0e8665a42a8b97da15c9a7187c79e697b77b5e0ce4", size = 10351807, upload-time = "2026-10-15T15:23:05.628Z" }, + { url = "https://files.pythonhosted.org/packages/49/fb/db15b4f8fa24a1e36b99e1401f42ef6df16ee299a48670d5d262b771e7d7/ruff-0.17.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:ce05e6f3973611d9550a600e04bb531061e368baba31610eb57c1509a8f21a0a", size = 10765716, upload-time = "2026-10-15T15:23:08.092Z" }, + { url = "https://files.pythonhosted.org/packages/0a/e8/1e186578076aff7a80735f625005fde678412ab609cd323ed182eb7d6b87/ruff-0.17.1-py3-none-win32.whl", hash = "sha256:1de0030a8aa78c65c5fecb84b2e5c390ac684c0452875a02afc7c083d49a496a", size = 9895093, upload-time = "2026-10-15T15:23:10.582Z" }, + { url = "https://files.pythonhosted.org/packages/6a/2e/da3d9589bae8e8b58f3a48bd5bc0b980232fb14a83bf02c3be7660f87c1b/ruff-0.17.1-py3-none-win_amd64.whl", hash = "sha256:5df76f16580fab58d0f6c98a3ac254a4abceb7b2489b8b678335869cff77b3d7", size = 10097955, upload-time = "2026-10-15T15:23:13.309Z" }, + { url = "https://files.pythonhosted.org/packages/df/c9/0f2a4dea06a5e12c1191ee3d96b1e1233797d0d5e18f97a2b909a95a9836/ruff-0.17.1-py3-none-win_arm64.whl", hash = "sha256:42400b9a6ff8515ff27796f07f92e0195be011c86e81e381606f58ea5502fef4", size = 9853542, upload-time = "2026-10-15T15:23:16.05Z" }, +] + +[[package]] +name = "urllib3" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, +] + +[[package]] +name = "valinor-integration" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "docker" }, + { name = "pyyaml" }, + { name = "requests" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "docker", specifier = ">=7.2.0" }, + { name = "pyyaml", specifier = ">=6.0.3" }, + { name = "requests", specifier = ">=2.34.2" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=9.1.1" }, + { name = "ruff", specifier = ">=0.17.1" }, +] diff --git a/valhalla-graphtile/Cargo.toml b/valhalla-graphtile/Cargo.toml index 62370c8..88ba8a8 100644 --- a/valhalla-graphtile/Cargo.toml +++ b/valhalla-graphtile/Cargo.toml @@ -37,6 +37,7 @@ zerocopy-derive = { workspace = true } insta = { workspace = true } proptest = { workspace = true } rand = { workspace = true } +tempfile = { workspace = true } walkdir = { workspace = true } [lints] diff --git a/valhalla-graphtile/src/extract/mod.rs b/valhalla-graphtile/src/extract/mod.rs new file mode 100644 index 0000000..f9de6c5 --- /dev/null +++ b/valhalla-graphtile/src/extract/mod.rs @@ -0,0 +1,6 @@ +//! Offline tile extracts: tarballs with a subset of the tiles of a +//! [`GraphTileProvider`](crate::tile_provider::GraphTileProvider). + +mod writer; + +pub use writer::{ExtractWriteError, write_extract}; diff --git a/valhalla-graphtile/src/extract/writer.rs b/valhalla-graphtile/src/extract/writer.rs new file mode 100644 index 0000000..52a1238 --- /dev/null +++ b/valhalla-graphtile/src/extract/writer.rs @@ -0,0 +1,837 @@ +//! A streaming tar writer for tile extracts. +//! +//! The writer computes every offset from the tile sizes before it writes, +//! and writes the archive in one pass. The archive contains: +//! +//! 1. `index.bin`, which Valhalla requires as the first entry of a remote tarball. +//! 2. The tiles at the paths of [`GraphId::file_path`] with `/` separators, +//! in ascending (level, tile index) order. +//! 3. The two-block end-of-archive marker. + +use crate::GraphId; +use crate::graph_id::InvalidGraphIdError; +use crate::tile_provider::{ + GraphTileProvider, GraphTileProviderError, INDEX_BIN_FILE_NAME, INDEX_ENTRY_SIZE, + InvalidIndexBinError, TAR_BLOCK_SIZE, TILE_FILE_EXTENSION, TileIndexBinEntry, + serialize_index_bin, +}; +use std::io::Write; +use std::path::Path; +use tar::{EntryType, Header}; +use thiserror::Error; + +/// Zeros for the payload padding and the end-of-archive marker. +#[expect(clippy::cast_possible_truncation, reason = "512 fits in every usize")] +const ZERO_BLOCK: [u8; TAR_BLOCK_SIZE as usize] = [0; TAR_BLOCK_SIZE as usize]; + +/// The length of the end-of-archive marker: two zero blocks. +const END_OF_ARCHIVE_MARKER_LEN: u64 = 2 * TAR_BLOCK_SIZE; + +/// An error while writing a tile extract. +#[derive(Debug, Error)] +pub enum ExtractWriteError { + /// The tile list is empty. + #[error("the extract contains no tiles")] + NoTiles, + /// A requested tile is not in the provider. + #[error("tile {0} is not in the provider")] + MissingTile(GraphId), + /// A requested graph ID has no tar entry path. + #[error("invalid graph ID: {0}")] + InvalidGraphId(#[from] InvalidGraphIdError), + /// A tile has no valid `index.bin` entry. + #[error("invalid index.bin: {0}")] + InvalidIndexBin(#[from] InvalidIndexBinError), + /// The length of a tile differs from the size that the provider reported, + /// for example because a tile file changed during the write. + #[error( + "the provider gave {actual} bytes for tile {graph_id}, but it reported a size of {expected} bytes" + )] + SizeMismatch { + /// The base graph ID of the tile. + graph_id: GraphId, + /// The size that the provider reported before the write. + expected: u32, + /// The length of the tile bytes from the provider. + actual: usize, + }, + /// The provider failed to give the size of a tile, or to read or decode a tile. + #[error("the provider failed to give tile {graph_id}: {source}")] + Provider { + /// The base graph ID of the tile. + graph_id: GraphId, + /// The error from the provider. + source: GraphTileProviderError, + }, + /// Writing to the destination failed. + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), +} + +impl ExtractWriteError { + /// Wraps an error from the provider for the tile `graph_id`. + fn from_provider(graph_id: GraphId, source: GraphTileProviderError) -> Self { + match source { + GraphTileProviderError::TileDoesNotExist => Self::MissingTile(graph_id), + source => Self::Provider { graph_id, source }, + } + } +} + +/// Writes a deterministic extract with the tiles that contain `tile_ids` from `provider` +/// into `dest`, and returns the number of bytes written. +/// +/// For a [`DirectoryGraphTileProvider`](crate::tile_provider::DirectoryGraphTileProvider), +/// the tile files must not change during the write, +/// and the cache of the provider must not hold older versions of the tiles. +/// +/// # Performance +/// +/// The writer makes three small writes for each tile, and it does not buffer them. +/// Wrap a `dest` without a buffer, such as a [`File`](std::fs::File), in a +/// [`BufWriter`](std::io::BufWriter). +/// +/// # Errors +/// +/// The writer finds these errors before it writes to `dest`: +/// +/// - [`ExtractWriteError::NoTiles`] if `tile_ids` is empty. +/// - [`ExtractWriteError::MissingTile`] if the provider does not have a tile. +/// - [`ExtractWriteError::Provider`] if the provider cannot get the size of a tile. +/// +/// The writer finds these errors during the write, so `dest` then contains a partial archive: +/// +/// - [`ExtractWriteError::MissingTile`] or [`ExtractWriteError::Provider`] +/// if the provider cannot read or decode a tile. +/// - [`ExtractWriteError::SizeMismatch`] if the length of a tile differs from its size. +/// - [`ExtractWriteError::Io`] if a write to `dest` fails. +pub fn write_extract( + provider: &P, + tile_ids: I, + mut dest: W, +) -> Result +where + P: GraphTileProvider, + I: IntoIterator, + W: Write, +{ + let mut tiles: Vec = tile_ids + .into_iter() + .map(|graph_id| graph_id.tile_base_id()) + .collect(); + // `GraphId` orders higher levels first, so it cannot give the archive order. + tiles.sort_unstable_by_key(|graph_id| (graph_id.level(), graph_id.tile_id())); + tiles.dedup(); + if tiles.is_empty() { + return Err(ExtractWriteError::NoTiles); + } + + let manifest = tiles + .into_iter() + .map(|graph_id| { + provider + .tile_size(graph_id) + .map(|size| (graph_id, size)) + .map_err(|source| ExtractWriteError::from_provider(graph_id, source)) + }) + .collect::, _>>()?; + let layout = ExtractLayout::from_manifest(&manifest)?; + + let index_bytes = serialize_index_bin(&layout.entries)?; + write_entry(&mut dest, INDEX_BIN_FILE_NAME, index_bytes)?; + + for entry in &layout.entries { + let graph_id = entry.graph_id()?; + let path = graph_id.file_path(TILE_FILE_EXTENSION)?; + provider + .with_tile_containing(graph_id, |tile| { + let tile_bytes = tile.tile_bytes(); + let expected = entry.size(); + if tile_bytes.len() != expected as usize { + return Err(ExtractWriteError::SizeMismatch { + graph_id, + expected, + actual: tile_bytes.len(), + }); + } + write_entry(&mut dest, &path, tile_bytes)?; + Ok(()) + }) + .map_err(|source| ExtractWriteError::from_provider(graph_id, source))??; + } + + dest.write_all(&ZERO_BLOCK)?; + dest.write_all(&ZERO_BLOCK)?; + dest.flush()?; + + Ok(layout.total_len) +} + +/// Where each tile goes in an extract. +/// +/// The writer must write `index.bin` before the tiles, +/// so it computes the offset of every tile from the tile sizes before it writes. +#[derive(Debug)] +struct ExtractLayout { + /// The `index.bin` entry of each tile, in archive order. + entries: Vec, + /// The length of the archive in bytes, with the end-of-archive marker. + total_len: u64, +} + +impl ExtractLayout { + /// Computes the layout for `(GraphId, size)` pairs of base tile IDs in archive order. + /// + /// # Errors + /// + /// Returns an error if `manifest` contains an ID that is not a base tile ID. + fn from_manifest(manifest: &[(GraphId, u32)]) -> Result { + let index_len = (INDEX_ENTRY_SIZE * manifest.len()) as u64; + let mut next_entry_offset = entry_len(index_len); + + let mut entries = Vec::with_capacity(manifest.len()); + for &(graph_id, size) in manifest { + let payload_offset = next_entry_offset + TAR_BLOCK_SIZE; + entries.push(TileIndexBinEntry::new(graph_id, payload_offset, size)?); + next_entry_offset += entry_len(u64::from(size)); + } + + Ok(Self { + entries, + total_len: next_entry_offset + END_OF_ARCHIVE_MARKER_LEN, + }) + } +} + +/// The length of a tar entry with a payload of `payload_len` bytes: +/// a header block, then the payload padded to a whole number of blocks. +const fn entry_len(payload_len: u64) -> u64 { + TAR_BLOCK_SIZE + payload_len.next_multiple_of(TAR_BLOCK_SIZE) +} + +/// Writes a tar entry for `payload` at `path`. +fn write_entry( + dest: &mut W, + path: impl AsRef, + payload: &[u8], +) -> std::io::Result<()> { + // Keep all metadata fixed, so that the same tiles always give the same bytes. + let mut header = Header::new_ustar(); + header.set_entry_type(EntryType::Regular); + header.set_path(path)?; + header.set_size(payload.len() as u64); + header.set_mode(0o644); + header.set_uid(0); + header.set_gid(0); + header.set_mtime(0); + header.set_cksum(); + + dest.write_all(header.as_bytes())?; + dest.write_all(payload)?; + + let padding = payload.len().next_multiple_of(ZERO_BLOCK.len()) - payload.len(); + dest.write_all(&ZERO_BLOCK[..padding]) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::graph_tile::GraphTileView; + use crate::graph_tile::handles::OwnedGraphTileHandle; + use crate::test_fixtures; + use crate::tile_provider::{DirectoryGraphTileProvider, parse_index_bin}; + use geo::{CoordFloat, Point}; + use num_traits::FromPrimitive; + use std::collections::{HashMap, HashSet}; + use std::io::Read; + use std::num::NonZeroUsize; + use std::path::Path; + use std::sync::Arc; + use tar::Archive; + + fn tile((level, tile_id): (u8, u64)) -> GraphId { + GraphId::try_from_components(level, tile_id, 0).expect("valid tile ID in test") + } + + fn directory_provider(base: &Path) -> DirectoryGraphTileProvider { + DirectoryGraphTileProvider::new(base.to_path_buf(), NonZeroUsize::MIN) + } + + /// The Andorra tile directory, with sizes and read errors that a test can change. + struct TestProvider { + inner: DirectoryGraphTileProvider, + /// Sizes which `tile_size` reports in place of the real sizes. + reported_sizes: HashMap, + /// Tiles whose size the provider fails to get. + size_failures: HashSet, + /// Tiles which the provider has, but fails to read. + read_failures: HashSet, + } + + impl TestProvider { + fn new() -> Self { + Self { + inner: directory_provider(&test_fixtures::andorra_tiles_dir()), + reported_sizes: HashMap::new(), + size_failures: HashSet::new(), + read_failures: HashSet::new(), + } + } + + fn check_read(&self, graph_id: GraphId) -> Result<(), GraphTileProviderError> { + if self.read_failures.contains(&graph_id.tile_base_id()) { + return Err(test_failure()); + } + Ok(()) + } + } + + fn test_failure() -> GraphTileProviderError { + GraphTileProviderError::TileFetchError("a failure from the test provider".to_string()) + } + + impl GraphTileProvider for TestProvider { + type TileHandle = Arc; + + fn with_tile_containing( + &self, + graph_id: GraphId, + process: F, + ) -> Result + where + F: FnOnce(&GraphTileView) -> T, + { + self.check_read(graph_id)?; + self.inner.with_tile_containing(graph_id, process) + } + + fn get_handle_for_tile_containing( + &self, + graph_id: GraphId, + ) -> Result { + self.check_read(graph_id)?; + self.inner.get_handle_for_tile_containing(graph_id) + } + + fn tile_size(&self, graph_id: GraphId) -> Result { + let base_graph_id = graph_id.tile_base_id(); + if self.size_failures.contains(&base_graph_id) { + return Err(test_failure()); + } + if let Some(size) = self.reported_sizes.get(&base_graph_id) { + return Ok(*size); + } + self.inner.tile_size(graph_id) + } + + fn tiles_within_radius( + &self, + center: Point, + radius: N, + ) -> Vec { + self.inner.tiles_within_radius(center, radius) + } + } + + /// Returns the raw header path and the payload offset of each entry. + fn read_entries(archive_bytes: &[u8]) -> Vec<(String, u64)> { + Archive::new(archive_bytes) + .entries() + .expect("the archive is readable") + .map(|entry| { + let entry = entry.expect("every entry is readable"); + let path = String::from_utf8_lossy(&entry.path_bytes()).into_owned(); + (path, entry.raw_file_position()) + }) + .collect() + } + + fn read_index_bytes(archive_bytes: &[u8]) -> Vec { + let mut index_bytes = Vec::new(); + Archive::new(archive_bytes) + .entries() + .expect("the archive is readable") + .next() + .expect("the archive has entries") + .expect("the index entry is readable") + .read_to_end(&mut index_bytes) + .expect("the index payload is readable"); + index_bytes + } + + /// Checks an archive with the `tar` crate as an independent reader, + /// and checks each tile against the tile file in `tile_dir`. + fn check_archive(archive_bytes: &[u8], expected_tiles: &[GraphId], tile_dir: &Path) { + assert!( + archive_bytes.len().is_multiple_of(ZERO_BLOCK.len()), + "The archive must be a whole number of tar blocks" + ); + let marker_start = archive_bytes.len() - ZERO_BLOCK.len() * 2; + assert!( + archive_bytes[marker_start..].iter().all(|byte| *byte == 0), + "The archive must end with two zero blocks" + ); + + let entries = read_entries(archive_bytes); + let mut expected_paths = vec![INDEX_BIN_FILE_NAME.to_string()]; + // Tar entry names use `/` on every platform. + expected_paths.extend(expected_tiles.iter().map(|graph_id| { + let file_path = graph_id.file_path(TILE_FILE_EXTENSION).expect("valid tile"); + let components: Vec<&str> = file_path + .iter() + .map(|component| component.to_str().expect("tile paths are ASCII")) + .collect(); + components.join("/") + })); + let entry_paths: Vec<&String> = entries.iter().map(|(path, _)| path).collect(); + assert_eq!( + entry_paths, + expected_paths.iter().collect::>(), + "The archive must contain index.bin, then the tiles in archive order" + ); + + let index_bytes = read_index_bytes(archive_bytes); + let index_entries = parse_index_bin(&index_bytes).expect("the index parses"); + assert_eq!( + index_entries.len(), + expected_tiles.len(), + "The index must have one entry for each tile" + ); + for (index_entry, (path, payload_offset)) in index_entries.iter().zip(&entries[1..]) { + assert_eq!( + index_entry.offset(), + *payload_offset, + "The index offset of {path} must be where the tar reader finds its payload" + ); + + let start = usize::try_from(index_entry.offset()).expect("the offset fits in memory"); + let payload = &archive_bytes[start..start + index_entry.size() as usize]; + let tile_file = std::fs::read(tile_dir.join(path)).expect("the tile file is readable"); + assert!( + payload == tile_file.as_slice(), + "The payload of {path} must be the bytes of its tile file" + ); + } + } + + #[test] + fn test_layout_offsets() { + const B: u64 = TAR_BLOCK_SIZE; + + // Sizes at, below, and above a block boundary, in archive order. + let manifest = [ + (tile((0, 3015)), 512), + (tile((1, 47_700)), 511), + (tile((1, 47_701)), 513), + (tile((2, 762_484)), 1024), + (tile((2, 762_485)), 100), + ]; + let layout = ExtractLayout::from_manifest(&manifest).expect("the manifest is well-formed"); + + // The index entry (a header and 5 × 16 bytes) fills blocks 0 and 1. + // Each payload starts one block after its header. + let expected_offsets = [ + 3 * B, // header at block 2, payload is 1 block + 5 * B, // header at block 4, payload is padded to 1 block + 7 * B, // header at block 6, payload is padded to 2 blocks + 10 * B, // header at block 9, payload is 2 blocks + 13 * B, // header at block 12, payload is padded to 1 block + ]; + + assert_eq!( + layout.entries.len(), + manifest.len(), + "The layout must have one entry for each tile" + ); + for ((entry, (graph_id, size)), offset) in + layout.entries.iter().zip(manifest).zip(expected_offsets) + { + assert_eq!( + entry.graph_id(), + Ok(graph_id), + "The layout must keep the order of the manifest" + ); + assert_eq!( + entry.offset(), + offset, + "The payload of {graph_id} must start one block after its header" + ); + assert_eq!( + entry.size(), + size, + "The index entry of {graph_id} must have its size" + ); + } + + assert_eq!( + layout.total_len, + 16 * B, + "The last payload ends at block 14, then the end-of-archive marker is 2 blocks" + ); + } + + #[test] + fn test_layout_rejects_non_base_tile_ids() { + let with_feature_index = + GraphId::try_from_components(1, 47_701, 7).expect("valid graph ID in test"); + assert_eq!( + ExtractLayout::from_manifest(&[(with_feature_index, 100)]).unwrap_err(), + InvalidIndexBinError::FeatureIndex(with_feature_index), + "Only base tile IDs can be indexed" + ); + } + + #[test] + fn test_write_rejects_empty_tile_list() { + let mut dest = Vec::new(); + assert!( + matches!( + write_extract(&TestProvider::new(), [], &mut dest), + Err(ExtractWriteError::NoTiles) + ), + "An extract must contain at least one tile" + ); + assert!( + dest.is_empty(), + "An empty tile list is found before any write" + ); + } + + #[test] + fn test_write_rejects_missing_tiles() { + let missing = tile((2, 0)); + let mut dest = Vec::new(); + assert!( + matches!( + write_extract(&TestProvider::new(), [test_fixtures::smallest_andorra_tile(), missing], &mut dest), + Err(ExtractWriteError::MissingTile(graph_id)) if graph_id == missing + ), + "A tile that is not in the provider must be a MissingTile error" + ); + assert!(dest.is_empty(), "A missing tile is found before any write"); + } + + #[test] + fn test_write_reports_size_errors_before_write() { + let mut provider = TestProvider::new(); + provider + .size_failures + .insert(test_fixtures::smallest_andorra_tile()); + + let mut dest = Vec::new(); + assert!( + matches!( + write_extract(&provider, [test_fixtures::smallest_andorra_tile()], &mut dest), + Err(ExtractWriteError::Provider { + graph_id, + source: GraphTileProviderError::TileFetchError(_), + }) if graph_id == test_fixtures::smallest_andorra_tile() + ), + "A failure to get a tile size must be a Provider error" + ); + assert!( + dest.is_empty(), + "A tile size error is found before any write" + ); + } + + #[test] + fn test_write_reports_read_errors() { + let mut provider = TestProvider::new(); + provider + .read_failures + .insert(test_fixtures::smallest_andorra_tile()); + + assert!( + matches!( + write_extract(&provider, [test_fixtures::smallest_andorra_tile()], Vec::new()), + Err(ExtractWriteError::Provider { + graph_id, + source: GraphTileProviderError::TileFetchError(_), + }) if graph_id == test_fixtures::smallest_andorra_tile() + ), + "A failure to read a tile must be a Provider error" + ); + } + + #[test] + fn test_write_rejects_size_mismatch() { + let mut provider = TestProvider::new(); + let real_size = provider + .tile_size(test_fixtures::smallest_andorra_tile()) + .expect("the tile has a size"); + provider + .reported_sizes + .insert(test_fixtures::smallest_andorra_tile(), real_size + 1); + + assert!( + matches!( + write_extract(&provider, [test_fixtures::smallest_andorra_tile()], Vec::new()), + Err(ExtractWriteError::SizeMismatch { + graph_id, + expected, + actual, + }) if graph_id == test_fixtures::smallest_andorra_tile() + && expected == real_size + 1 + && actual == real_size as usize + ), + "Tile bytes of a different length than the reported size must be an error" + ); + } + + /// Copies the small Andorra tile into a new tile directory. + fn copy_small_tile() -> (tempfile::TempDir, std::path::PathBuf) { + let base = tempfile::tempdir().expect("Unable to create a temporary directory"); + let relative_path = test_fixtures::smallest_andorra_tile() + .file_path(TILE_FILE_EXTENSION) + .expect("valid tile"); + let tile_path = base.path().join(&relative_path); + std::fs::create_dir_all(tile_path.parent().expect("the tile path has a parent")) + .expect("Unable to create the tile directory"); + // Miri does not support the `copy_file_range` call that `fs::copy` uses on Linux. + let bytes = std::fs::read(test_fixtures::andorra_tiles_dir().join(&relative_path)) + .expect("Unable to read the tile"); + std::fs::write(&tile_path, bytes).expect("Unable to write the tile"); + (base, tile_path) + } + + #[test] + fn test_write_rejects_tile_changed_after_cache() { + let (base, tile_path) = copy_small_tile(); + let provider = directory_provider(base.path()); + let cached_len = provider + .with_tile_containing(test_fixtures::smallest_andorra_tile(), |tile| { + tile.tile_bytes().len() + }) + .expect("Unable to get tile"); + + // The file grows, but the cache of the provider keeps the old bytes. + let mut file = std::fs::OpenOptions::new() + .append(true) + .open(&tile_path) + .expect("Unable to open the tile file"); + file.write_all(&[0]).expect("Unable to grow the tile file"); + + assert!( + matches!( + write_extract(&provider, [test_fixtures::smallest_andorra_tile()], Vec::new()), + Err(ExtractWriteError::SizeMismatch { + expected, + actual, + .. + }) if expected as usize == cached_len + 1 && actual == cached_len + ), + "A tile file that changed after the provider cached it must be a SizeMismatch" + ); + } + + #[test] + fn test_write_rejects_empty_tile_file() { + let (base, tile_path) = copy_small_tile(); + std::fs::File::create(&tile_path).expect("Unable to empty the tile file"); + + assert!( + matches!( + write_extract(&directory_provider(base.path()), [test_fixtures::smallest_andorra_tile()], Vec::new()), + Err(ExtractWriteError::Provider { + graph_id, + source: GraphTileProviderError::DecodingError(_), + }) if graph_id == test_fixtures::smallest_andorra_tile() + ), + "An empty tile file must fail to decode" + ); + } + + #[test] + fn test_write_small_tile() { + let provider = TestProvider::new(); + let mut archive_bytes = Vec::new(); + let written = write_extract( + &provider, + [test_fixtures::smallest_andorra_tile()], + &mut archive_bytes, + ) + .expect("the extract writes cleanly"); + + assert_eq!( + written, + archive_bytes.len() as u64, + "The return value must be the number of bytes written" + ); + check_archive( + &archive_bytes, + &[test_fixtures::smallest_andorra_tile()], + &test_fixtures::andorra_tiles_dir(), + ); + } + + // Miri is too slow to decode all of the Andorra tiles, and it does not support memory maps. + #[cfg(not(miri))] + mod all_andorra_tiles { + use super::*; + use crate::tile_provider::TarballTileProvider; + use proptest::collection::vec; + use proptest::prelude::ProptestConfig; + use proptest::sample::select; + use proptest::{prop_assert, prop_assert_eq, proptest}; + + /// The tiles of the Andorra fixtures, in archive order. + const ANDORRA_TILES: [(u8, u64); 7] = [ + (0, 3015), + (1, 47_701), + (2, 762_485), + (2, 762_486), + (2, 763_925), + (2, 763_926), + (2, 763_927), + ]; + + fn andorra_tiles() -> Vec { + ANDORRA_TILES.into_iter().map(tile).collect() + } + + #[test] + fn test_write_all_andorra_tiles() { + let provider = TestProvider::new(); + let mut archive_bytes = Vec::new(); + let written = write_extract(&provider, andorra_tiles(), &mut archive_bytes) + .expect("the extract writes cleanly"); + + assert_eq!( + written, + archive_bytes.len() as u64, + "The return value must be the number of bytes written" + ); + check_archive( + &archive_bytes, + &andorra_tiles(), + &test_fixtures::andorra_tiles_dir(), + ); + } + + #[test] + fn test_order_and_duplicates_do_not_change_output() { + let provider = TestProvider::new(); + let mut in_order = Vec::new(); + write_extract(&provider, andorra_tiles(), &mut in_order).expect("first write succeeds"); + + // Reverse order, with every tile twice: once as a base ID and once with a feature index. + let scrambled = andorra_tiles().into_iter().rev().flat_map(|graph_id| { + let with_feature_index = graph_id + .with_feature_index(42) + .expect("valid feature index"); + [graph_id, with_feature_index] + }); + let mut from_scrambled = Vec::new(); + write_extract(&provider, scrambled, &mut from_scrambled) + .expect("second write succeeds"); + + assert!( + in_order == from_scrambled, + "The order of the tile IDs, duplicates, and feature indexes must not change the output" + ); + } + + #[test] + fn test_andorra_round_trip() { + let source_provider = + TarballTileProvider::new_readonly(test_fixtures::andorra_tiles_tar()) + .expect("Unable to init the source tile provider"); + + // Out of archive order, and with one nonzero feature index. + let tile_ids = [ + tile((2, 763_926)), + tile((0, 3015)), + GraphId::try_from_components(1, 47_701, 42).expect("valid graph ID in test"), + tile((2, 762_485)), + ]; + let excluded = tile((2, 763_925)); + assert!( + source_provider.tile_size(excluded).is_ok(), + "The excluded tile must be in the source" + ); + + let mut archive_bytes = Vec::new(); + write_extract(&source_provider, tile_ids, &mut archive_bytes) + .expect("the extract writes cleanly"); + + let mut archive_file = + tempfile::NamedTempFile::new().expect("a temporary file is created"); + archive_file + .write_all(&archive_bytes) + .expect("the extract is written to disk"); + let extract_provider = TarballTileProvider::new_readonly(archive_file.path()) + .expect("the generated extract opens cleanly"); + + for graph_id in tile_ids { + let extract_tile = extract_provider + .with_tile_containing(graph_id, |tile| tile.tile_bytes().to_vec()) + .expect("the tile is in the extract"); + let source_tile = source_provider + .with_tile_containing(graph_id, |tile| tile.tile_bytes().to_vec()) + .expect("the tile is in the source"); + assert!( + extract_tile == source_tile, + "Tile {graph_id} must have the same bytes in the extract and in the source" + ); + } + assert!( + matches!( + extract_provider.tile_size(excluded), + Err(GraphTileProviderError::TileDoesNotExist) + ), + "A tile that was not requested must not be in the extract" + ); + } + + #[test] + fn test_directory_and_tarball_providers_write_the_same_extract() { + // The two fixtures contain the same tiles. + let tarball = TarballTileProvider::new_readonly(test_fixtures::andorra_tiles_tar()) + .expect("Unable to init the tarball tile provider"); + let directory = directory_provider(&test_fixtures::andorra_tiles_dir()); + + let mut from_tarball = Vec::new(); + write_extract(&tarball, andorra_tiles(), &mut from_tarball) + .expect("the extract writes cleanly from the tarball"); + let mut from_directory = Vec::new(); + write_extract(&directory, andorra_tiles(), &mut from_directory) + .expect("the extract writes cleanly from the directory"); + assert!( + from_tarball == from_directory, + "The tarball and the directory of the same tiles must give the same extract" + ); + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(32))] + + #[test] + fn test_archive_matches_independent_reader( + // Andorra tiles in any order, with duplicates and feature indexes. + tile_ids in vec((select(&ANDORRA_TILES[..]), 0u64..4), 1..12) + ) { + let tile_ids: Vec = tile_ids + .into_iter() + .map(|(tile_key, feature_index)| { + tile(tile_key) + .with_feature_index(feature_index) + .expect("valid feature index") + }) + .collect(); + let mut expected_tiles: Vec = + tile_ids.iter().map(GraphId::tile_base_id).collect(); + expected_tiles.sort_unstable_by_key(|graph_id| (graph_id.level(), graph_id.tile_id())); + expected_tiles.dedup(); + + let mut archive_bytes = Vec::new(); + let written = write_extract(&TestProvider::new(), tile_ids, &mut archive_bytes) + .expect("the extract writes cleanly"); + + prop_assert_eq!(written, archive_bytes.len() as u64); + prop_assert!(!expected_tiles.is_empty()); + check_archive(&archive_bytes, &expected_tiles, &test_fixtures::andorra_tiles_dir()); + } + } + } +} diff --git a/valhalla-graphtile/src/graph_id.rs b/valhalla-graphtile/src/graph_id.rs index fa765b9..6ef358f 100644 --- a/valhalla-graphtile/src/graph_id.rs +++ b/valhalla-graphtile/src/graph_id.rs @@ -221,7 +221,9 @@ impl GraphId { let level = if level_number == TRANSIT_LEVEL.level { &*TRANSIT_LEVEL } else { - &STANDARD_LEVELS[level_number as usize] + STANDARD_LEVELS + .get(level_number as usize) + .ok_or(InvalidGraphIdError::Level)? }; let max_id = level.tiling_system.n_cols * level.tiling_system.n_rows - 1; @@ -423,4 +425,35 @@ mod tests { Ok("3/001/000/000.gph".into()) ); } + + #[test] + fn test_file_path_invalid() { + // Each level has a 4°, 1°, or 0.25° grid of 4050, 64800, or 1036800 tiles. + // Level 2 has one more directory, because its tile IDs have more digits. + for (level, max_tile_id, path) in [ + (0, 4049, "0/004/049.gph"), + (1, 64_799, "1/064/799.gph"), + (2, 1_036_799, "2/001/036/799.gph"), + ] { + assert_eq!( + GraphId::try_from_components(level, max_tile_id, 0) + .unwrap() + .file_path("gph"), + Ok(path.into()) + ); + assert_eq!( + GraphId::try_from_components(level, max_tile_id + 1, 0) + .unwrap() + .file_path("gph"), + Err(InvalidGraphIdError::GraphTileId) + ); + } + // Levels 4 to 7 fit in the bit field, but they have no tiling system. + assert_eq!( + GraphId::try_from_components(5, 0, 0) + .unwrap() + .file_path("gph"), + Err(InvalidGraphIdError::Level) + ); + } } diff --git a/valhalla-graphtile/src/graph_tile.rs b/valhalla-graphtile/src/graph_tile.rs index 48e2df0..e7199b6 100644 --- a/valhalla-graphtile/src/graph_tile.rs +++ b/valhalla-graphtile/src/graph_tile.rs @@ -12,6 +12,8 @@ use std::sync::Arc; use thiserror::Error; use zerocopy::{FromBytes, I16, LE, U32}; +#[cfg(test)] +use crate::test_fixtures; use enumset::EnumSet; use geo::{CoordFloat, Point, coord}; use num_traits::FromPrimitive; @@ -458,6 +460,8 @@ pub struct GraphNode<'a> { /// Cloning this struct is cheap, since the internal types are all references. #[derive(Clone)] pub struct GraphTileView<'a> { + /// All bytes of the tile, from the start of the header to the end of the last section. + tile_bytes: &'a [u8], /// Header with various metadata about the tile and internal sizes. header: &'a GraphTileHeader, /// The list of nodes in the graph tile. @@ -489,6 +493,16 @@ pub struct GraphTileView<'a> { // TODO: Operator one stops(?) } +impl<'a> GraphTileView<'a> { + /// Returns the bytes that this view decoded, with no copy. + /// + /// The slice is exactly one full tile, with no trailing bytes. + #[inline] + pub fn tile_bytes(&self) -> &'a [u8] { + self.tile_bytes + } +} + impl GraphTile for GraphTileView<'_> { #[inline] fn graph_id(&self) -> GraphId { @@ -665,6 +679,7 @@ impl<'a> TryFrom<&'a [u8]> for GraphTileView<'a> { type Error = GraphTileDecodingError; fn try_from(bytes: &'a [u8]) -> Result { + let tile_bytes = bytes; let (header, bytes) = GraphTileHeader::ref_from_prefix(bytes).map_err(|e| { GraphTileDecodingError::CastError { field: "header".to_string(), @@ -882,6 +897,7 @@ impl<'a> TryFrom<&'a [u8]> for GraphTileView<'a> { if bytes.is_empty() { Ok(Self { + tile_bytes, header, nodes, transitions, @@ -932,10 +948,7 @@ static TEST_GRAPH_TILE_L0: LazyLock = LazyLock::new(|| { let relative_path = TEST_GRAPH_TILE_ID_L0 .file_path("gph") .expect("Unable to get relative path"); - let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles") - .join(relative_path); + let path = test_fixtures::andorra_tiles_dir().join(relative_path); let bytes = std::fs::read(path).expect("Unable to read file"); OwnedGraphTileHandle::try_from(bytes).expect("Unable to get tile") @@ -946,10 +959,7 @@ static TEST_GRAPH_TILE_L2: LazyLock = LazyLock::new(|| { let relative_path = TEST_GRAPH_TILE_ID_L2 .file_path("gph") .expect("Unable to get relative path"); - let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles") - .join(relative_path); + let path = test_fixtures::andorra_tiles_dir().join(relative_path); let bytes = std::fs::read(path).expect("Unable to read file"); OwnedGraphTileHandle::try_from(bytes).expect("Unable to get tile") @@ -971,10 +981,7 @@ static TEST_GRAPH_TILE_WITH_FLOW: LazyLock = LazyLock::new let relative_path = TEST_GRAPH_TILE_ID_L0 .file_path("gph") .expect("Unable to get relative path"); - let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles-with-traffic") - .join(relative_path); + let path = test_fixtures::andorra_tiles_with_traffic_dir().join(relative_path); let bytes = std::fs::read(path).expect("Unable to read file"); OwnedGraphTileHandle::try_from(bytes).expect("Unable to get tile") @@ -992,6 +999,23 @@ mod tests { use enumset::{EnumSet, enum_set}; use rand::{RngExt, rng}; + #[test] + fn test_tile_bytes_is_the_decoded_slice() { + for tile in [ + &*TEST_GRAPH_TILE_L0, + &*TEST_GRAPH_TILE_L2, + &*TEST_GRAPH_TILE_WITH_FLOW, + ] { + let decoded = tile.borrow_owner().as_slice(); + let tile_bytes = tile.borrow_dependent().tile_bytes(); + assert!( + core::ptr::eq(tile_bytes, decoded), + "The view of {} must return the slice that it decoded, with no copy", + tile.graph_id() + ); + } + } + #[test] fn test_get_opp_edge_index() { let tile = &*TEST_GRAPH_TILE_L0; diff --git a/valhalla-graphtile/src/graph_tile/builder.rs b/valhalla-graphtile/src/graph_tile/builder.rs index 18504cb..6df6647 100644 --- a/valhalla-graphtile/src/graph_tile/builder.rs +++ b/valhalla-graphtile/src/graph_tile/builder.rs @@ -129,6 +129,7 @@ fn build_speed_profile_dedup_map(memory: &[I16]) -> HashMap<[i16; COEFFICIEN impl<'a> From<&'a OwnedGraphTileHandle> for GraphTileBuilder<'a> { fn from(value: &'a OwnedGraphTileHandle) -> Self { let GraphTileView { + tile_bytes: _, header, nodes, transitions, @@ -665,6 +666,7 @@ where mod tests { use crate::graph_tile::predicted_speeds::COEFFICIENT_COUNT; use crate::graph_tile::{GraphTileBuilder, GraphTileHeader, OwnedGraphTileHandle}; + use crate::test_fixtures; use std::path::Path; use walkdir::WalkDir; @@ -709,9 +711,7 @@ mod tests { #[test] fn round_trip_from_builder_without_changes() { - let base_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base_dir = test_fixtures::andorra_tiles_dir(); for entry in WalkDir::new(base_dir) { let entry = entry.expect("Unable to traverse directory"); @@ -733,9 +733,7 @@ mod tests { #[test] fn round_trip_from_builder_without_changes_including_traffic() { - let base_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles-with-traffic"); + let base_dir = test_fixtures::andorra_tiles_with_traffic_dir(); for entry in WalkDir::new(base_dir) { let entry = entry.expect("Unable to traverse directory"); @@ -757,9 +755,7 @@ mod tests { #[test] fn add_predicted_speeds() { - let base_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base_dir = test_fixtures::andorra_tiles_dir(); let in_bytes = std::fs::read(base_dir.join("0").join("003").join("015.gph")) .expect("Unable to read file"); @@ -782,9 +778,7 @@ mod tests { .with_predicted_encoded_speeds(42, "BcUACQAEACEADP/7/9sAGf/fAAwAGwARAAX/+AAAAB0AFAAS//AAF//+ACwACQAqAAAAKP/6AEMABABsAAsBBAAq/rcAAP+bAAz/zv/s/9MABf/Y//X/8//9/+wACf/P//EADv/8//L//P/y//H////7AAwAFf/5//oADgAZAAQAFf/3/+8AB//yAB8ABv/0AAUAEf/8//QAFAAG//b////j//v//QAT//7/+f/kABMABwABAAv/6//8//cAEwAAABT/8v/6//wAAAAQ//cACwAFAAT/1//sAAEADAABAAYAE//9AAn/7gAH/+AAFQAB//4AC//o/+gAE//+AAAAFf/l//kABP/+//kACAAG//cAHv/qAB0AAv/4/+v/+wALAAMABP/3AAT/8wAIAAr/9wAK//j/+wAEAAD/+P/8//v/8f/2//L//AALAAcABgAG//gAAv/5AAoAHv//AAcAFf/zABD/7AAUAAv/7v/8AAgACAAN//0ADP/iABD/9f/3//7/+P/3AAQADP//AAMABw==").unwrap() .into_bytes().unwrap(); - let base_dir_with_traffic = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles-with-traffic"); + let base_dir_with_traffic = test_fixtures::andorra_tiles_with_traffic_dir(); let expected_out_bytes = std::fs::read(base_dir_with_traffic.join("0").join("003").join("015.gph")) .expect("Unable to read file"); @@ -798,9 +792,7 @@ mod tests { #[test] fn predicted_speeds_deduplication() { - let base_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base_dir = test_fixtures::andorra_tiles_dir(); let in_bytes = std::fs::read(base_dir.join("0").join("003").join("015.gph")) .expect("Unable to read file"); diff --git a/valhalla-graphtile/src/lib.rs b/valhalla-graphtile/src/lib.rs index 179c3c5..ef4cc19 100644 --- a/valhalla-graphtile/src/lib.rs +++ b/valhalla-graphtile/src/lib.rs @@ -1,9 +1,12 @@ #![doc = include_str!("../README.md")] +pub mod extract; mod graph_id; pub mod graph_tile; pub mod shape_codec; pub mod spatial; +#[cfg(test)] +mod test_fixtures; pub mod tile_hierarchy; pub mod tile_provider; pub mod traffic_tile; @@ -19,7 +22,7 @@ use serde::{Deserialize, Serialize}; // The implementations are sufficiently complex that we want to have lots of files, // But many of those only have one or two useful definitions to re-export, // so this flattens things for better ergonomics. -pub use graph_id::GraphId; +pub use graph_id::{GraphId, InvalidGraphIdError}; /// Road class; broad hierarchies of relative (and sometimes locally specific) importance. /// diff --git a/valhalla-graphtile/src/test_fixtures.rs b/valhalla-graphtile/src/test_fixtures.rs new file mode 100644 index 0000000..1b69025 --- /dev/null +++ b/valhalla-graphtile/src/test_fixtures.rs @@ -0,0 +1,74 @@ +//! The test fixtures in the `fixtures` directory of this crate. +//! +//! All fixtures come from the same OpenStreetMap extract of Andorra. +//! Miri does not support memory maps, so the tarball paths are not compiled under Miri. + +use crate::GraphId; +use crate::graph_tile::GraphTile; +use crate::graph_tile::handles::OwnedGraphTileHandle; +use crate::tile_provider::TILE_FILE_EXTENSION; +use std::path::PathBuf; +use std::sync::LazyLock; +use walkdir::WalkDir; + +fn fixture(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("fixtures") + .join(name) +} + +/// A tile directory, as `valhalla_build_tiles` writes it. +pub(crate) fn andorra_tiles_dir() -> PathBuf { + fixture("andorra-tiles") +} + +/// The base graph ID of the smallest tile in [`andorra_tiles_dir`]. +/// +/// Tests that run under Miri use this tile to stay fast. +pub(crate) fn smallest_andorra_tile() -> GraphId { + static SMALLEST: LazyLock = LazyLock::new(|| { + let path = WalkDir::new(andorra_tiles_dir()) + .into_iter() + .map(|entry| entry.expect("Unable to read the fixture directory")) + .filter(|entry| { + entry.file_type().is_file() + && entry + .path() + .extension() + .is_some_and(|extension| extension == TILE_FILE_EXTENSION) + }) + .min_by_key(|entry| { + let len = entry + .metadata() + .expect("Unable to read the tile metadata") + .len(); + (len, entry.path().to_path_buf()) + }) + .expect("The fixture directory has tiles") + .into_path(); + let bytes = std::fs::read(path).expect("Unable to read the tile"); + OwnedGraphTileHandle::try_from(bytes) + .expect("Unable to decode the tile") + .graph_id() + }); + *SMALLEST +} + +/// The tiles of [`andorra_tiles_dir`] in an extract tarball, +/// as `valhalla_build_extract` writes it. +#[cfg(not(miri))] +pub(crate) fn andorra_tiles_tar() -> PathBuf { + fixture("andorra-tiles.tar") +} + +/// The tiles of [`andorra_tiles_dir`] with predicted traffic speeds +/// from `valhalla_add_predicted_traffic`. +pub(crate) fn andorra_tiles_with_traffic_dir() -> PathBuf { + fixture("andorra-tiles-with-traffic") +} + +/// A live traffic tarball for the Andorra tiles. +#[cfg(not(miri))] +pub(crate) fn andorra_traffic_tar() -> PathBuf { + fixture("andorra-traffic.tar") +} diff --git a/valhalla-graphtile/src/tile_provider/directory.rs b/valhalla-graphtile/src/tile_provider/directory.rs index 01212a3..154b1e0 100644 --- a/valhalla-graphtile/src/tile_provider/directory.rs +++ b/valhalla-graphtile/src/tile_provider/directory.rs @@ -4,7 +4,9 @@ use crate::graph_tile::{GraphTileBuildError, GraphTileBuilder}; use crate::graph_tile::{GraphTileView, handles::OwnedGraphTileHandle}; use crate::spatial::bbox_with_center; use crate::tile_hierarchy::STANDARD_LEVELS; -use crate::tile_provider::{GraphTileProvider, GraphTileProviderError, LockTable}; +use crate::tile_provider::{ + GraphTileProvider, GraphTileProviderError, LockTable, TILE_FILE_EXTENSION, +}; use geo::{CoordFloat, Point}; use lru::LruCache; use num_traits::FromPrimitive; @@ -124,7 +126,9 @@ impl DirectoryGraphTileProvider { /// /// NOTE: This function assumes that the graph ID is already a base ID. fn path_for_graph_id(&self, graph_id: GraphId) -> Result { - Ok(self.base_directory.join(graph_id.file_path("gph")?)) + Ok(self + .base_directory + .join(graph_id.file_path(TILE_FILE_EXTENSION)?)) } } @@ -176,6 +180,28 @@ impl GraphTileProvider for DirectoryGraphTileProvider { Ok(tile) } + /// The size comes from the metadata of the tile file, so this reads no tile data. + /// + /// The size is the size of the file at the time of the call. + /// If the file changes after the provider caches the tile, + /// the size and the cached tile do not agree. + fn tile_size(&self, graph_id: GraphId) -> Result { + let base_graph_id = graph_id.tile_base_id(); + let size = match std::fs::metadata(self.path_for_graph_id(base_graph_id)?) { + Ok(metadata) if metadata.is_file() => metadata.len(), + Ok(_) => return Err(GraphTileProviderError::TileDoesNotExist), + Err(e) if e.kind() == ErrorKind::NotFound => { + return Err(GraphTileProviderError::TileDoesNotExist); + } + Err(e) => return Err(e.into()), + }; + + u32::try_from(size).map_err(|_| GraphTileProviderError::TileTooLarge { + graph_id: base_graph_id, + size, + }) + } + fn tiles_within_radius( &self, center: Point, @@ -202,21 +228,19 @@ mod test { use super::DirectoryGraphTileProvider; use crate::GraphId; use crate::graph_tile::GraphTile; + use crate::test_fixtures; use crate::tile_hierarchy::STANDARD_LEVELS; - use crate::tile_provider::GraphTileProvider; + use crate::tile_provider::{GraphTileProvider, GraphTileProviderError, TILE_FILE_EXTENSION}; use core::num::NonZeroUsize; use rand::{ distr::{Distribution, Uniform}, rng, }; use std::collections::HashSet; - use std::path::PathBuf; #[test] fn test_get_tile() { - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(1).unwrap()); let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); let tile = provider @@ -232,9 +256,7 @@ mod test { fn test_get_opp_edge() { let mut rng = rng(); - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(1).unwrap()); let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); let tile = provider @@ -264,9 +286,7 @@ mod test { #[test] fn test_get_shortcut_fixture() { - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(1).unwrap()); // This test relies on the state of the andorra-tiles fixtures. @@ -310,9 +330,7 @@ mod test { #[cfg(not(miri))] #[test] fn test_level_zero_shortcut_sanity() { - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(1).unwrap()); let base_id = GraphId::try_from_components(0, 3015, 0).unwrap(); @@ -393,9 +411,7 @@ mod test { // how the tile structure works. // Specifically, there is only a single "copy" of a feature in the routing graph, // and all features of a specific road class exist in specific tile levels. - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(1).unwrap()); let graph_ids = vec![ GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"), @@ -441,4 +457,92 @@ mod test { } } } + + #[test] + fn test_tile_size_matches_view_bytes() { + let provider = + DirectoryGraphTileProvider::new(test_fixtures::andorra_tiles_dir(), NonZeroUsize::MIN); + // The feature index is not zero, so the provider must use the base ID of the tile. + let graph_id = GraphId::try_from_components(0, 3015, 1).expect("Unable to create graph ID"); + + let size = provider + .tile_size(graph_id) + .expect("Unable to get the tile size"); + let tile_bytes_len = provider + .with_tile_containing(graph_id, |tile| tile.tile_bytes().len()) + .expect("Unable to get tile"); + assert_eq!( + usize::try_from(size).expect("The size fits in a usize"), + tile_bytes_len, + "The tile size must be the length of the tile bytes" + ); + } + + #[test] + fn test_tile_size_of_missing_tile() { + let provider = + DirectoryGraphTileProvider::new(test_fixtures::andorra_tiles_dir(), NonZeroUsize::MIN); + // This tile is not in the Andorra extract. + let graph_id = GraphId::try_from_components(0, 0, 0).expect("Unable to create graph ID"); + + assert!( + matches!( + provider.tile_size(graph_id), + Err(GraphTileProviderError::TileDoesNotExist) + ), + "A tile without a file must not exist" + ); + } + + #[test] + fn test_tile_size_of_directory_at_tile_path() { + let base = tempfile::tempdir().expect("Unable to create a temporary directory"); + let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); + let tile_path = base.path().join( + graph_id + .file_path(TILE_FILE_EXTENSION) + .expect("Unable to get the tile path"), + ); + std::fs::create_dir_all(tile_path).expect("Unable to create a directory at the tile path"); + let provider = + DirectoryGraphTileProvider::new(base.path().to_path_buf(), NonZeroUsize::MIN); + + assert!( + matches!( + provider.tile_size(graph_id), + Err(GraphTileProviderError::TileDoesNotExist) + ), + "A directory at the tile path must not be a tile" + ); + } + + // On Unix, a sparse file of this size uses almost no disk space. + #[cfg(all(unix, not(miri)))] + #[test] + fn test_tile_size_too_large() { + let base = tempfile::tempdir().expect("Unable to create a temporary directory"); + let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); + let tile_path = base.path().join( + graph_id + .file_path(TILE_FILE_EXTENSION) + .expect("Unable to get the tile path"), + ); + let parent = tile_path.parent().expect("The tile path has a parent"); + std::fs::create_dir_all(parent).expect("Unable to create the tile directory"); + let file_len = u64::from(u32::MAX) + 1; + std::fs::File::create(&tile_path) + .and_then(|file| file.set_len(file_len)) + .expect("Unable to create a large sparse file"); + let provider = + DirectoryGraphTileProvider::new(base.path().to_path_buf(), NonZeroUsize::MIN); + + assert!( + matches!( + provider.tile_size(graph_id), + Err(GraphTileProviderError::TileTooLarge { graph_id: id, size }) + if id == graph_id && size == file_len + ), + "A tile file larger than u32::MAX bytes must be an error" + ); + } } diff --git a/valhalla-graphtile/src/tile_provider/mod.rs b/valhalla-graphtile/src/tile_provider/mod.rs index d618b47..8427787 100644 --- a/valhalla-graphtile/src/tile_provider/mod.rs +++ b/valhalla-graphtile/src/tile_provider/mod.rs @@ -26,6 +26,10 @@ use crate::graph_tile::{ pub use directory::DirectoryGraphTileProvider; pub use iterators::GraphSearchResult; use iterators::ResultsWithinRadius; +pub(crate) use tarball::{INDEX_BIN_FILE_NAME, INDEX_ENTRY_SIZE, TAR_BLOCK_SIZE}; + +/// The extension of graph tile files, in tile directories and in extract tarballs. +pub(crate) const TILE_FILE_EXTENSION: &str = "gph"; pub use tarball::{ InvalidIndexBinError, TarballTileProvider, TileIndexBinEntry, parse_index_bin, serialize_index_bin, @@ -54,6 +58,8 @@ pub enum GraphTileProviderError { InvalidIndexBin(#[from] InvalidIndexBinError), #[error("Unsupported tile version; this may or may not be compatible.")] UnsupportedTileVersion, + #[error("the tile file for {graph_id} has {size} bytes, which is more than a tile can have")] + TileTooLarge { graph_id: GraphId, size: u64 }, } /// A generic trait for all graph tile providers. @@ -110,6 +116,24 @@ pub trait GraphTileProvider { graph_id: GraphId, ) -> Result; + /// Gets the size in bytes of the tile containing the given graph ID, + /// without reading or decoding the tile. + /// + /// While the tile data does not change, the size is the length of + /// [`GraphTileView::tile_bytes`] for the same tile. + /// + /// # Performance + /// + /// Implementations SHOULD get the size from an index or from file metadata. + /// A caller can then get the sizes of many tiles before it reads any of them. + /// + /// # Errors + /// + /// If the provider does not have the tile, + /// this returns [`GraphTileProviderError::TileDoesNotExist`]. + /// Other errors are possible, such as I/O errors. + fn tile_size(&self, graph_id: GraphId) -> Result; + /// Enumerate base tile Graph IDs across all hierarchy levels that intersect a circle around /// `center` with radius `radius`. /// @@ -519,13 +543,13 @@ impl LockTable { mod tests { use crate::GraphId; use crate::graph_tile::GraphTile; + use crate::test_fixtures; use crate::tile_hierarchy::STANDARD_LEVELS; use crate::tile_provider::{DirectoryGraphTileProvider, GraphTileProvider}; use geo::{Destination, HasDimensions, Haversine, LineString, Validation, coord, point}; use rand::rngs::ThreadRng; use rand::{RngExt, rng}; use std::num::NonZeroUsize; - use std::path::PathBuf; #[test] fn haversine_antimeridian_wraps() { @@ -544,9 +568,7 @@ mod tests { fn test_nodes_within_radius() { let mut rng = rng(); - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(4).unwrap()); let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); provider.with_tile_containing_or_panic(graph_id, |tile_view| { @@ -665,9 +687,7 @@ mod tests { #[test] fn test_edges_within_radius_level0() { - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(4).unwrap()); let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); sanity_check_edges_within_radius(provider, graph_id, rng()); @@ -675,9 +695,7 @@ mod tests { #[test] fn test_edges_within_radius_level2() { - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(4).unwrap()); // Use a known level 2 tile from fixtures to exercise edge bins path let graph_id = diff --git a/valhalla-graphtile/src/tile_provider/tarball.rs b/valhalla-graphtile/src/tile_provider/tarball.rs index 3866fd7..2f6e91a 100644 --- a/valhalla-graphtile/src/tile_provider/tarball.rs +++ b/valhalla-graphtile/src/tile_provider/tarball.rs @@ -102,7 +102,9 @@ impl TarballTileProvider { ); } - let mut archive = Archive::new(File::open(&path)?); + let archive_file = File::open(&path)?; + let archive_len = archive_file.metadata()?.len(); + let mut archive = Archive::new(archive_file); let mut entries = archive.entries()?; let Some(entry) = entries.next() else { @@ -112,10 +114,10 @@ impl TarballTileProvider { }; let mut entry = entry?; - if entry.path()?.to_string_lossy() != "index.bin" { - return Err(GraphTileProviderError::InvalidTarball( - "Expected index.bin at the start of the archive".to_string(), - )); + if entry.path()?.to_string_lossy() != INDEX_BIN_FILE_NAME { + return Err(GraphTileProviderError::InvalidTarball(format!( + "Expected {INDEX_BIN_FILE_NAME} at the start of the archive" + ))); } // Read the index file @@ -128,7 +130,7 @@ impl TarballTileProvider { let mut tile_index: HashMap = HashMap::with_capacity(index_entries.len()); for entry in index_entries { - let (graph_id, offset) = entry.validated()?; + let (graph_id, offset) = entry.validated(archive_len)?; tile_index.insert( graph_id, TileOffset { @@ -149,6 +151,15 @@ impl TarballTileProvider { Arc::new(MmapOptions::new().map_raw_read_only(&file)?) }; + // Tile pointers slice the map without a bounds check, + // so the length of the map must equal the length that the index check used. + if mmap.len() as u64 != archive_len { + return Err(GraphTileProviderError::InvalidTarball(format!( + "the archive changed size while it was opened ({archive_len} bytes, then {} bytes)", + mmap.len() + ))); + } + let mut tile_cache = HashMap::new(); if CACHE { @@ -211,6 +222,11 @@ impl GraphTileProvider for TarballTileProvider { Ok(Arc::new(unsafe { MmapGraphTileHandle::try_from(pointer)? })) } + #[inline] + fn tile_size(&self, graph_id: GraphId) -> Result { + self.tile_size(graph_id) + } + #[inline] fn tiles_within_radius( &self, @@ -252,6 +268,11 @@ impl GraphTileProvider for TarballTileProvider { Ok(tile.clone()) } + #[inline] + fn tile_size(&self, graph_id: GraphId) -> Result { + self.tile_size(graph_id) + } + #[inline] fn tiles_within_radius( &self, @@ -263,6 +284,15 @@ impl GraphTileProvider for TarballTileProvider { } impl TarballTileProvider { + /// The size comes from the `index.bin` of the tarball, so this reads no tile data. + #[inline] + fn tile_size(&self, graph_id: GraphId) -> Result { + self.tile_index + .get(&graph_id.tile_base_id()) + .map(|tile_offset| tile_offset.size) + .ok_or(GraphTileProviderError::TileDoesNotExist) + } + #[inline] fn tiles_within_radius( &self, @@ -430,11 +460,15 @@ pub struct TileIndexBinEntry { } /// The size of one [`TileIndexBinEntry`] in bytes. -const INDEX_ENTRY_SIZE: usize = size_of::(); +pub(crate) const INDEX_ENTRY_SIZE: usize = size_of::(); /// The block size of a tar archive. -/// The data of each tar entry starts on a block boundary. -const TAR_BLOCK_SIZE: u64 = 512; +/// +/// Each entry is a header block, then its data, padded with zeros to whole blocks. +pub(crate) const TAR_BLOCK_SIZE: u64 = 512; + +/// The file name of the tile index, which is the first entry of an extract tarball. +pub(crate) const INDEX_BIN_FILE_NAME: &str = "index.bin"; /// The reasons why an `index.bin` file or one of its entries is not valid. #[derive(Debug, Error, PartialEq, Eq)] @@ -453,6 +487,15 @@ pub enum InvalidIndexBinError { "the entry for {graph_id} has offset {offset}; expected a non-zero multiple of {TAR_BLOCK_SIZE}" )] Offset { graph_id: GraphId, offset: u64 }, + #[error( + "the entry for {graph_id} has {size} bytes at offset {offset}, which is past the end of the {archive_len}-byte archive" + )] + OutOfBounds { + graph_id: GraphId, + offset: u64, + size: u32, + archive_len: u64, + }, } impl TileIndexBinEntry { @@ -475,20 +518,25 @@ impl TileIndexBinEntry { }) } - /// Validates a parsed entry, and returns its graph ID and offset. + /// Checks the entry against an archive of `archive_len` bytes, + /// and returns its graph ID and offset. /// - /// [`parse_index_bin`] does not examine the entry values, - /// so the read path must call this before it uses an entry. - /// The checks are the same as in [`TileIndexBinEntry::new`]. - fn validated(&self) -> Result<(GraphId, u64), InvalidIndexBinError> { + /// Call this before you use an entry from [`parse_index_bin`], which does not check the values. + fn validated(&self, archive_len: u64) -> Result<(GraphId, u64), InvalidIndexBinError> { let graph_id = self.graph_id()?; let offset = self.offset.get(); check_offset(graph_id, offset)?; + check_range(graph_id, offset, self.size.get(), archive_len)?; Ok((graph_id, offset)) } - fn graph_id(&self) -> Result { + /// The graph ID of the indexed tile. + /// + /// # Errors + /// + /// Returns an error if the stored tile ID has nonzero feature index bits. + pub fn graph_id(&self) -> Result { // SAFETY: We know that the bit field cannot contain a value // larger than the max allowed value (it's limited to 46 bits). // Therefore, this is guaranteed to be a valid Graph ID bit pattern. @@ -497,6 +545,18 @@ impl TileIndexBinEntry { Ok(graph_id) } + + /// The byte offset of the tile payload from the beginning of the tarball. + #[inline] + pub fn offset(&self) -> u64 { + self.offset.get() + } + + /// The size of the tile in bytes. + #[inline] + pub fn size(&self) -> u32 { + self.size.get() + } } /// Makes sure that `graph_id` identifies a full tile, @@ -524,6 +584,29 @@ fn check_offset(graph_id: GraphId, offset: u64) -> Result<(), InvalidIndexBinErr Ok(()) } +/// Makes sure that the tile at `offset` with `size` bytes +/// lies inside an archive of `archive_len` bytes. +fn check_range( + graph_id: GraphId, + offset: u64, + size: u32, + archive_len: u64, +) -> Result<(), InvalidIndexBinError> { + if offset + .checked_add(u64::from(size)) + .is_none_or(|end| end > archive_len) + { + return Err(InvalidIndexBinError::OutOfBounds { + graph_id, + offset, + size, + archive_len, + }); + } + + Ok(()) +} + /// Parses an `index.bin` file to enable random access. /// /// See [`TileIndexBinEntry`] for a description of the tile format. @@ -574,11 +657,12 @@ pub fn serialize_index_bin(entries: &[TileIndexBinEntry]) -> Result<&[u8], Inval mod test { use super::*; use crate::graph_tile::{GraphTile, GraphTileView}; + #[cfg(not(miri))] + use crate::test_fixtures; use crate::tile_provider::DirectoryGraphTileProvider; use proptest::collection::vec; use proptest::{prop_assert, prop_assert_eq, proptest}; use std::num::NonZeroUsize; - use std::path::PathBuf; /// Bytes taken from the start of a large extract generated by official Valhalla tooling. const INDEX_BIN_FIXTURE: &[u8] = &[ @@ -618,14 +702,30 @@ mod test { fn test_validate_parsed_entry() { let graph_id = GraphId::try_from_components(0, 2540, 0).expect("Unable to create graph ID"); let index_entries = parse_index_bin(INDEX_BIN_FIXTURE).expect("Unable to parse fixture"); - assert_eq!(index_entries[0].validated(), Ok((graph_id, 23040))); + // The smallest archive that can contain the first tile ends where that tile ends. + let first_entry_end = index_entries[0].offset() + u64::from(index_entries[0].size()); + assert_eq!( + index_entries[0].validated(first_entry_end), + Ok((graph_id, 23040)), + "A tile may end exactly at the end of the archive" + ); + assert_eq!( + index_entries[0].validated(first_entry_end - 1), + Err(InvalidIndexBinError::OutOfBounds { + graph_id, + offset: 23040, + size: 1_181_852, + archive_len: first_entry_end - 1 + }), + "A tile must not end past the end of the archive" + ); // Change the low byte of the first offset (23040 = 0x5a00) to make it unaligned let mut bytes = INDEX_BIN_FIXTURE.to_vec(); bytes[0] = 0x01; let index_entries = parse_index_bin(&bytes).expect("Unable to parse modified fixture"); assert_eq!( - index_entries[0].validated(), + index_entries[0].validated(first_entry_end), Err(InvalidIndexBinError::Offset { graph_id, offset: 23041 @@ -638,7 +738,7 @@ mod test { bytes[..8].fill(0); let index_entries = parse_index_bin(&bytes).expect("Unable to parse modified fixture"); assert_eq!( - index_entries[0].validated(), + index_entries[0].validated(first_entry_end), Err(InvalidIndexBinError::Offset { graph_id, offset: 0 @@ -651,7 +751,7 @@ mod test { bytes[11] |= 0x02; let index_entries = parse_index_bin(&bytes).expect("Unable to parse modified fixture"); assert_eq!( - index_entries[0].validated(), + index_entries[0].validated(first_entry_end), Err(InvalidIndexBinError::FeatureIndex( graph_id .with_feature_index(1) @@ -691,6 +791,72 @@ mod test { assert_eq!(index_entries[2].size.get(), 1_809_704); } + #[test] + fn test_check_range() { + let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); + + assert_eq!(check_range(graph_id, 512, 512, 1024), Ok(())); + assert_eq!( + check_range(graph_id, 512, 513, 1024), + Err(InvalidIndexBinError::OutOfBounds { + graph_id, + offset: 512, + size: 513, + archive_len: 1024 + }), + "A tile must end at or before the end of the archive" + ); + assert_eq!( + check_range(graph_id, u64::MAX - 511, 1024, u64::MAX), + Err(InvalidIndexBinError::OutOfBounds { + graph_id, + offset: u64::MAX - 511, + size: 1024, + archive_len: u64::MAX + }), + "The end of the range must not overflow" + ); + } + + // Tragically, memory maps are not supported by Miri! + #[cfg(not(miri))] + #[test] + fn test_reject_index_past_end_of_archive() { + let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); + // The index gives 10,000 bytes, but the archive has only 100 bytes of tile data. + let index_entry = + TileIndexBinEntry::new(graph_id, 1536, 10_000).expect("Unable to create index entry"); + let index_bytes = serialize_index_bin(core::slice::from_ref(&index_entry)) + .expect("Unable to serialize the index"); + + let file = tempfile::NamedTempFile::new().expect("Unable to create a temporary file"); + let mut builder = tar::Builder::new(file.as_file()); + for (path, payload) in [("index.bin", index_bytes), ("0/003/015.gph", &[0x5a; 100])] { + let mut header = tar::Header::new_ustar(); + header.set_size(payload.len() as u64); + header.set_cksum(); + builder + .append_data(&mut header, path, payload) + .expect("Unable to append a tar entry"); + } + builder.finish().expect("Unable to finish the archive"); + + let result = TarballTileProvider::new_readonly(file.path()); + assert!( + matches!( + result, + Err(GraphTileProviderError::InvalidIndexBin( + InvalidIndexBinError::OutOfBounds { + offset: 1536, + size: 10_000, + .. + } + )) + ), + "An index entry past the end of the archive must be rejected" + ); + } + #[test] fn test_tile_index_bin_entry_new() { let graph_id = GraphId::try_from_components(0, 2540, 0).expect("Unable to create graph ID"); @@ -776,8 +942,12 @@ mod test { proptest! { #[test] fn test_index_bin_roundtrip( - // Levels are 3 bits, tile IDs are 22 bits, and offsets are counted in 512-byte blocks - raw_entries in vec((0u8..=7, 0u64..1 << 22, 1u64..=u64::MAX / 512, 0u32..=u32::MAX), 1..64) + // Levels are 3 bits, tile IDs are 22 bits, and offsets count 512-byte blocks. + // The offset bound keeps the end of every tile inside a u64. + raw_entries in vec( + (0u8..=7, 0u64..1 << 22, 1u64..=(u64::MAX - u64::from(u32::MAX)) / 512, 0u32..=u32::MAX), + 1..64 + ) ) { let entries: Vec = raw_entries .into_iter() @@ -793,9 +963,14 @@ mod test { let parsed = parse_index_bin(bytes).expect("Serialized bytes always parse"); prop_assert_eq!(parsed, entries.as_slice()); - // The read path accepts every entry that the write path creates + // Every entry is valid in an archive that ends where the last tile ends. + let archive_len = parsed + .iter() + .map(|entry| entry.offset() + u64::from(entry.size())) + .max() + .expect("The entry list is non-empty"); for entry in parsed { - prop_assert!(entry.validated().is_ok()); + prop_assert!(entry.validated(archive_len).is_ok()); } } } @@ -804,9 +979,7 @@ mod test { #[cfg(not(miri))] #[test] fn test_get_tile() { - let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles.tar"); + let path = test_fixtures::andorra_tiles_tar(); let provider = TarballTileProvider::new_readonly(path).expect("Unable to init tile provider"); let graph_id = GraphId::try_from_components(0, 3015, 0).expect("Unable to create graph ID"); @@ -841,15 +1014,11 @@ mod test { GraphId::try_from_components(2, 763927, 1).expect("Unable to create graph ID"), ]; - let base = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles"); + let base = test_fixtures::andorra_tiles_dir(); let directory_provider = DirectoryGraphTileProvider::new(base, NonZeroUsize::new(1).unwrap()); - let tarball_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-tiles.tar"); + let tarball_path = test_fixtures::andorra_tiles_tar(); let tarball_provider = TarballTileProvider::new_readonly(tarball_path.clone()) .expect("Unable to init tile provider"); let immutable_tarball_provider = unsafe { @@ -876,6 +1045,70 @@ mod test { .as_tile_bytes() }; assert_eq!(directory_tile.borrow_owner(), immutable_tarball_tile_bytes); + + for (provider_name, tile_bytes, tile_size) in [ + ( + "directory", + directory_provider + .with_tile_containing(*graph_id, |tile| tile.tile_bytes().to_vec()), + directory_provider.tile_size(*graph_id), + ), + ( + "tarball", + tarball_provider + .with_tile_containing(*graph_id, |tile| tile.tile_bytes().to_vec()), + tarball_provider.tile_size(*graph_id), + ), + ( + "immutable tarball", + immutable_tarball_provider + .with_tile_containing(*graph_id, |tile| tile.tile_bytes().to_vec()), + immutable_tarball_provider.tile_size(*graph_id), + ), + ] { + let tile_bytes = tile_bytes.expect("Unable to get tile"); + assert_eq!( + tile_bytes, + directory_tile.borrow_owner().as_slice(), + "The {provider_name} view must give the bytes of the tile file for {graph_id}" + ); + let tile_size = tile_size.expect("Unable to get the tile size"); + assert_eq!( + usize::try_from(tile_size).expect("The size fits in a usize"), + tile_bytes.len(), + "The {provider_name} size of {graph_id} must be the length of its bytes" + ); + } } } + + // Tragically, memory maps are not supported by Miri! + #[cfg(not(miri))] + #[test] + fn test_tile_size_of_missing_tile() { + let tarball_path = test_fixtures::andorra_tiles_tar(); + let tarball_provider = TarballTileProvider::new_readonly(tarball_path.clone()) + .expect("Unable to init tile provider"); + // SAFETY: Nothing writes to the fixture tarball while the tests run. + let immutable_tarball_provider = unsafe { + TarballTileProvider::new_immutable(tarball_path).expect("Unable to init tile provider") + }; + // This tile is not in the Andorra extract. + let graph_id = GraphId::try_from_components(0, 0, 0).expect("Unable to create graph ID"); + + assert!( + matches!( + tarball_provider.tile_size(graph_id), + Err(GraphTileProviderError::TileDoesNotExist) + ), + "A tile that is not in the index must not exist" + ); + assert!( + matches!( + immutable_tarball_provider.tile_size(graph_id), + Err(GraphTileProviderError::TileDoesNotExist) + ), + "A tile that is not in the index must not exist" + ); + } } diff --git a/valhalla-graphtile/src/tile_provider/traffic.rs b/valhalla-graphtile/src/tile_provider/traffic.rs index 7e7a5b4..876c818 100644 --- a/valhalla-graphtile/src/tile_provider/traffic.rs +++ b/valhalla-graphtile/src/tile_provider/traffic.rs @@ -231,15 +231,14 @@ impl TrafficTileProvider { #[cfg(all(test, not(miri)))] mod tests { use crate::GraphId; + use crate::test_fixtures; use crate::tile_provider::TrafficTileProvider; use crate::traffic_tile::{SpeedValue, TrafficSpeed}; use std::path::PathBuf; #[test] fn test_get_speed() { - let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-traffic.tar"); + let path = test_fixtures::andorra_traffic_tar(); let provider = TrafficTileProvider::new_readonly(path).expect("Unable to init tile provider"); @@ -269,9 +268,7 @@ mod tests { fn test_set_speed() { const DESIRED_SPEED: u8 = 32; - let fixture_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures") - .join("andorra-traffic.tar"); + let fixture_path = test_fixtures::andorra_traffic_tar(); // let tmp_path = PathBuf::from("/Users/ianthetechie/valhalla-docker/valhalla/traffic.tar"); let tmp_dir = option_env!("RUNNER_TEMP").unwrap_or("/tmp"); diff --git a/valinor-cli/Cargo.toml b/valinor-cli/Cargo.toml index cc0b362..1961b76 100644 --- a/valinor-cli/Cargo.toml +++ b/valinor-cli/Cargo.toml @@ -12,6 +12,7 @@ rust-version.workspace = true anyhow = { workspace = true } clap = { workspace = true, features = ["derive", "env"] } serde_json = { workspace = true } +tempfile = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter"] } valhalla-graphtile = { workspace = true, features = ["serde"] } diff --git a/valinor-cli/README.md b/valinor-cli/README.md index f620a82..df299af 100644 --- a/valinor-cli/README.md +++ b/valinor-cli/README.md @@ -1,8 +1,8 @@ # Valinor CLI A CLI multi-tool for quickly inspecting Valhalla graphs. -This utility takes a Valhalla JSON configuration as input, -and has subcommands with various functionality. +Most subcommands take a Valhalla JSON configuration as input +(`extract` is the exception; it takes a tile extract tarball or a tile directory directly). This is primarily meant as a low-level debugging tool for Valinor developers. If this looks useful to you, it probably is; otherwise, it's probably not. @@ -12,9 +12,26 @@ If this looks useful to you, it probably is; otherwise, it's probably not. As an example, you can get edge info like this (works with either the integer or slash separated form): ```shell -RUST_LOG=info cargo run --package valinor-cli -- ~/valhalla-docker/valhalla/valhalla.json get-edge 901881911882 +RUST_LOG=info cargo run --package valinor-cli -- get-edge ~/valhalla-docker/valhalla/valhalla.json 901881911882 ``` Then you'll get some JSON output on your terminal with the details. -Run the `help` subcommand for more details (everything is automatically documented via clap). \ No newline at end of file +## Example: writing an offline sub-extract + +You can write a smaller, fully spec-compliant extract +which contains only some tiles of a source (each tile is `level/tile_id`). +The source can be a tile extract tarball or a tile directory: + +```shell +cargo run --package valinor-cli -- extract valhalla-graphtile/fixtures/andorra-tiles.tar andorra-sub.tar --tile 0/3015,1/47701,2/763926 +cargo run --package valinor-cli -- extract valhalla-graphtile/fixtures/andorra-tiles andorra-sub.tar --tile 0/3015,1/47701,2/763926 +``` + +A one-line summary (tile count and output size) is printed on stderr. +The output is interchangeable with `valhalla_build_extract` output. +The command writes to a temporary file in the output directory, +and renames it to the output path only when the extract is complete. +So the output path never contains a partial extract. + +Run the `help` subcommand for more details (everything is automatically documented via clap). diff --git a/valinor-cli/src/commands/extract.rs b/valinor-cli/src/commands/extract.rs new file mode 100644 index 0000000..e223b9e --- /dev/null +++ b/valinor-cli/src/commands/extract.rs @@ -0,0 +1,300 @@ +//! The `extract` subcommand: writes an offline sub-extract of a tile extract or tile directory. + +use std::{ + collections::HashSet, + io::BufWriter, + num::NonZeroUsize, + path::{Path, PathBuf}, +}; + +#[cfg(unix)] +use std::{fs::Permissions, os::unix::fs::PermissionsExt}; + +use anyhow::{Context, anyhow}; +use clap::Args; +use tempfile::NamedTempFile; +use valhalla_graphtile::GraphId; +use valhalla_graphtile::extract::write_extract; +use valhalla_graphtile::tile_provider::{ + DirectoryGraphTileProvider, GraphTileProvider, TarballTileProvider, +}; + +/// Write an offline sub-extract of a tile extract tarball or a tile directory +/// +/// The output is an extract tarball in the format of `valhalla_build_extract`, +/// with the given tiles copied byte for byte. +#[derive(Args, Debug)] +pub(crate) struct ExtractArgs { + /// Path to the source: a tile extract tarball (e.g. tiles.tar) + /// or a tile directory (e.g. the `tile_dir` of a Valhalla config) + source: PathBuf, + + /// Path to write the output extract tarball to + output: PathBuf, + + /// A tile to copy, as level/tile_id (e.g. `--tile 2/763926`) + /// + /// Give the option once for each tile, + /// or give a comma-separated list (e.g. `--tile 0/3015,1/47701`). + #[arg( + long = "tile", + value_name = "LEVEL/TILE_ID", + required = true, + value_delimiter = ',', + value_parser = parse_tile + )] + pub(crate) tiles: Vec, +} + +/// Parses the base graph ID of a tile from the `level/tile_id` form. +fn parse_tile(input: &str) -> anyhow::Result { + let (level, tile_id) = input + .split_once('/') + .ok_or_else(|| anyhow!("expected the level/tile_id form"))?; + let level = level.parse().context("invalid level")?; + let tile_id = tile_id.parse().context("invalid tile id")?; + Ok(GraphId::try_from_components(level, tile_id, 0)?) +} + +/// A successful extract, as reported on stderr. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct ExtractSummary { + tile_count: usize, + bytes_written: u64, +} + +/// Writes the sub-extract from the tile directory or the tarball at the source path. +fn write_sub_extract(args: &ExtractArgs) -> anyhow::Result { + let bytes_written = if args.source.is_dir() { + // The writer reads each tile once, so a cache is of no use. + let provider = DirectoryGraphTileProvider::new(args.source.clone(), NonZeroUsize::MIN); + write_atomically(&provider, &args.tiles, &args.output)? + } else { + let provider = TarballTileProvider::new_readonly(&args.source).with_context(|| { + format!( + "Failed to open source tile extract at {}", + args.source.display() + ) + })?; + write_atomically(&provider, &args.tiles, &args.output)? + }; + + // The extract contains each tile once, even if the arguments name it more than once. + let tile_count = args.tiles.iter().collect::>().len(); + Ok(ExtractSummary { + tile_count, + bytes_written, + }) +} + +/// Writes the extract to a temporary file next to `output`, and then renames it to `output`. +/// +/// If the write fails, `output` keeps its previous contents. +fn write_atomically( + provider: &P, + tiles: &[GraphId], + output: &Path, +) -> anyhow::Result { + // The rename is atomic only within one file system. + let output_dir = output + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + + let mut builder = tempfile::Builder::new(); + builder.prefix(".valinor-extract-").suffix(".tar.partial"); + // Use the permissions of `File::create`, not the owner-only default of `tempfile`. + #[cfg(unix)] + builder.permissions(Permissions::from_mode(0o666)); + let temp_file: NamedTempFile = builder.tempfile_in(output_dir).with_context(|| { + format!( + "Failed to create a temporary file in {}", + output_dir.display() + ) + })?; + + let mut dest = BufWriter::new(temp_file); + let bytes_written = write_extract(provider, tiles.iter().copied(), &mut dest) + .context("Failed to write the extract")?; + let temp_file = dest + .into_inner() + .map_err(|e| e.into_error()) + .context("Failed to flush the output extract")?; + // Make sure that the data is on disk before the rename makes the file visible. + temp_file + .as_file() + .sync_all() + .context("Failed to sync the output extract to disk")?; + temp_file.persist(output).with_context(|| { + format!( + "Failed to move the output extract into place at {}", + output.display() + ) + })?; + + Ok(bytes_written) +} + +/// Runs the `extract` subcommand, and prints a one-line summary on stderr. +pub(crate) fn run(args: &ExtractArgs) -> anyhow::Result<()> { + let summary = write_sub_extract(args)?; + let noun = if summary.tile_count == 1 { + "tile" + } else { + "tiles" + }; + eprintln!( + "Extracted {} {noun} ({} bytes) to {}", + summary.tile_count, + summary.bytes_written, + args.output.display() + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + /// Path to a fixture in the sibling `valhalla-graphtile` crate. + fn fixture(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("valhalla-graphtile") + .join("fixtures") + .join(name) + } + + fn tile(level: u8, tile_id: u64) -> GraphId { + GraphId::try_from_components(level, tile_id, 0).expect("valid tile ID in test") + } + + // Tragically, memory maps are not supported by Miri! + #[cfg(not(miri))] + #[test] + fn extract_round_trips_from_a_tarball_and_a_directory() { + let temp_dir = tempfile::tempdir().expect("a temporary directory is created"); + let tiles = vec![tile(0, 3015), tile(1, 47_701), tile(2, 763_926)]; + // The arguments name one tile twice, but the extract contains it once. + let mut tile_args = tiles.clone(); + tile_args.push(tile(1, 47_701)); + + let mut outputs = Vec::new(); + for (source, name) in [ + (fixture("andorra-tiles.tar"), "from-tarball.tar"), + (fixture("andorra-tiles"), "from-directory.tar"), + ] { + let output = temp_dir.path().join(name); + let args = ExtractArgs { + source, + output: output.clone(), + tiles: tile_args.clone(), + }; + + let summary = write_sub_extract(&args).expect("the extract writes cleanly"); + assert_eq!( + summary.tile_count, + tiles.len(), + "The summary must count each tile once" + ); + + let on_disk = fs::metadata(&output).expect("the output exists").len(); + assert_eq!( + summary.bytes_written, on_disk, + "The summary must give the size of the output file" + ); + + let extract = TarballTileProvider::new_readonly(&output) + .expect("the generated extract opens cleanly"); + let mut extracted: Vec = extract.tile_ids().copied().collect(); + extracted.sort_unstable(); + let mut expected = tiles.clone(); + expected.sort_unstable(); + assert_eq!( + extracted, expected, + "The extract must contain each requested tile once" + ); + + outputs.push(fs::read(&output).expect("the output is readable")); + } + + assert_eq!( + outputs[0], outputs[1], + "The tarball and directory fixtures must give the same extract" + ); + + let mut names: Vec<_> = fs::read_dir(temp_dir.path()) + .expect("the directory is readable") + .map(|entry| entry.expect("the entry is readable").file_name()) + .collect(); + names.sort_unstable(); + assert_eq!( + names, + ["from-directory.tar", "from-tarball.tar"], + "No temporary files must remain after a successful extract" + ); + } + + #[cfg(not(miri))] + #[test] + fn failed_extract_keeps_the_previous_output() { + let temp_dir = tempfile::tempdir().expect("a temporary directory is created"); + let output = temp_dir.path().join("andorra-sub.tar"); + fs::write(&output, b"previous contents").expect("the previous output is written"); + + let args = ExtractArgs { + source: fixture("andorra-tiles.tar"), + output: output.clone(), + // The fixture has no tile 2/0. + tiles: vec![tile(0, 3015), tile(2, 0)], + }; + + let error = write_sub_extract(&args).expect_err("a missing tile fails the extract"); + assert!( + format!("{error:#}").contains("GraphId 2/0/0 is not in the provider"), + "The error must name the missing tile: {error:#}" + ); + assert_eq!( + fs::read(&output).expect("the previous output still exists"), + b"previous contents", + "A failed extract must not change the previous output" + ); + assert_eq!( + fs::read_dir(temp_dir.path()) + .expect("the directory is readable") + .count(), + 1, + "The temporary file must be removed after a failed extract" + ); + } + + #[cfg(unix)] + #[cfg(not(miri))] + #[test] + fn output_has_the_permissions_of_a_new_file() { + let temp_dir = tempfile::tempdir().expect("a temporary directory is created"); + let output = temp_dir.path().join("andorra-sub.tar"); + let reference = temp_dir.path().join("reference"); + fs::File::create(&reference).expect("the reference file is created"); + + let args = ExtractArgs { + source: fixture("andorra-tiles.tar"), + output: output.clone(), + tiles: vec![tile(0, 3015)], + }; + write_sub_extract(&args).expect("the extract writes cleanly"); + + let mode = |path: &Path| { + fs::metadata(path) + .expect("the file exists") + .permissions() + .mode() + }; + assert_eq!( + mode(&output), + mode(&reference), + "The output must have the permissions of a file from File::create" + ); + } +} diff --git a/valinor-cli/src/commands/get_edge.rs b/valinor-cli/src/commands/get_edge.rs new file mode 100644 index 0000000..22d53fd --- /dev/null +++ b/valinor-cli/src/commands/get_edge.rs @@ -0,0 +1,146 @@ +//! The `get-edge` subcommand: pretty-prints information about a directed edge. + +use std::{ + fs, + path::{Path, PathBuf}, +}; + +use anyhow::{Context, anyhow}; +use clap::Args; +use serde_json::Value as JsonValue; +use tracing::info; +use valhalla_graphtile::tile_provider::TrafficTileProvider; +use valhalla_graphtile::{ + GraphId, + graph_tile::GraphTile, + tile_provider::{DirectoryGraphTileProvider, GraphTileProvider, TarballTileProvider}, +}; + +/// Pretty-print information about a directed edge (incl. live traffic, if available) +#[derive(Args, Debug)] +pub(crate) struct GetEdgeArgs { + /// Path to valhalla.json + #[arg(env)] + valhalla_config: PathBuf, + + /// Graph ID (u64) or slash-form level/tile_id/index + #[arg(value_parser = parse_graph_id)] + pub(crate) graph_id: GraphId, +} + +/// Parses a graph ID from its packed `u64` value or from the `level/tile_id/index` form. +fn parse_graph_id(input: &str) -> anyhow::Result { + if let Ok(id) = input.parse::() { + return Ok(GraphId::try_from_id(id)?); + } + + let parts: Vec<&str> = input.split('/').collect(); + let [level, tile_id, index] = parts[..] else { + return Err(anyhow!( + "Unrecognized graph id format. Use a u64 integer or level/tile_id/index" + )); + }; + let level = level.parse().context("invalid level in graph id")?; + let tile_id = tile_id.parse().context("invalid tile id in graph id")?; + let index = index.parse().context("invalid index in graph id")?; + Ok(GraphId::try_from_components(level, tile_id, index)?) +} + +#[derive(Debug, Clone)] +struct DataSources { + routing_graph: Option, + traffic_extract: Option, +} + +#[derive(Debug, Clone)] +enum RoutingGraphDataSource { + Tarball(PathBuf), + TileDir(PathBuf), +} + +fn parse_valhalla_data_paths(path: &Path) -> anyhow::Result { + let bytes = + fs::read(path).with_context(|| format!("Failed to read config at {}", path.display()))?; + let json: JsonValue = + serde_json::from_slice(&bytes).context("Invalid JSON in valhalla config")?; + + let get_path_if_exists = |key: &str| -> Option { + if let JsonValue::String(s) = &json["mjolnir"][key] + && !s.is_empty() + && fs::exists(s).unwrap_or_default() + { + Some(PathBuf::from(s)) + } else { + None + } + }; + + let tile_extract = get_path_if_exists("tile_extract"); + let tile_dir = get_path_if_exists("tile_dir"); + let traffic_extract = get_path_if_exists("traffic_extract"); + Ok(DataSources { + routing_graph: match (tile_extract, tile_dir) { + (Some(tarball), _) => Some(RoutingGraphDataSource::Tarball(tarball)), + (_, Some(dir)) => Some(RoutingGraphDataSource::TileDir(dir)), + (None, None) => None, + }, + traffic_extract, + }) +} + +fn pretty_print_edge_info( + provider: &T, + traffic_provider: Option<&TrafficTileProvider>, + gid: GraphId, +) -> anyhow::Result<()> { + let output = provider.with_tile_containing(gid, |tile| { + let edge = tile.get_directed_edge(gid)?; + let edge_info = tile.get_edge_info(edge)?; + let traffic_info = traffic_provider.map(|tp| unsafe { tp.get_speeds_for_edge(gid).ok() }); + + Ok::(serde_json::json!({ + "graph_id": gid, + "directed_edge": edge, + "edge_info": edge_info, + "traffic": traffic_info, + })) + })??; + println!("{}", serde_json::to_string_pretty(&output)?); + Ok(()) +} + +/// Runs the `get-edge` subcommand. +pub(crate) fn run(args: &GetEdgeArgs) -> anyhow::Result<()> { + let gid = args.graph_id; + let sources = parse_valhalla_data_paths(&args.valhalla_config)?; + let traffic_extract = if let Some(path) = sources.traffic_extract { + info!(path = path.to_str(), "Using traffic extract"); + Some(TrafficTileProvider::new_readonly(path)?) + } else { + info!("No traffic extract could be found"); + None + }; + + // Prefer tarball if set, fall back to directory. + if let Some(graph) = sources.routing_graph { + match graph { + RoutingGraphDataSource::Tarball(path) => { + info!(path = path.to_str(), "Using tarball tile extract"); + + let provider = TarballTileProvider::new_readonly(&path)?; + pretty_print_edge_info(&provider, traffic_extract.as_ref(), gid) + } + RoutingGraphDataSource::TileDir(path) => { + info!(path = path.to_str(), "Using tile directory"); + + let provider = + DirectoryGraphTileProvider::new(path, std::num::NonZeroUsize::new(1).unwrap()); + pretty_print_edge_info(&provider, traffic_extract.as_ref(), gid) + } + } + } else { + Err(anyhow!( + "No routing graph data sources could be loaded. Expected a valid 'tile_extract' (tarball) or 'tile_dir' in the config." + )) + } +} diff --git a/valinor-cli/src/commands/mod.rs b/valinor-cli/src/commands/mod.rs new file mode 100644 index 0000000..f23f125 --- /dev/null +++ b/valinor-cli/src/commands/mod.rs @@ -0,0 +1,4 @@ +//! The subcommands of the CLI, with one module for each subcommand. + +pub(crate) mod extract; +pub(crate) mod get_edge; diff --git a/valinor-cli/src/main.rs b/valinor-cli/src/main.rs index 17eb3be..62b0957 100644 --- a/valinor-cli/src/main.rs +++ b/valinor-cli/src/main.rs @@ -1,26 +1,15 @@ -use std::{fs, path::PathBuf, str::FromStr}; +mod commands; -use anyhow::{Context, anyhow}; use clap::{Parser, Subcommand}; -use serde_json::Value as JsonValue; -use tracing::info; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::{EnvFilter, Layer}; -use valhalla_graphtile::tile_provider::TrafficTileProvider; -use valhalla_graphtile::{ - GraphId, - graph_tile::GraphTile, - tile_provider::{DirectoryGraphTileProvider, GraphTileProvider, TarballTileProvider}, -}; + +use commands::{extract, get_edge}; #[derive(Parser, Debug)] #[command(name = "valinor-cli", author, version, about, long_about = None)] struct Cli { - /// Path to valhalla.json - #[arg(env)] - valhalla_config: PathBuf, - /// Subcommand/tool to run #[command(subcommand)] command: Commands, @@ -28,94 +17,8 @@ struct Cli { #[derive(Subcommand, Debug)] enum Commands { - /// Pretty-print information about a directed edge (incl. live traffic, if available) - GetEdge { - /// Graph ID (u64) or slash-form level/tile/index - graph_id: String, - }, -} - -fn parse_graph_id(input: &str) -> anyhow::Result { - // Try to parse as a pure integer. - if let Ok(id) = input.parse::() { - return GraphId::try_from_id(id).map_err(|e| anyhow!(e)); - } - - // Try slash-separated level/tile/index - let parts: Vec<&str> = input.split('/').collect(); - if parts.len() == 3 { - let level = u8::from_str(parts[0]).context("invalid level in graph id")?; - let tile_id = u64::from_str(parts[1]).context("invalid tile id in graph id")?; - let index = u64::from_str(parts[2]).context("invalid index in graph id")?; - return GraphId::try_from_components(level, tile_id, index).map_err(|e| anyhow!(e)); - } - - Err(anyhow!( - "Unrecognized graph id format. Use a u64 integer or level/tile/index" - )) -} - -#[derive(Debug, Clone)] -struct DataSources { - routing_graph: Option, - traffic_extract: Option, -} - -#[derive(Debug, Clone)] -enum RoutingGraphDataSource { - Tarball(PathBuf), - TileDir(PathBuf), -} - -fn parse_valhalla_data_paths(path: &PathBuf) -> anyhow::Result { - let bytes = - fs::read(path).with_context(|| format!("Failed to read config at {}", path.display()))?; - let json: JsonValue = - serde_json::from_slice(&bytes).context("Invalid JSON in valhalla config")?; - - let get_path_if_exists = |key: &str| -> Option { - if let JsonValue::String(s) = &json["mjolnir"][key] - && !s.is_empty() - && fs::exists(s).unwrap_or_default() - { - Some(PathBuf::from(s)) - } else { - None - } - }; - - let tile_extract = get_path_if_exists("tile_extract"); - let tile_dir = get_path_if_exists("tile_dir"); - let traffic_extract = get_path_if_exists("traffic_extract"); - Ok(DataSources { - routing_graph: match (tile_extract, tile_dir) { - (Some(tarball), _) => Some(RoutingGraphDataSource::Tarball(tarball)), - (_, Some(dir)) => Some(RoutingGraphDataSource::TileDir(dir)), - (None, None) => None, - }, - traffic_extract, - }) -} - -fn pretty_print_edge_info( - provider: &T, - traffic_provider: Option<&TrafficTileProvider>, - gid: GraphId, -) -> anyhow::Result<()> { - let output = provider.with_tile_containing(gid, |tile| { - let edge = tile.get_directed_edge(gid)?; - let edge_info = tile.get_edge_info(edge)?; - let traffic_info = traffic_provider.map(|tp| unsafe { tp.get_speeds_for_edge(gid).ok() }); - - Ok::(serde_json::json!({ - "graph_id": gid, - "directed_edge": edge, - "edge_info": edge_info, - "traffic": traffic_info, - })) - })??; - println!("{}", serde_json::to_string_pretty(&output)?); - Ok(()) + GetEdge(get_edge::GetEdgeArgs), + Extract(extract::ExtractArgs), } fn main() -> anyhow::Result<()> { @@ -127,41 +30,97 @@ fn main() -> anyhow::Result<()> { let cli = Cli::parse(); match cli.command { - Commands::GetEdge { graph_id } => { - let gid = parse_graph_id(&graph_id)?; - let sources = parse_valhalla_data_paths(&cli.valhalla_config)?; - let traffic_extract = if let Some(path) = sources.traffic_extract { - info!(path = path.to_str(), "Using traffic extract"); - Some(TrafficTileProvider::new_readonly(path)?) - } else { - info!("No traffic extract could be found"); - None - }; + Commands::GetEdge(args) => get_edge::run(&args), + Commands::Extract(args) => extract::run(&args), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use clap::CommandFactory; + use valhalla_graphtile::GraphId; + + fn tile(level: u8, tile_index: u64) -> GraphId { + GraphId::try_from_components(level, tile_index, 0).expect("valid tile ID in test") + } + + #[test] + fn clap_derive_is_consistent() { + Cli::command().debug_assert(); + } - // Prefer tarball if set, fall back to directory. - if let Some(graph) = sources.routing_graph { - match graph { - RoutingGraphDataSource::Tarball(path) => { - info!(path = path.to_str(), "Using tarball tile extract"); + #[test] + fn extract_tiles_accept_repeated_and_comma_separated_options() { + let cli = Cli::try_parse_from([ + "valinor-cli", + "extract", + "in.tar", + "out.tar", + "--tile", + "0/3015,1/47701", + "--tile", + "2/763926", + ]) + .expect("the arguments parse"); + let Commands::Extract(args) = cli.command else { + panic!("expected the extract subcommand"); + }; + assert_eq!( + args.tiles, + vec![tile(0, 3015), tile(1, 47_701), tile(2, 763_926)] + ); + } - let provider = TarballTileProvider::new_readonly(&path)?; - pretty_print_edge_info(&provider, traffic_extract.as_ref(), gid) - } - RoutingGraphDataSource::TileDir(path) => { - info!(path = path.to_str(), "Using tile directory"); + #[test] + fn extract_tiles_reject_malformed_values() { + for input in [ + "2", + "2/763926/7", + "2/abc", + "x/0", + "8/0", + "256/0", + "0/4194304", + ] { + let error = Cli::try_parse_from([ + "valinor-cli", + "extract", + "in.tar", + "out.tar", + "--tile", + input, + ]) + .expect_err("a malformed tile is rejected"); + assert!(error.to_string().contains(input), "{input}: {error}"); + } + } - let provider = DirectoryGraphTileProvider::new( - path, - std::num::NonZeroUsize::new(1).unwrap(), - ); - pretty_print_edge_info(&provider, traffic_extract.as_ref(), gid) - } - } - } else { - Err(anyhow!( - "No routing graph data sources could be loaded. Expected a valid 'tile_extract' (tarball) or 'tile_dir' in the config." - )) - } + #[test] + fn get_edge_accepts_both_graph_id_forms() { + let expected = GraphId::try_from_components(2, 762_485, 42).expect("valid graph ID"); + for input in [expected.value().to_string(), "2/762485/42".to_string()] { + let cli = Cli::try_parse_from(["valinor-cli", "get-edge", "valhalla.json", &input]) + .expect("the arguments parse"); + let Commands::GetEdge(args) = cli.command else { + panic!("expected the get-edge subcommand"); + }; + assert_eq!(args.graph_id, expected); } } + + #[test] + fn get_edge_rejects_malformed_graph_ids() { + // The last value has a level of 8, which does not fit in the 3-bit level field. + for input in ["2/762485", "2/762485/42/0", "2/762485/x", "x/0/0", "8/0/0"] { + let error = Cli::try_parse_from(["valinor-cli", "get-edge", "valhalla.json", input]) + .expect_err("a malformed graph ID is rejected"); + assert!(error.to_string().contains(input), "{input}: {error}"); + } + } + + #[test] + fn extract_tiles_are_required() { + assert!(Cli::try_parse_from(["valinor-cli", "extract", "in.tar", "out.tar"]).is_err()); + } }