diff --git a/cachecontrol/adapter.py b/cachecontrol/adapter.py index d23d74a6..46be6f3a 100644 --- a/cachecontrol/adapter.py +++ b/cachecontrol/adapter.py @@ -4,6 +4,7 @@ from __future__ import annotations import functools +import time import weakref import zlib from typing import TYPE_CHECKING, Any, Collection, Mapping @@ -73,6 +74,8 @@ def send( # check for etags and add headers if appropriate request.headers.update(self.controller.conditional_headers(request)) + setattr(request, "_cachecontrol_request_time", time.time()) + setattr(request, "_cachecontrol_request_monotonic", time.monotonic()) resp = super().send(request, stream, timeout, verify, cert, proxies) return resp @@ -92,6 +95,21 @@ def build_response( # type: ignore[override] """ cacheable = cacheable_methods or self.cacheable_methods if not from_cache and request.method in cacheable: + response_time = time.time() + request_time = getattr(request, "_cachecontrol_request_time", None) + request_monotonic = getattr( + request, "_cachecontrol_request_monotonic", None + ) + if isinstance(request_monotonic, (int, float)): + response_delay = max(0.0, time.monotonic() - request_monotonic) + elif isinstance(request_time, (int, float)): + response_delay = max(0.0, response_time - request_time) + else: + response_delay = 0.0 + + setattr(response, "_cachecontrol_response_time", response_time) + setattr(response, "_cachecontrol_response_delay", response_delay) + # Check for any heuristics that might update headers # before trying to cache. if self.heuristic: diff --git a/cachecontrol/controller.py b/cachecontrol/controller.py index 5eca8de7..3ad91352 100644 --- a/cachecontrol/controller.py +++ b/cachecontrol/controller.py @@ -10,11 +10,12 @@ import calendar import logging +import math import re import time import weakref from email.utils import parsedate_tz -from typing import TYPE_CHECKING, Collection, Mapping +from typing import TYPE_CHECKING, Collection, Mapping, TypeGuard from requests.structures import CaseInsensitiveDict @@ -34,6 +35,8 @@ URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?") PERMANENT_REDIRECT_STATUSES = (301, 308) +MAX_AGE_VALUE = 2**31 +AGE_VALUE = re.compile(r"^[0-9]+$") def parse_uri(uri: str) -> tuple[str, str, str, str, str]: @@ -140,6 +143,107 @@ def parse_cache_control(self, headers: Mapping[str, str]) -> dict[str, int | Non return retval + @staticmethod + def _parse_age(headers: Mapping[str, str]) -> tuple[int, bool, bool]: + """Return the first valid Age value, saturating values per RFC 9111.""" + value = headers.get("age", headers.get("Age")) + if value is None: + return 0, False, False + + first_value = value.split(",", 1)[0].strip() + if not AGE_VALUE.fullmatch(first_value): + return 0, False, False + + normalized_age = first_value.lstrip("0") or "0" + max_age = str(MAX_AGE_VALUE) + if len(normalized_age) > len(max_age) or ( + len(normalized_age) == len(max_age) and normalized_age >= max_age + ): + # RFC 9111 section 1.2.2 defines 2**31 as the overflow/infinity + # sentinel. Compare bounded digit strings first so very large + # values saturate without relying on Python's configurable int limit. + return MAX_AGE_VALUE, True, True + return int(normalized_age), True, False + + @staticmethod + def _number(value: object) -> TypeGuard[int | float]: + return ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + ) + + @classmethod + def _age_metadata(cls, response: HTTPResponse) -> tuple[float, float, bool] | None: + initial_age = getattr(response, "_cachecontrol_initial_age", None) + response_time = getattr(response, "_cachecontrol_response_time", None) + if cls._number(initial_age) and cls._number(response_time): + age_overflow = getattr(response, "_cachecontrol_age_overflow", False) + _, has_valid_age, header_overflow = cls._parse_age(response.headers) + return ( + float(initial_age), + float(response_time), + (age_overflow is True) + or float(initial_age) >= MAX_AGE_VALUE + or (has_valid_age and header_overflow), + ) + return None + + @classmethod + def _initial_age( + cls, + date_headers: Mapping[str, str], + response_time: float, + response_delay: float, + age_headers: Mapping[str, str] | None = None, + ) -> tuple[float, bool]: + date_value = date_headers.get("date", date_headers.get("Date")) + parsed_date = parsedate_tz(date_value) if date_value is not None else None + apparent_age = 0.0 + if parsed_date is not None: + date = calendar.timegm(parsed_date[:6]) + apparent_age = max(0.0, response_time - date) + + age_value, _, age_overflow = cls._parse_age( + date_headers if age_headers is None else age_headers + ) + corrected_age_value = age_value + max(0.0, response_delay) + initial_age = max(apparent_age, corrected_age_value) + return initial_age, age_overflow or initial_age >= MAX_AGE_VALUE + + @staticmethod + def _format_age(age: float, age_overflow: bool = False) -> str: + if age_overflow or not math.isfinite(age) or age >= MAX_AGE_VALUE: + return str(MAX_AGE_VALUE) + return str(max(0, int(age))) + + def _ensure_age_metadata(self, response: HTTPResponse) -> None: + if self._age_metadata(response) is not None: + return + + response_time = getattr(response, "_cachecontrol_response_time", None) + if not self._number(response_time): + response_time = time.time() + + response_delay = getattr(response, "_cachecontrol_response_delay", None) + if not self._number(response_delay): + response_delay = 0.0 + request_time = getattr(response, "_cachecontrol_request_time", None) + if self._number(request_time): + response_delay = max(0.0, response_time - request_time) + + initial_age, age_overflow = self._initial_age( + response.headers, float(response_time), float(response_delay) + ) + try: + setattr(response, "_cachecontrol_initial_age", initial_age) + setattr(response, "_cachecontrol_response_time", float(response_time)) + setattr(response, "_cachecontrol_age_overflow", age_overflow) + except (AttributeError, TypeError): + # Some custom response classes cannot carry private attributes. + # Their custom serializer retains its existing behavior. + pass + def _load_from_cache(self, request: PreparedRequest) -> HTTPResponse | None: """ Load a cached response, or return None if it's not available. @@ -211,8 +315,28 @@ def cached_request(self, request: PreparedRequest) -> HTTPResponse | Literal[Fal logger.debug("Ignoring cached response: invalid date") return False date = calendar.timegm(time_tuple[:6]) - current_age = max(0, now - date) - logger.debug("Current age based on date: %i", current_age) + age_metadata = self._age_metadata(resp) + if age_metadata is not None: + initial_age, response_time, age_overflow = age_metadata + if age_overflow: + current_age = math.inf + else: + resident_age = initial_age + max(0.0, now - response_time) + current_age = max(resident_age, max(0.0, now - date)) + if current_age >= MAX_AGE_VALUE: + current_age = math.inf + logger.debug("Current age from stored timing metadata: %s", current_age) + else: + age_value, has_valid_age, _ = self._parse_age(headers) + if has_valid_age: + # A legacy/custom serializer did not preserve the timing + # needed to advance a received Age value. Revalidate rather + # than allowing that value to remain frozen across lookups. + current_age = math.inf + logger.debug("Ignoring cached response with Age but no timing metadata") + else: + current_age = max(0, now - date) + logger.debug("Current age based on date: %s", current_age) # TODO: There is an assumption that the result will be a # urllib3 response object. This may not be best since we @@ -240,8 +364,11 @@ def cached_request(self, request: PreparedRequest) -> HTTPResponse | Literal[Fal # Permanent redirects are heuristically cacheable when the response # does not provide an explicit freshness lifetime. elif int(resp.status) in PERMANENT_REDIRECT_STATUSES: - logger.debug("Returning cached permanent redirect response") - return resp + if math.isfinite(current_age): + headers["Age"] = self._format_age(current_age) + resp.headers["Age"] = headers["Age"] + logger.debug("Returning cached permanent redirect response") + return resp # Determine if we are setting freshness limit in the # request. Note, this overrides what was in the response. @@ -252,16 +379,19 @@ def cached_request(self, request: PreparedRequest) -> HTTPResponse | Literal[Fal "Freshness lifetime from request max-age: %i", freshness_lifetime ) + freshness_check_age = current_age min_fresh = cc.get("min-fresh") if min_fresh is not None: - # adjust our current age by our min fresh - current_age += min_fresh - logger.debug("Adjusted current age from min-fresh: %i", current_age) + # min-fresh affects the freshness check, not the response's Age. + freshness_check_age += min_fresh + logger.debug("Adjusted current age from min-fresh: %s", freshness_check_age) # Return entry if it is fresh enough - if freshness_lifetime > current_age: + if freshness_lifetime > freshness_check_age: logger.debug('The response is "fresh", returning cached response') - logger.debug("%i > %i", freshness_lifetime, current_age) + logger.debug("%s > %s", freshness_lifetime, freshness_check_age) + headers["Age"] = self._format_age(current_age) + resp.headers["Age"] = headers["Age"] return resp # we're not fresh. If we don't have an Etag, clear it out @@ -295,14 +425,15 @@ def _cache_set( body: bytes | None = None, expires_time: int | None = None, ) -> None: - """Store the data in the cache. - """ + """Store the data in the cache.""" if expires_time is not None and expires_time <= 0: # Already stale on arrival logger.debug("Purging cached response: expires in the past") self.cache.delete(cache_url) return + self._ensure_age_metadata(response) + if isinstance(self.cache, SeparateBodyBaseCache): # We pass in the body separately; just put a placeholder empty # string in the metadata. @@ -510,6 +641,29 @@ def update_cached_response( # we want a 200 b/c we have content via the cache cached_response.status = 200 + response_time = getattr(response, "_cachecontrol_response_time", None) + if not self._number(response_time): + response_time = time.time() + response_delay = getattr(response, "_cachecontrol_response_delay", None) + if not self._number(response_delay): + response_delay = 0.0 + + initial_age, age_overflow = self._initial_age( + cached_response.headers, + float(response_time), + float(response_delay), + age_headers=response.headers, + ) + try: + setattr(cached_response, "_cachecontrol_initial_age", initial_age) + setattr( + cached_response, "_cachecontrol_response_time", float(response_time) + ) + setattr(cached_response, "_cachecontrol_age_overflow", age_overflow) + except (AttributeError, TypeError): + pass + cached_response.headers["Age"] = self._format_age(initial_age, age_overflow) + # update our cache self._cache_set(cache_url, request, cached_response) diff --git a/cachecontrol/serialize.py b/cachecontrol/serialize.py index 83bce073..af7df912 100644 --- a/cachecontrol/serialize.py +++ b/cachecontrol/serialize.py @@ -4,6 +4,7 @@ from __future__ import annotations import io +import math from typing import IO, TYPE_CHECKING, Any, Mapping, cast import msgpack @@ -46,6 +47,28 @@ def dumps( } } + initial_age = getattr(response, "_cachecontrol_initial_age", None) + response_time = getattr(response, "_cachecontrol_response_time", None) + if ( + isinstance(initial_age, (int, float)) + and not isinstance(initial_age, bool) + and math.isfinite(initial_age) + and initial_age >= 0 + and isinstance(response_time, (int, float)) + and not isinstance(response_time, bool) + and math.isfinite(response_time) + and response_time >= 0 + ): + # This optional top-level field is additive to cc=4: existing + # readers ignore unknown keys, and new readers accept old records + # where timing metadata is absent. + data["cachecontrol_age"] = { + "initial_age": float(initial_age), + "response_time": float(response_time), + } + if getattr(response, "_cachecontrol_age_overflow", False) is True: + data["cachecontrol_age"]["overflow"] = True + # Construct our vary headers data["vary"] = {} if "vary" in response_headers: @@ -130,7 +153,29 @@ def prepare_response( # Discard any `strict` parameter serialized by older version of cachecontrol. cached["response"].pop("strict", None) - return HTTPResponse(body=body, preload_content=False, **cached["response"]) + response = HTTPResponse(body=body, preload_content=False, **cached["response"]) + age_metadata = cached.get("cachecontrol_age") + if isinstance(age_metadata, Mapping): + initial_age = age_metadata.get("initial_age") + response_time = age_metadata.get("response_time") + age_overflow = age_metadata.get("overflow", False) + if ( + isinstance(initial_age, (int, float)) + and not isinstance(initial_age, bool) + and math.isfinite(initial_age) + and initial_age >= 0 + and isinstance(response_time, (int, float)) + and not isinstance(response_time, bool) + and math.isfinite(response_time) + and response_time >= 0 + and isinstance(age_overflow, bool) + ): + setattr(response, "_cachecontrol_initial_age", float(initial_age)) + setattr(response, "_cachecontrol_response_time", float(response_time)) + if age_overflow: + setattr(response, "_cachecontrol_age_overflow", True) + + return response def _loads_v4( self, diff --git a/tests/test_cache_control.py b/tests/test_cache_control.py index 6e3f21bd..13ea6995 100644 --- a/tests/test_cache_control.py +++ b/tests/test_cache_control.py @@ -6,15 +6,18 @@ Unit tests that verify our caching methods work correctly. """ +import logging import os import time from unittest.mock import ANY, Mock import pytest +from urllib3 import HTTPResponse from cachecontrol import CacheController from cachecontrol.cache import DictCache from cachecontrol.caches import SeparateBodyFileCache +from cachecontrol.serialize import Serializer from .utils import DummyRequest, DummyResponse, NullSerializer @@ -345,6 +348,253 @@ def test_cache_request_fresh_max_age(self): r = self.req({}) assert r == resp + def test_cache_request_stale_age_header_not_returned(self, monkeypatch): + now = 1_700_000_020 + monkeypatch.setattr(time, "time", lambda: now) + date = time.strftime(TIME_FMT, time.gmtime(now - 5)) + # RFC 9111 section 4.2.3: corrected age must account for Age as well + # as apparent age from Date when the two estimates differ. + resp = Mock( + headers={"cache-control": "max-age=60", "date": date, "age": "120"}, + status=200, + ) + self.c.cache = DictCache({self.url: resp}) + + assert self.req({}) is False + + def test_legacy_age_with_min_fresh_logs_without_integer_formatting(self, caplog): + now = 1_700_000_020 + date = time.strftime(TIME_FMT, time.gmtime(now - 5)) + resp = Mock( + headers={"cache-control": "max-age=60", "date": date, "age": "120"}, + status=200, + ) + self.c.cache = DictCache({self.url: resp}) + + with caplog.at_level(logging.DEBUG, logger="cachecontrol.controller"): + assert not self.req({"cache-control": "min-fresh=1"}) + + adjusted_age = next( + record + for record in caplog.records + if record.msg == "Adjusted current age from min-fresh: %s" + ) + assert adjusted_age.getMessage().endswith("inf") + + def test_cached_age_advances_after_storage_and_between_hits(self, monkeypatch): + response_time = 1_700_000_000.0 + monkeypatch.setattr(time, "time", lambda: response_time) + future_date = time.strftime(TIME_FMT, time.gmtime(response_time + 60)) + response = HTTPResponse( + body=b"body", + status=200, + headers={ + "Cache-Control": "max-age=30", + "Date": future_date, + "Age": "10", + }, + ) + response._cachecontrol_response_time = response_time + response._cachecontrol_response_delay = 2.5 + request = DummyRequest(self.url, {}) + cache = DictCache() + controller = CacheController(cache, serializer=Serializer()) + controller._cache_set(self.url, request, response, b"body") + + monkeypatch.setattr(time, "time", lambda: response_time + 7) + first = controller.cached_request(request) + assert first is not False + assert first.headers["Age"] == "19" + + monkeypatch.setattr(time, "time", lambda: response_time + 12) + second = controller.cached_request(request) + assert second is not False + assert second.headers["Age"] == "24" + + def test_min_fresh_does_not_change_emitted_age(self, monkeypatch): + now = 1_700_000_020.0 + monkeypatch.setattr(time, "time", lambda: now) + date = time.strftime(TIME_FMT, time.gmtime(now)) + request = DummyRequest(self.url, {"Cache-Control": "min-fresh=5"}) + response = HTTPResponse( + body=b"body", + status=200, + headers={ + "Cache-Control": "max-age=20", + "Date": date, + "Age": "10", + }, + ) + cache = DictCache() + controller = CacheController(cache, serializer=Serializer()) + controller._cache_set(self.url, request, response, b"body") + + cached = controller.cached_request(request) + + assert cached is not False + assert cached.headers["Age"] == "10" + + @pytest.mark.parametrize( + ("age", "expected_age", "valid", "overflow"), + [ + ("12, 99", 12, True, False), + ("-1", 0, False, False), + ("invalid", 0, False, False), + (str(2**31 - 1), 2**31 - 1, True, False), + (str(2**31), 2**31, True, True), + (str(2**40), 2**31, True, True), + ("0" * 5000, 0, True, False), + ("0" * 5000 + "7", 7, True, False), + ], + ) + def test_parse_age_header(self, age, expected_age, valid, overflow): + assert CacheController._parse_age({"Age": age}) == ( + expected_age, + valid, + overflow, + ) + + def test_oversized_age_cannot_be_fresh_against_larger_max_age(self, monkeypatch): + now = 1_700_000_020.0 + monkeypatch.setattr(time, "time", lambda: now) + date = time.strftime(TIME_FMT, time.gmtime(now)) + request = DummyRequest(self.url, {}) + response = HTTPResponse( + body=b"body", + status=200, + headers={ + "Cache-Control": "max-age=4000000000", + "Date": date, + "Age": "6000000000", + "ETag": '"v1"', + }, + ) + cache = DictCache() + controller = CacheController(cache, serializer=Serializer()) + controller._cache_set(self.url, request, response, b"body") + + assert controller.cached_request(request) is False + + def test_saturated_age_cannot_use_permanent_redirect_fallback(self, monkeypatch): + now = 1_700_000_020.0 + monkeypatch.setattr(time, "time", lambda: now) + date = time.strftime(TIME_FMT, time.gmtime(now)) + request = DummyRequest(self.url, {}) + response = HTTPResponse( + body=b"body", + status=301, + headers={ + "Date": date, + "Age": str(2**31), + "ETag": '"v1"', + }, + ) + cache = DictCache() + controller = CacheController(cache, serializer=Serializer()) + controller._cache_set(self.url, request, response, b"body") + + assert controller.cached_request(request) is False + + def test_computed_age_saturates_to_infinity_marker(self): + now = 1_700_000_020.0 + initial_age, overflow = CacheController._initial_age( + {}, + now, + 1.0, + age_headers={"Age": str(2**31 - 1)}, + ) + + assert initial_age == 2**31 + assert overflow is True + + @pytest.mark.parametrize( + ("validation_age", "validation_delay", "date_offset", "expected_age"), + [(None, 3.0, 0, 3), ("20", 2.0, 3, 22)], + ) + def test_304_refreshes_age_from_validation_response( + self, + monkeypatch, + validation_age, + validation_delay, + date_offset, + expected_age, + ): + now = 1_700_000_020.0 + monkeypatch.setattr(time, "time", lambda: now) + date = time.strftime(TIME_FMT, time.gmtime(now)) + validation_date = time.strftime(TIME_FMT, time.gmtime(now - date_offset)) + request = DummyRequest(self.url, {"If-None-Match": '"v1"'}) + cache = DictCache() + controller = CacheController(cache, serializer=Serializer()) + original = HTTPResponse( + body=b"body", + status=200, + headers={ + "Cache-Control": "max-age=60", + "Date": date, + "ETag": '"v1"', + "Age": "800", + }, + ) + controller._cache_set(self.url, request, original, b"body") + + validation_headers = { + "Cache-Control": "max-age=60", + "Date": validation_date, + "ETag": '"v1"', + } + if validation_age is not None: + validation_headers["Age"] = validation_age + not_modified = HTTPResponse( + body=b"", + status=304, + headers=validation_headers, + ) + not_modified._cachecontrol_response_time = now + not_modified._cachecontrol_response_delay = validation_delay + refreshed = controller.update_cached_response(request, not_modified) + assert refreshed.headers["Age"] == str(expected_age) + + monkeypatch.setattr(time, "time", lambda: now + 4) + cached = controller.cached_request(request) + assert cached is not False + assert cached.headers["Age"] == str(expected_age + 4) + assert cached.read() == b"body" + + def test_304_saturated_age_is_emitted_and_remains_stale(self, monkeypatch): + now = 1_700_000_020.0 + monkeypatch.setattr(time, "time", lambda: now) + date = time.strftime(TIME_FMT, time.gmtime(now)) + request = DummyRequest(self.url, {"If-None-Match": '"v1"'}) + cache = DictCache() + controller = CacheController(cache, serializer=Serializer()) + original = HTTPResponse( + body=b"body", + status=200, + headers={ + "Cache-Control": "max-age=4000000000", + "Date": date, + "ETag": '"v1"', + }, + ) + controller._cache_set(self.url, request, original, b"body") + + not_modified = HTTPResponse( + body=b"", + status=304, + headers={ + "Cache-Control": "max-age=4000000000", + "Date": date, + "Age": "6000000000", + }, + ) + not_modified._cachecontrol_response_time = now + refreshed = controller.update_cached_response(request, not_modified) + assert refreshed.headers["Age"] == str(2**31) + + monkeypatch.setattr(time, "time", lambda: now + 1) + assert controller.cached_request(request) is False + def test_cache_request_unfresh_max_age(self): earlier = time.time() - 3700 # epoch - 1h01m40s now = time.strftime(TIME_FMT, time.gmtime(earlier)) diff --git a/tests/test_serialization.py b/tests/test_serialization.py index eab1c13f..caea49f7 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -6,6 +6,7 @@ import msgpack import requests +from urllib3 import HTTPResponse from cachecontrol.serialize import Serializer @@ -60,6 +61,48 @@ def test_read_version_v4(self): resp = self.serializer._loads_v4(req, msgpack.dumps(self.response_data)) # We have to decode our urllib3 data back into a unicode string. assert resp.data == b"Hello World" + assert not hasattr(resp, "_cachecontrol_initial_age") + + def test_age_timing_metadata_round_trips_as_optional_v4_field(self): + req = requests.Request("GET", "https://example.test/").prepare() + response = HTTPResponse( + body=b"Hello World", + headers={"Cache-Control": "max-age=60"}, + status=200, + ) + response._cachecontrol_initial_age = 12.5 + response._cachecontrol_response_time = 1234.5 + + serialized = self.serializer.dumps(req, response, b"Hello World") + payload = msgpack.loads(serialized[5:], raw=False) + assert payload["cachecontrol_age"] == { + "initial_age": 12.5, + "response_time": 1234.5, + } + + restored = self.serializer.loads(req, serialized) + assert restored is not None + assert restored._cachecontrol_initial_age == 12.5 + assert restored._cachecontrol_response_time == 1234.5 + + def test_age_overflow_metadata_round_trips_as_optional_v4_field(self): + req = requests.Request("GET", "https://example.test/").prepare() + response = HTTPResponse( + body=b"Hello World", + headers={"Cache-Control": "max-age=60"}, + status=200, + ) + response._cachecontrol_initial_age = 12.5 + response._cachecontrol_response_time = 1234.5 + response._cachecontrol_age_overflow = True + + serialized = self.serializer.dumps(req, response, b"Hello World") + payload = msgpack.loads(serialized[5:], raw=False) + assert payload["cachecontrol_age"]["overflow"] is True + + restored = self.serializer.loads(req, serialized) + assert restored is not None + assert restored._cachecontrol_age_overflow is True def test_read_latest_version_streamable(self, url): original_resp = requests.get(url, stream=True) diff --git a/tests/test_vary.py b/tests/test_vary.py index c8e0cecc..8214d154 100644 --- a/tests/test_vary.py +++ b/tests/test_vary.py @@ -26,13 +26,22 @@ def cached_equal(self, cached, resp): if "chunked" in resp.raw.headers.get("transfer-encoding", ""): resp.raw.headers.pop("transfer-encoding") + cached_headers = {key.lower(): value for key, value in cached.headers.items()} + response_headers = { + key.lower(): value for key, value in resp.raw.headers.items() + } + response_age = response_headers.pop("age", None) + cached_headers.pop("age", None) + checks = [ cached._fp.getvalue() == resp.content, - cached.headers == resp.raw.headers, + cached_headers == response_headers, cached.status == resp.raw.status, cached.version == resp.raw.version, cached.reason == resp.raw.reason, cached.decode_content == resp.raw.decode_content, + not resp.from_cache + or (response_age is not None and response_age.isdigit()), ] print(checks)