diff --git a/lightpanda/browser.py b/lightpanda/browser.py index d7b15d2..61ae108 100644 --- a/lightpanda/browser.py +++ b/lightpanda/browser.py @@ -84,6 +84,10 @@ class PageVerdict: Questions passed to :meth:`Session.judge` are asked in the same call, and their answers land in :attr:`answers`. + + A page that answered 4xx or 5xx is judged by its status alone, without + asking the model: :attr:`http_status` holds it, the check it names is + true, and every score and answer is None. """ is_blocked: bool @@ -112,6 +116,9 @@ class PageVerdict: """The answers to the questions passed to :meth:`Session.judge`, keyed as asked and shaped as :meth:`Session.classify` returns them. Empty when none were passed.""" + http_status: int | None + """The 4xx or 5xx status that decided the verdict without the model, or + None when the model was asked.""" def __init__(self, answers: dict, threshold: float, asked: Iterable[str] = ()): asked = set(asked) @@ -119,6 +126,15 @@ def __init__(self, answers: dict, threshold: float, asked: Iterable[str] = ()): self.scores = {_snake(preset): score for preset, score in answers.items() if preset not in asked} for attr, score in self.scores.items(): setattr(self, attr, score is not None and score >= threshold) + self.http_status = None + + @classmethod + def _from_status(cls, http_status: int, asked: Iterable[str]) -> PageVerdict: + checks = [attr for attr, kind in cls.__annotations__.items() if kind == "bool"] + verdict = cls(dict.fromkeys([*checks, *asked]), 0.5, asked) + verdict.http_status = http_status + setattr(verdict, _STATUS_CHECKS.get(http_status, "is_error_page"), True) + return verdict @property def ok(self) -> bool: @@ -137,6 +153,12 @@ def __repr__(self) -> str: return f"PageVerdict({fired})" +# Of 1,781 pages that answered 4xx or 5xx in the classifier's crawl, the model +# found none worth reading, so judge() skips them without asking it. The +# statuses that say why name their check; any other names an error page. +_STATUS_CHECKS = {401: "is_login_wall", 402: "is_paywall", 403: "is_blocked", 429: "is_blocked"} + + def _snake(name: str) -> str: return re.sub(r"(? Pag print(verdict.answers["has_price"]) # 0.99 ``` + A page that answered 4xx or 5xx is not worth reading whatever it says, + so its verdict comes from the status and the model is not asked; see + :attr:`PageVerdict.http_status`. + Args: questions: Questions in the object form :meth:`classify` takes. threshold: The probability at which a check counts as true. """ + if self._http_status is not None and self._http_status >= 400: + return PageVerdict._from_status(self._http_status, asked=questions or ()) if not questions: # An empty questions object asks the browser for every preset. return PageVerdict(self.call("classify", questions={}), threshold) diff --git a/tests/test_classify.py b/tests/test_classify.py index 4bd815a..37827a0 100644 --- a/tests/test_classify.py +++ b/tests/test_classify.py @@ -124,6 +124,23 @@ def test_judge_with_questions(classify_browser, fixture_url): assert verdict.is_login_wall and not verdict.ok +def test_judge_error_status_skips_the_model(classify_browser, fixture_url): + with classify_browser.new_session() as page: + page.goto(url=f"{fixture_url}/missing.html") + verdict = page.judge(questions={"has_price": "Does the page show a price?"}) + assert verdict.http_status == 404 + assert verdict.is_error_page and not verdict.is_paywall and not verdict.ok + assert verdict.reason == "is_error_page" + # The mock answers 0.9 to everything, so None means it was never asked. + assert set(verdict.scores.values()) == {None} + assert verdict.answers == {"has_price": None} + + page.goto(url=f"{fixture_url}/index.html") + verdict = page.judge() + assert verdict.http_status is None + assert set(verdict.scores.values()) == {0.9} + + async def test_judge_async(classify_browser, fixture_url): from lightpanda import AsyncBrowser