Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions lightpanda/browser.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,10 @@ class PageVerdict:

Questions passed to :meth:`Session.judge` are asked in the same call, and
their answers land in :attr:`answers`.

A page that answered 4xx or 5xx is judged by its status alone, without
asking the model: :attr:`http_status` holds it, the check it names is
true, and every score and answer is None.
"""

is_blocked: bool
Expand Down Expand Up @@ -112,13 +116,25 @@ class PageVerdict:
"""The answers to the questions passed to :meth:`Session.judge`, keyed as
asked and shaped as :meth:`Session.classify` returns them. Empty when none
were passed."""
http_status: int | None
"""The 4xx or 5xx status that decided the verdict without the model, or
None when the model was asked."""

def __init__(self, answers: dict, threshold: float, asked: Iterable[str] = ()):
asked = set(asked)
self.answers = {key: answers.get(key) for key in asked}
self.scores = {_snake(preset): score for preset, score in answers.items() if preset not in asked}
for attr, score in self.scores.items():
setattr(self, attr, score is not None and score >= threshold)
self.http_status = None

@classmethod
def _from_status(cls, http_status: int, asked: Iterable[str]) -> PageVerdict:
checks = [attr for attr, kind in cls.__annotations__.items() if kind == "bool"]
verdict = cls(dict.fromkeys([*checks, *asked]), 0.5, asked)
verdict.http_status = http_status
setattr(verdict, _STATUS_CHECKS.get(http_status, "is_error_page"), True)
return verdict

@property
def ok(self) -> bool:
Expand All @@ -137,6 +153,12 @@ def __repr__(self) -> str:
return f"PageVerdict({fired})"


# Of 1,781 pages that answered 4xx or 5xx in the classifier's crawl, the model
# found none worth reading, so judge() skips them without asking it. The
# statuses that say why name their check; any other names an error page.
_STATUS_CHECKS = {401: "is_login_wall", 402: "is_paywall", 403: "is_blocked", 429: "is_blocked"}


def _snake(name: str) -> str:
return re.sub(r"(?<!^)(?=[A-Z])", "_", name).lower()

Expand Down Expand Up @@ -220,6 +242,8 @@ def __init__(self, browser: Browser, session_id: str):
self._snake_map = browser._snake_map
self._id = session_id
self._closed = False
# The status of the page the last navigation or action left us on.
self._http_status: int | None = None

self._client.request(
"initialize",
Expand Down Expand Up @@ -283,6 +307,7 @@ def call(self, tool: str, **kwargs):
# older binaries send only the sentence, which still falls through.
page_state = result.get("structuredContent")
if page_state is not None:
self._http_status = page_state.get("httpStatus")
return PageResult(text, page_state)
try:
parsed = json.loads(text)
Expand All @@ -307,10 +332,16 @@ def judge(self, *, questions: dict | None = None, threshold: float = 0.5) -> Pag
print(verdict.answers["has_price"]) # 0.99
```

A page that answered 4xx or 5xx is not worth reading whatever it says,
so its verdict comes from the status and the model is not asked; see
:attr:`PageVerdict.http_status`.

Args:
questions: Questions in the object form :meth:`classify` takes.
threshold: The probability at which a check counts as true.
"""
if self._http_status is not None and self._http_status >= 400:
return PageVerdict._from_status(self._http_status, asked=questions or ())
if not questions:
# An empty questions object asks the browser for every preset.
return PageVerdict(self.call("classify", questions={}), threshold)
Expand Down
17 changes: 17 additions & 0 deletions tests/test_classify.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,23 @@ def test_judge_with_questions(classify_browser, fixture_url):
assert verdict.is_login_wall and not verdict.ok


def test_judge_error_status_skips_the_model(classify_browser, fixture_url):
with classify_browser.new_session() as page:
page.goto(url=f"{fixture_url}/missing.html")
verdict = page.judge(questions={"has_price": "Does the page show a price?"})
assert verdict.http_status == 404
assert verdict.is_error_page and not verdict.is_paywall and not verdict.ok
assert verdict.reason == "is_error_page"
# The mock answers 0.9 to everything, so None means it was never asked.
assert set(verdict.scores.values()) == {None}
assert verdict.answers == {"has_price": None}

page.goto(url=f"{fixture_url}/index.html")
verdict = page.judge()
assert verdict.http_status is None
assert set(verdict.scores.values()) == {0.9}


async def test_judge_async(classify_browser, fixture_url):
from lightpanda import AsyncBrowser

Expand Down
Loading