Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 3 additions & 5 deletions examples/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -270,19 +270,17 @@ The same `judge()` call over 30 real pages, six at a time in one browser. The
verdict decides what the crawler does next: back off when the site says we are
going too fast, accept a cookie wall, wait for a page that is still loading,
and then look again; or keep the page, or skip it and say why. "Too fast" is
our own question, asked in the same call as the wall checks:
one of the checks, set by a 429 or by a page that says so in words; the kind of
page is our own question, asked in the same call:

```python
QUESTIONS = {
"rate_limited": "Does the page say we sent too many requests and should slow down or retry later?",
"kind": {"question": "What kind of page is this?",
"options": ["product", "listing", "article", "docs", "search", "other"]},
}

verdict = await page.judge(questions=QUESTIONS)
if verdict.http_status is not None: # 4xx or 5xx: judged by status, no answers
...
elif verdict.answers["rate_limited"] >= 0.5:
if verdict.is_rate_limited:
await asyncio.sleep(3) # and load it again
elif verdict.is_consent_wall:
found = await page.find_element(role="button", name="/^(accept|aceptar|akzeptieren)/i")
Expand Down
16 changes: 4 additions & 12 deletions examples/triage_crawl.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,6 @@
]

QUESTIONS = {
"rate_limited": "Does the page say we sent too many requests and should slow down or retry later?",
"kind": {
"question": "What kind of page is this?",
"options": ["product", "listing", "article", "docs", "search", "other"],
Expand Down Expand Up @@ -90,16 +89,9 @@ async def judge(page):
return verdict, decision


def rate_limited(verdict):
# A 4xx or 5xx page is judged by its status alone, so our question has no answer.
if verdict.http_status is not None:
return verdict.http_status == 429
return verdict.answers["rate_limited"] >= 0.5


def describe(verdict):
"""The page's kind when it is worth reading, else why not."""
if rate_limited(verdict):
if verdict.is_rate_limited: # name it even when a block fired beside it
return "rate limited"
if verdict.ok:
return verdict.answers["kind"]["choice"]
Expand All @@ -117,7 +109,7 @@ def show(verdict, decision, action=None, detail=""):
if action:
actions.append(action)
status, note = paint("retry", "yellow"), paint(f" -> {action}{detail}", "yellow")
elif verdict.ok and not rate_limited(verdict):
elif verdict.ok:
status, note = paint("keep ", "green"), ""
else:
status, note = paint("skip ", "red"), ""
Expand All @@ -132,7 +124,7 @@ def show(verdict, decision, action=None, detail=""):
verdict, decision = await judge(page)

for attempt in (1, 2):
if not rate_limited(verdict):
if not verdict.is_rate_limited:
break
show(verdict, decision, "back off", f" {3 * attempt}s")
await asyncio.sleep(3 * attempt)
Expand Down Expand Up @@ -167,7 +159,7 @@ async def triage(browser, slots, url):
print(f"{paint('fail ', 'red')} {url.split('://', 1)[1][:44]:<44} {err}")
return "failed", [], perf_counter() - start
took = perf_counter() - start
outcome = "kept" if verdict.ok and not rate_limited(verdict) else describe(verdict)
outcome = "kept" if verdict.ok else describe(verdict)
return outcome, actions, took


Expand Down
8 changes: 4 additions & 4 deletions lightpanda/_methods.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ def call_page_tool(self, *, name: str, arguments: dict | None = None) -> Any:
"""
return self.call("callPageTool", name=name, arguments=arguments)
def classify(self, *, questions: str | dict | list, selector: str | None = None, presets: bool | None = None) -> Any:
"""Classify the page or an element subtree using TypeSafe System One (Jev). `questions` is a JSON array of categories (returns the one that fits), or a JSON object of questions keyed by the name each answer comes back under: `true` for a preset (isBlocked, isCaptcha, isConsentWall, isEmptyCatalog, isErrorPage, isLoginWall, isPaywall, isUnsupportedBrowser, isLoading; an empty object `{}` asks all of them), a string for a yes/no question (returns its probability), `{question, options}` for a choice, or `{question, levels}` for a score on 2-10 levels ordered low to high (returns the `score` between level indices, the nearest `level`, and per-level `probabilities`). A missing answer is null.
"""Classify the page or an element subtree using TypeSafe System One (Jev). `questions` is a JSON array of categories (returns the one that fits), or a JSON object of questions keyed by the name each answer comes back under: `true` for a preset (isBlocked, isCaptcha, isConsentWall, isEmptyCatalog, isErrorPage, isLoginWall, isPaywall, isUnsupportedBrowser, isLoading, isAgeGate, isRegionBlocked, isRateLimited; an empty object `{}` asks all of them), a string for a yes/no question (returns its probability), `{question, options}` for a choice, or `{question, levels}` for a score on 2-10 levels ordered low to high (returns the `score` between level indices, the nearest `level`, and per-level `probabilities`). A missing answer is null.

Args:
questions: JSON string: array of categories, or object of questions (`{}` for every preset)
Expand Down Expand Up @@ -350,7 +350,7 @@ def set_checked(self, *, checked: bool, selector: str | None = None, backend_nod
"""
return self.call("setChecked", checked=checked, selector=selector, backend_node_id=backend_node_id)
def structured_data(self, *, url: str | None = None, timeout: int | None = None) -> Any:
"""Page metadata as JSON: `jsonLd` (each JSON-LD block as a string), `openGraph`, `twitterCard`, `meta` and `links` (key/value lists), plus `alternate` (hreflang variants) and `linkHeaders` (relations from the HTTP Link header) when present. Empty sections come back as empty arrays. Use it for publisher-declared facts such as product price, article author or canonical URL before scraping the visible text for them. If a url is provided, it navigates there first.
"""Page metadata as JSON: `jsonLd` (each JSON-LD block as a string), `openGraph`, `twitterCard`, `meta` and `links` (key/value lists), plus `microdata` (itemscope items as `{type, id, properties}`, every property value an array), `alternate` (hreflang variants) and `linkHeaders` (relations from the HTTP Link header) when present. Empty sections come back as empty arrays. Use it for publisher-declared facts such as product price, article author or canonical URL before scraping the visible text for them. If a url is provided, it navigates there first.

Args:
url: Optional URL to navigate to before processing. An empty string is treated as omitted.
Expand Down Expand Up @@ -549,7 +549,7 @@ async def call_page_tool(self, *, name: str, arguments: dict | None = None) -> A
"""
return await self.call("callPageTool", name=name, arguments=arguments)
async def classify(self, *, questions: str | dict | list, selector: str | None = None, presets: bool | None = None) -> Any:
"""Classify the page or an element subtree using TypeSafe System One (Jev). `questions` is a JSON array of categories (returns the one that fits), or a JSON object of questions keyed by the name each answer comes back under: `true` for a preset (isBlocked, isCaptcha, isConsentWall, isEmptyCatalog, isErrorPage, isLoginWall, isPaywall, isUnsupportedBrowser, isLoading; an empty object `{}` asks all of them), a string for a yes/no question (returns its probability), `{question, options}` for a choice, or `{question, levels}` for a score on 2-10 levels ordered low to high (returns the `score` between level indices, the nearest `level`, and per-level `probabilities`). A missing answer is null.
"""Classify the page or an element subtree using TypeSafe System One (Jev). `questions` is a JSON array of categories (returns the one that fits), or a JSON object of questions keyed by the name each answer comes back under: `true` for a preset (isBlocked, isCaptcha, isConsentWall, isEmptyCatalog, isErrorPage, isLoginWall, isPaywall, isUnsupportedBrowser, isLoading, isAgeGate, isRegionBlocked, isRateLimited; an empty object `{}` asks all of them), a string for a yes/no question (returns its probability), `{question, options}` for a choice, or `{question, levels}` for a score on 2-10 levels ordered low to high (returns the `score` between level indices, the nearest `level`, and per-level `probabilities`). A missing answer is null.

Args:
questions: JSON string: array of categories, or object of questions (`{}` for every preset)
Expand Down Expand Up @@ -862,7 +862,7 @@ async def set_checked(self, *, checked: bool, selector: str | None = None, backe
"""
return await self.call("setChecked", checked=checked, selector=selector, backend_node_id=backend_node_id)
async def structured_data(self, *, url: str | None = None, timeout: int | None = None) -> Any:
"""Page metadata as JSON: `jsonLd` (each JSON-LD block as a string), `openGraph`, `twitterCard`, `meta` and `links` (key/value lists), plus `alternate` (hreflang variants) and `linkHeaders` (relations from the HTTP Link header) when present. Empty sections come back as empty arrays. Use it for publisher-declared facts such as product price, article author or canonical URL before scraping the visible text for them. If a url is provided, it navigates there first.
"""Page metadata as JSON: `jsonLd` (each JSON-LD block as a string), `openGraph`, `twitterCard`, `meta` and `links` (key/value lists), plus `microdata` (itemscope items as `{type, id, properties}`, every property value an array), `alternate` (hreflang variants) and `linkHeaders` (relations from the HTTP Link header) when present. Empty sections come back as empty arrays. Use it for publisher-declared facts such as product price, article author or canonical URL before scraping the visible text for them. If a url is provided, it navigates there first.

Args:
url: Optional URL to navigate to before processing. An empty string is treated as omitted.
Expand Down
11 changes: 10 additions & 1 deletion lightpanda/browser.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,15 @@ class PageVerdict:
is_loading: bool
"""The page says it is still loading (spinners, placeholders, loading
text). A page that is blank while it loads says nothing, so this misses it."""
is_age_gate: bool
"""An age check covers the content, asking the visitor to confirm they are
an adult or to enter their date of birth."""
is_region_blocked: bool
"""The site says the content is not available in the visitor's country or
region."""
is_rate_limited: bool
"""The site says too many requests were sent, and to slow down or try again
later."""
scores: dict[str, float | None]
"""Each check's probability, keyed by its attribute name."""
answers: dict
Expand Down Expand Up @@ -156,7 +165,7 @@ def __repr__(self) -> str:
# Of 1,781 pages that answered 4xx or 5xx in the classifier's crawl, the model
# found none worth reading, so judge() skips them without asking it. The
# statuses that say why name their check; any other names an error page.
_STATUS_CHECKS = {401: "is_login_wall", 402: "is_paywall", 403: "is_blocked", 429: "is_blocked"}
_STATUS_CHECKS = {401: "is_login_wall", 402: "is_paywall", 403: "is_blocked", 429: "is_rate_limited"}


def _snake(name: str) -> str:
Expand Down
3 changes: 2 additions & 1 deletion tests/test_classify.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,8 @@ def test_judge(classify_browser, fixture_url):
page.goto(url=f"{fixture_url}/index.html")
verdict = page.judge()
assert verdict.is_blocked and verdict.is_paywall and not verdict.ok
assert set(verdict.scores) >= {"is_blocked", "is_login_wall", "is_unsupported_browser"}
assert set(verdict.scores) >= {"is_blocked", "is_login_wall", "is_unsupported_browser", "is_rate_limited"}
assert verdict.is_age_gate and verdict.is_region_blocked
assert set(verdict.scores.values()) == {0.9}
assert verdict.reason in verdict.scores
assert page.judge(threshold=0.95).ok
Expand Down
Loading