From e2456e2d4720980d5896421797d594e4538b92e5 Mon Sep 17 00:00:00 2001 From: patchflux <94029+moul@users.noreply.github.com> Date: Wed, 23 Sep 2026 14:18:03 +0000 Subject: [PATCH] feat: photograph pages that are not gnoweb, and wait for the ones that draw themselves --- README.md | 28 ++++++++++++--- main.go | 12 ++++--- pkg/shot/capture.go | 59 +++++++++++++++++++++++++++++-- pkg/shot/resolve.go | 56 +++++++++++++++++++++++++----- pkg/shot/service.go | 15 ++++++-- pkg/shot/shot_test.go | 81 ++++++++++++++++++++++++++++++++++++++++++- pkg/shot/types.go | 8 +++++ 7 files changed, 236 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 5acb8c3..fd42d2e 100644 --- a/README.md +++ b/README.md @@ -181,10 +181,30 @@ screenshot is the worse answer. ## Safety -`?url=` is restricted to an allowlist (`-allow`, default `gno.land` and the -testnets). An unbounded one is an open proxy and a way to spend this box's CPU -on headless Chrome. `POST /invalidate` needs a bearer token and is refused -outright when none is configured. +`?url=` is restricted to an allowlist. An unbounded one is an open proxy and a +way to spend this box's CPU on headless Chrome. `POST /invalidate` needs a +bearer token and is refused outright when none is configured. + +There are two lists, and the split says what kind of page a host serves: + +| flag | for | how it is photographed | +|---|---|---| +| `-allow` | gnoweb hosts (default `gno.land` and the testnets) | through the selector chain, cropped to the realm's own render | +| `-allow-site` | anything else, empty by default | whole page, no selector chain | + +The second exists because "no selector in the chain matched" means two +different things depending on where you are. On gnoweb it means an error page, +which is what `matched=status` records and why a placeholder tile is served +instead of a screenshot. On somebody else's app it means nothing at all: the +chain is gnoweb's own class names. A site host is recorded as `matched=site`, +is never demoted to a status tile for failing to look like gnoweb, and the +ladder is derived from its page master, so `size=thumb` and `size=og` answer +normally. + +Keeping the two apart is deliberate: the operator states which kind a host is, +rather than the service guessing from what it found. A host named in both is a +configuration mistake and gnoweb wins, because the selector chain produces a +better picture wherever it applies. ## Cost diff --git a/main.go b/main.go index ab5b50b..6df73dc 100644 --- a/main.go +++ b/main.go @@ -100,7 +100,8 @@ func cmdServe(args []string) error { addr := fs.String("addr", env("GNOSHOT_ADDR", ":8890"), "listen address") root := fs.String("root", defaultRoot(), "storage root") workers := fs.Int("workers", 2, "capture workers; each holds a browser at roughly 1.3 GB") - allow := fs.String("allow", shot.DefaultAllowHosts, "comma-separated host allowlist") + allow := fs.String("allow", shot.DefaultAllowHosts, "comma-separated gnoweb host allowlist") + allowSites := fs.String("allow-site", "", "comma-separated allowlist of non-gnoweb hosts, photographed whole rather than through the selector chain") base := fs.String("gnoweb", env("GNOSHOT_GNOWEB", "https://gno.land"), "gnoweb base URL for the sweep") source := fs.String("source", env("GNOSHOT_SOURCE", ""), "mygnoscan base URL to enumerate paths from; empty disables the corpus sweep") network := fs.String("network", env("GNOSHOT_NETWORK", "mainnet"), "network id to enumerate") @@ -110,7 +111,7 @@ func cmdServe(args []string) error { fs.Parse(args) svc, err := shot.New(shot.Config{ - Root: *root, Workers: *workers, AllowHosts: *allow, + Root: *root, Workers: *workers, AllowHosts: *allow, AllowSites: *allowSites, RefreshAfter: *refresh, SweepEvery: *sweep, SweepSource: *source, SweepNetwork: *network, GnowebBase: *base, }) @@ -148,6 +149,7 @@ func cmdCapture(args []string) error { out := fs.String("out", ".", "output directory") theme := fs.String("theme", "light", "light|dark") modes := fs.String("mode", "both", "page|render|both") + site := fs.Bool("site", false, "the URL is not gnoweb: photograph the whole page, skip the selector chain") fs.Parse(args) if fs.NArg() != 1 { return fmt.Errorf("usage: gnoshot capture [-out DIR] [-theme light|dark] [-mode page|render|both] ") @@ -201,7 +203,7 @@ func cmdCapture(args []string) error { } defer br.Close() t0 := time.Now() - res, err := br.Capture(ctx, pageURL, th, want) + res, err := br.Capture(ctx, pageURL, th, want, *site) if err != nil { return err } @@ -271,7 +273,9 @@ func cmdResolve(args []string) error { continue } // An empty mode list resolves the chain and captures nothing. - res, err := br.Capture(ctx, u, th, nil) + // resolve is about the selector chain by definition, so it never asks + // for the whole-page path. + res, err := br.Capture(ctx, u, th, nil, false) if err != nil { fmt.Printf("%-4d %-9s %-26s %-11s %s\n", p.Status, "err", "-", "-", u) continue diff --git a/pkg/shot/capture.go b/pkg/shot/capture.go index 989a4e6..5340130 100644 --- a/pkg/shot/capture.go +++ b/pkg/shot/capture.go @@ -157,7 +157,49 @@ func (b *Browser) Close() { // // The probe has already decided whether the page is worth visiting; this is // only ever called for a page that answered 200. -func (b *Browser) Capture(ctx context.Context, pageURL string, theme Theme, modes []Mode) (*Result, error) { +// settleJS resolves once the DOM has stopped changing, or at the cap. +// +// fonts.ready plus two frames is enough for gnoweb, which server-renders and +// then hydrates. It is not enough for a page that paints nothing until its +// JavaScript has fetched and rendered, and those are pages this service is now +// also asked to photograph. Measured 2026-09-23 in page mode, all three +// answering 200 with real content in a browser: gnoswap.io came out a +// 228-byte blank, gnolove.world and kourt.xyz came out as their own loading +// spinners. A confident picture of a spinner is worse than no picture, because +// nothing downstream can tell the two apart. +// +// A MutationObserver quiesce rather than a fixed sleep or network idle: +// +// - A fixed sleep taxes every gnoweb capture for the sake of the handful that +// need it, and the sweep runs over every realm on the chain. This returns +// in one quiet window on a page that was already done. +// - Network idle never arrives on a page holding a websocket or a poll open, +// and several of these do. +// +// Quiet means the whole document, subtree and attributes included, because a +// spinner that swaps one class is a page still deciding what it is. The cap +// bounds the worst case, a page that never stops animating: it fires and the +// capture proceeds, which is exactly what happened before this existed. +const ( + settleQuiet = 400 // ms without a mutation that counts as done + settleCap = 2500 // ms after which we photograph whatever is there +) + +var settleJS = fmt.Sprintf(`new Promise(resolve => { + let timer = null; + const done = () => { obs.disconnect(); clearTimeout(timer); clearTimeout(cap); resolve(true); }; + const cap = setTimeout(done, %d); + const bump = () => { clearTimeout(timer); timer = setTimeout(done, %d); }; + const obs = new MutationObserver(bump); + obs.observe(document.documentElement, {childList: true, subtree: true, attributes: true, characterData: true}); + bump(); +})`, settleCap, settleQuiet) + +// Capture photographs one page. +// +// site says the URL is on a host that does not serve gnoweb, so the selector +// chain is skipped and the whole document is the picture. See docHeightJS. +func (b *Browser) Capture(ctx context.Context, pageURL string, theme Theme, modes []Mode, site bool) (*Result, error) { b.mu.Lock() defer b.mu.Unlock() @@ -197,6 +239,10 @@ func (b *Browser) Capture(ctx context.Context, pageURL string, theme Theme, mode awaitPromise := func(p *runtime.EvaluateParams) *runtime.EvaluateParams { return p.WithAwaitPromise(true) } + probeScript := resolveJS + if site { + probeScript = docHeightJS + } tasks := chromedp.Tasks{ // Device metrics stay at scale 1 and the master's 2x comes from the // screenshot clip. Setting both is how you silently get a 4x image. @@ -214,7 +260,10 @@ func (b *Browser) Capture(ctx context.Context, pageURL string, theme Theme, mode chromedp.Evaluate( `new Promise(r => document.fonts.ready.then(() => requestAnimationFrame(() => requestAnimationFrame(() => r(true)))))`, nil, awaitPromise), - chromedp.Evaluate(resolveJS, &res), + // And then wait for the DOM to stop moving, which fonts.ready does not + // imply on a page that renders itself in JavaScript. + chromedp.Evaluate(settleJS, nil, awaitPromise), + chromedp.Evaluate(probeScript, &res), } if err := chromedp.Run(tab, tasks); err != nil { return nil, fmt.Errorf("load %s: %w", pageURL, err) @@ -231,7 +280,11 @@ func (b *Browser) Capture(ctx context.Context, pageURL string, theme Theme, mode var clip Box switch m { case ModeRender: - if out.Matched == MatchedStatus { + // Nothing to crop to on either: an error page has no render, and a + // site host never ran the chain that would have found one. Both + // are answered by the page master, which the ladder then derives + // every rung from. + if out.Matched == MatchedStatus || out.Matched == MatchedSite { continue } clip = out.Box diff --git a/pkg/shot/resolve.go b/pkg/shot/resolve.go index b63d368..d49ced7 100644 --- a/pkg/shot/resolve.go +++ b/pkg/shot/resolve.go @@ -95,6 +95,18 @@ var resolveJS = func() string { return sb.String() }() +// docHeightJS is what runs instead of the selector chain on a site host. +// +// The chain is gnoweb's own class names and it also *mutates* what it matches, +// padding the element so the crop has breathing room. Neither is right on +// somebody else's page: the class names mean nothing there, and rewriting a +// third-party document before photographing it would be photographing +// something that never existed. All a whole-page capture needs is how tall the +// document is. +const docHeightJS = `(() => ({matched: "site", selector: "", x: 0, y: 0, w: 0, h: 0, + doc: Math.max(document.body ? document.body.scrollHeight : 0, + document.documentElement.scrollHeight)}))()` + // FramePadding is the breathing room added around a render before it is // photographed, in CSS pixels. // @@ -194,20 +206,47 @@ func DefaultProbeClient() *http.Client { // is a 400 before any work happens. type Allowlist struct { hosts map[string]bool + sites map[string]bool } -// NewAllowlist builds an allowlist from a comma-separated host list. -func NewAllowlist(spec string) *Allowlist { - a := &Allowlist{hosts: map[string]bool{}} - for _, h := range strings.Split(spec, ",") { - h = strings.ToLower(strings.TrimSpace(h)) - if h != "" { - a.hosts[h] = true +// NewAllowlist builds an allowlist from two comma-separated host lists. +// +// The split is not cosmetic. A gnoweb host is photographed through the selector +// chain, which is a set of gnoweb's own class names; a site host has no such +// structure and is photographed whole. Running the chain against a host it was +// never written for is how every third-party app came out classified as an +// error page. Keeping the two lists apart means the operator states which kind +// of page a host serves, rather than the service guessing from what it found. +func NewAllowlist(spec, siteSpec string) *Allowlist { + a := &Allowlist{hosts: map[string]bool{}, sites: map[string]bool{}} + fill := func(dst map[string]bool, spec string) { + for _, h := range strings.Split(spec, ",") { + h = strings.ToLower(strings.TrimSpace(h)) + if h != "" { + dst[h] = true + } } } + fill(a.hosts, spec) + fill(a.sites, siteSpec) + // A host named in both is a configuration mistake with a quiet failure + // mode, so gnoweb wins and the site entry is dropped: the selector chain + // produces a better picture wherever it applies. + for h := range a.hosts { + delete(a.sites, h) + } return a } +// IsSite reports whether a URL is on a host photographed whole. +func (a *Allowlist) IsSite(raw string) bool { + u, err := url.Parse(raw) + if err != nil { + return false + } + return a.sites[strings.ToLower(u.Hostname())] +} + // DefaultAllowHosts is the production set: mainnet and the two public networks // that currently serve a gnoweb. // @@ -225,7 +264,8 @@ func (a *Allowlist) Check(raw string) (string, error) { if u.Scheme != "https" && u.Scheme != "http" { return "", fmt.Errorf("scheme %q not allowed", u.Scheme) } - if !a.hosts[strings.ToLower(u.Hostname())] { + h := strings.ToLower(u.Hostname()) + if !a.hosts[h] && !a.sites[h] { return "", fmt.Errorf("host %q not on the allowlist", u.Hostname()) } u.Fragment = "" diff --git a/pkg/shot/service.go b/pkg/shot/service.go index 50dff85..5e9e849 100644 --- a/pkg/shot/service.go +++ b/pkg/shot/service.go @@ -13,7 +13,8 @@ import ( type Config struct { Root string // storage root Workers int // browsers, each ~1.3 GB of RSS - AllowHosts string // comma-separated allowlist + AllowHosts string // comma-separated gnoweb host allowlist + AllowSites string // comma-separated non-gnoweb host allowlist RefreshAfter time.Duration // how old a probe may be before a read re-probes SweepEvery time.Duration // background sweep interval, 0 to disable SweepSource string // mygnoscan-compatible base URL to enumerate paths from @@ -90,7 +91,7 @@ func New(cfg Config) (*Service, error) { return &Service{ cfg: cfg, store: st, - allow: NewAllowlist(cfg.AllowHosts), + allow: NewAllowlist(cfg.AllowHosts, cfg.AllowSites), probe: DefaultProbeClient(), interactive: make(chan job, cfg.QueueDepth), background: make(chan job, cfg.QueueDepth), @@ -202,9 +203,17 @@ func (s *Service) done(j job) { // is the whole pipeline, and the CLI runs the identical code path so the // service is not the only way to reproduce a capture. func (s *Service) Refresh(ctx context.Context, br *Browser, pageURL string, theme Theme, modes ...Mode) error { + // Whether this is gnoweb is a property of the host, not of the request, so + // it is read here rather than carried through the queue: a caller asking + // for a render of a site host is asking for something that does not exist, + // and the honest answer is the page, not an error. + site := s.allow.IsSite(pageURL) if len(modes) == 0 { modes = []Mode{ModeRender} } + if site { + modes = []Mode{ModePage} + } p, err := DoProbe(ctx, s.probe, pageURL) if err != nil { return fmt.Errorf("probe: %w", err) @@ -244,7 +253,7 @@ func (s *Service) Refresh(ctx context.Context, br *Browser, pageURL string, them if prev != nil && prev.Freshness == p.Freshness { modes = union(modes, capturedModes(prev)) } - res, err := br.Capture(ctx, pageURL, theme, modes) + res, err := br.Capture(ctx, pageURL, theme, modes, site) if err != nil { return fmt.Errorf("capture: %w", err) } diff --git a/pkg/shot/shot_test.go b/pkg/shot/shot_test.go index 723b4e4..926e409 100644 --- a/pkg/shot/shot_test.go +++ b/pkg/shot/shot_test.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "fmt" "image" "image/color" "io" @@ -15,7 +16,7 @@ import ( ) func TestAllowlistRefusesEverythingOffTheList(t *testing.T) { - a := NewAllowlist(DefaultAllowHosts) + a := NewAllowlist(DefaultAllowHosts, "adena.app") tests := []struct { name string in string @@ -25,6 +26,7 @@ func TestAllowlistRefusesEverythingOffTheList(t *testing.T) { {"path args kept", "https://gno.land/r/gnoland/blog:p/hello", "https://gno.land/r/gnoland/blog:p/hello"}, {"fragment dropped", "https://gno.land/r/gov/dao#x", "https://gno.land/r/gov/dao"}, {"bare host gets a slash", "https://gno.land", "https://gno.land/"}, + {"site host passes too", "https://adena.app/", "https://adena.app/"}, {"other host", "https://evil.example/r/gov/dao", ""}, {"lookalike host", "https://gno.land.evil.example/", ""}, {"file scheme", "file:///etc/passwd", ""}, @@ -700,3 +702,80 @@ func TestServeDoesNotPromiseAnImmutableImage(t *testing.T) { } } } + +// The settle wait has two numbers and they are only meaningful against each +// other and against the deadline. Both invariants are the kind a later tweak +// breaks silently: the capture still succeeds, it just stops doing the thing +// the numbers were chosen for. +func TestSettleWindowsAreOrdered(t *testing.T) { + if settleQuiet >= settleCap { + // The cap would fire first on every page, so the quiesce would never + // decide anything and this would be a fixed sleep with extra steps. + t.Fatalf("settleQuiet %d must be shorter than settleCap %d", settleQuiet, settleCap) + } + // The cap is only the wait. Navigation, fonts, the resolve evaluate and one + // screenshot per mode all come out of the same deadline, and a page master + // of a tall site is the slowest of those. + if budget := CaptureDeadline - time.Duration(settleCap)*time.Millisecond; budget < 20*time.Second { + t.Fatalf("settleCap %dms leaves only %s of the %s deadline for the capture itself", + settleCap, budget, CaptureDeadline) + } +} + +// The generated JS has to carry both numbers, because they are formatted in +// rather than referenced: a %d that lost its argument produces a script that +// still runs and waits for the wrong thing, or forever. +func TestSettleJSCarriesBothWindows(t *testing.T) { + for _, want := range []string{ + fmt.Sprintf("setTimeout(done, %d)", settleCap), + fmt.Sprintf("setTimeout(done, %d)", settleQuiet), + "MutationObserver", + // Subtree and attributes both: a spinner that swaps one class on one + // node is a page still deciding what it is, and a childList-only + // observer would call that quiet. + "subtree: true", + "attributes: true", + } { + if !strings.Contains(settleJS, want) { + t.Errorf("settleJS is missing %q", want) + } + } +} + +// The two lists are two kinds of page, not one list split in half, and every +// consumer downstream branches on the difference: a site host skips the +// selector chain, is photographed whole, and is never classified as an error +// page for failing to look like gnoweb. +func TestAllowlistTellsAGnowebHostFromASite(t *testing.T) { + a := NewAllowlist("gno.land", "adena.app,kourt.xyz") + for _, tt := range []struct { + url string + site bool + }{ + {"https://gno.land/r/gov/dao", false}, + {"https://adena.app/", true}, + {"https://KOURT.xyz/anything", true}, + {"https://evil.example/", false}, + {"not a url at all", false}, + } { + if got := a.IsSite(tt.url); got != tt.site { + t.Errorf("IsSite(%q) = %v, want %v", tt.url, got, tt.site) + } + } +} + +// A host in both lists is a configuration mistake whose failure mode is quiet: +// the picture silently gets worse, because the whole-page path replaces a crop +// the selector chain would have found. gnoweb wins. +func TestAHostInBothListsStaysGnoweb(t *testing.T) { + a := NewAllowlist("gno.land,staging.gno.land", "gno.land,adena.app") + if a.IsSite("https://gno.land/r/gov/dao") { + t.Error("gno.land is listed in both and came out a site host") + } + if !a.IsSite("https://adena.app/") { + t.Error("adena.app is only a site host and came out a gnoweb one") + } + if _, err := a.Check("https://gno.land/r/gov/dao"); err != nil { + t.Errorf("a host in both lists must still be allowed: %v", err) + } +} diff --git a/pkg/shot/types.go b/pkg/shot/types.go index 4ddd33d..9e5aec6 100644 --- a/pkg/shot/types.go +++ b/pkg/shot/types.go @@ -51,6 +51,14 @@ const ( // MatchedStatus is an error, a 404, or a package submitted and not yet // enabled. Never present one of these as a picture of a working realm. MatchedStatus Matched = "status" + // MatchedSite is a page on a host that is not gnoweb, photographed whole + // because there is no realm render to crop to. + // + // It exists so that "no selector in the chain matched" stops meaning "this + // is an error page". On gnoweb those are the same statement; on adena.app + // they are not, and collapsing them served a placeholder tile for every + // working third-party app. + MatchedSite Matched = "site" ) // Status refines MatchedStatus. All three render the same full-screen box and