Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 24 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -181,10 +181,30 @@ screenshot is the worse answer.

## Safety

`?url=` is restricted to an allowlist (`-allow`, default `gno.land` and the
testnets). An unbounded one is an open proxy and a way to spend this box's CPU
on headless Chrome. `POST /invalidate` needs a bearer token and is refused
outright when none is configured.
`?url=` is restricted to an allowlist. An unbounded one is an open proxy and a
way to spend this box's CPU on headless Chrome. `POST /invalidate` needs a
bearer token and is refused outright when none is configured.

There are two lists, and the split says what kind of page a host serves:

| flag | for | how it is photographed |
|---|---|---|
| `-allow` | gnoweb hosts (default `gno.land` and the testnets) | through the selector chain, cropped to the realm's own render |
| `-allow-site` | anything else, empty by default | whole page, no selector chain |

The second exists because "no selector in the chain matched" means two
different things depending on where you are. On gnoweb it means an error page,
which is what `matched=status` records and why a placeholder tile is served
instead of a screenshot. On somebody else's app it means nothing at all: the
chain is gnoweb's own class names. A site host is recorded as `matched=site`,
is never demoted to a status tile for failing to look like gnoweb, and the
ladder is derived from its page master, so `size=thumb` and `size=og` answer
normally.

Keeping the two apart is deliberate: the operator states which kind a host is,
rather than the service guessing from what it found. A host named in both is a
configuration mistake and gnoweb wins, because the selector chain produces a
better picture wherever it applies.

## Cost

Expand Down
12 changes: 8 additions & 4 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,8 @@ func cmdServe(args []string) error {
addr := fs.String("addr", env("GNOSHOT_ADDR", ":8890"), "listen address")
root := fs.String("root", defaultRoot(), "storage root")
workers := fs.Int("workers", 2, "capture workers; each holds a browser at roughly 1.3 GB")
allow := fs.String("allow", shot.DefaultAllowHosts, "comma-separated host allowlist")
allow := fs.String("allow", shot.DefaultAllowHosts, "comma-separated gnoweb host allowlist")
allowSites := fs.String("allow-site", "", "comma-separated allowlist of non-gnoweb hosts, photographed whole rather than through the selector chain")
base := fs.String("gnoweb", env("GNOSHOT_GNOWEB", "https://gno.land"), "gnoweb base URL for the sweep")
source := fs.String("source", env("GNOSHOT_SOURCE", ""), "mygnoscan base URL to enumerate paths from; empty disables the corpus sweep")
network := fs.String("network", env("GNOSHOT_NETWORK", "mainnet"), "network id to enumerate")
Expand All @@ -110,7 +111,7 @@ func cmdServe(args []string) error {
fs.Parse(args)

svc, err := shot.New(shot.Config{
Root: *root, Workers: *workers, AllowHosts: *allow,
Root: *root, Workers: *workers, AllowHosts: *allow, AllowSites: *allowSites,
RefreshAfter: *refresh, SweepEvery: *sweep, SweepSource: *source,
SweepNetwork: *network, GnowebBase: *base,
})
Expand Down Expand Up @@ -148,6 +149,7 @@ func cmdCapture(args []string) error {
out := fs.String("out", ".", "output directory")
theme := fs.String("theme", "light", "light|dark")
modes := fs.String("mode", "both", "page|render|both")
site := fs.Bool("site", false, "the URL is not gnoweb: photograph the whole page, skip the selector chain")
fs.Parse(args)
if fs.NArg() != 1 {
return fmt.Errorf("usage: gnoshot capture [-out DIR] [-theme light|dark] [-mode page|render|both] <url>")
Expand Down Expand Up @@ -201,7 +203,7 @@ func cmdCapture(args []string) error {
}
defer br.Close()
t0 := time.Now()
res, err := br.Capture(ctx, pageURL, th, want)
res, err := br.Capture(ctx, pageURL, th, want, *site)
if err != nil {
return err
}
Expand Down Expand Up @@ -271,7 +273,9 @@ func cmdResolve(args []string) error {
continue
}
// An empty mode list resolves the chain and captures nothing.
res, err := br.Capture(ctx, u, th, nil)
// resolve is about the selector chain by definition, so it never asks
// for the whole-page path.
res, err := br.Capture(ctx, u, th, nil, false)
if err != nil {
fmt.Printf("%-4d %-9s %-26s %-11s %s\n", p.Status, "err", "-", "-", u)
continue
Expand Down
59 changes: 56 additions & 3 deletions pkg/shot/capture.go
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,49 @@ func (b *Browser) Close() {
//
// The probe has already decided whether the page is worth visiting; this is
// only ever called for a page that answered 200.
func (b *Browser) Capture(ctx context.Context, pageURL string, theme Theme, modes []Mode) (*Result, error) {
// settleJS resolves once the DOM has stopped changing, or at the cap.
//
// fonts.ready plus two frames is enough for gnoweb, which server-renders and
// then hydrates. It is not enough for a page that paints nothing until its
// JavaScript has fetched and rendered, and those are pages this service is now
// also asked to photograph. Measured 2026-09-23 in page mode, all three
// answering 200 with real content in a browser: gnoswap.io came out a
// 228-byte blank, gnolove.world and kourt.xyz came out as their own loading
// spinners. A confident picture of a spinner is worse than no picture, because
// nothing downstream can tell the two apart.
//
// A MutationObserver quiesce rather than a fixed sleep or network idle:
//
// - A fixed sleep taxes every gnoweb capture for the sake of the handful that
// need it, and the sweep runs over every realm on the chain. This returns
// in one quiet window on a page that was already done.
// - Network idle never arrives on a page holding a websocket or a poll open,
// and several of these do.
//
// Quiet means the whole document, subtree and attributes included, because a
// spinner that swaps one class is a page still deciding what it is. The cap
// bounds the worst case, a page that never stops animating: it fires and the
// capture proceeds, which is exactly what happened before this existed.
const (
settleQuiet = 400 // ms without a mutation that counts as done
settleCap = 2500 // ms after which we photograph whatever is there
)

var settleJS = fmt.Sprintf(`new Promise(resolve => {
let timer = null;
const done = () => { obs.disconnect(); clearTimeout(timer); clearTimeout(cap); resolve(true); };
const cap = setTimeout(done, %d);
const bump = () => { clearTimeout(timer); timer = setTimeout(done, %d); };
const obs = new MutationObserver(bump);
obs.observe(document.documentElement, {childList: true, subtree: true, attributes: true, characterData: true});
bump();
})`, settleCap, settleQuiet)

// Capture photographs one page.
//
// site says the URL is on a host that does not serve gnoweb, so the selector
// chain is skipped and the whole document is the picture. See docHeightJS.
func (b *Browser) Capture(ctx context.Context, pageURL string, theme Theme, modes []Mode, site bool) (*Result, error) {
b.mu.Lock()
defer b.mu.Unlock()

Expand Down Expand Up @@ -197,6 +239,10 @@ func (b *Browser) Capture(ctx context.Context, pageURL string, theme Theme, mode
awaitPromise := func(p *runtime.EvaluateParams) *runtime.EvaluateParams {
return p.WithAwaitPromise(true)
}
probeScript := resolveJS
if site {
probeScript = docHeightJS
}
tasks := chromedp.Tasks{
// Device metrics stay at scale 1 and the master's 2x comes from the
// screenshot clip. Setting both is how you silently get a 4x image.
Expand All @@ -214,7 +260,10 @@ func (b *Browser) Capture(ctx context.Context, pageURL string, theme Theme, mode
chromedp.Evaluate(
`new Promise(r => document.fonts.ready.then(() => requestAnimationFrame(() => requestAnimationFrame(() => r(true)))))`,
nil, awaitPromise),
chromedp.Evaluate(resolveJS, &res),
// And then wait for the DOM to stop moving, which fonts.ready does not
// imply on a page that renders itself in JavaScript.
chromedp.Evaluate(settleJS, nil, awaitPromise),
chromedp.Evaluate(probeScript, &res),
}
if err := chromedp.Run(tab, tasks); err != nil {
return nil, fmt.Errorf("load %s: %w", pageURL, err)
Expand All @@ -231,7 +280,11 @@ func (b *Browser) Capture(ctx context.Context, pageURL string, theme Theme, mode
var clip Box
switch m {
case ModeRender:
if out.Matched == MatchedStatus {
// Nothing to crop to on either: an error page has no render, and a
// site host never ran the chain that would have found one. Both
// are answered by the page master, which the ladder then derives
// every rung from.
if out.Matched == MatchedStatus || out.Matched == MatchedSite {
continue
}
clip = out.Box
Expand Down
56 changes: 48 additions & 8 deletions pkg/shot/resolve.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,18 @@ var resolveJS = func() string {
return sb.String()
}()

// docHeightJS is what runs instead of the selector chain on a site host.
//
// The chain is gnoweb's own class names and it also *mutates* what it matches,
// padding the element so the crop has breathing room. Neither is right on
// somebody else's page: the class names mean nothing there, and rewriting a
// third-party document before photographing it would be photographing
// something that never existed. All a whole-page capture needs is how tall the
// document is.
const docHeightJS = `(() => ({matched: "site", selector: "", x: 0, y: 0, w: 0, h: 0,
doc: Math.max(document.body ? document.body.scrollHeight : 0,
document.documentElement.scrollHeight)}))()`

// FramePadding is the breathing room added around a render before it is
// photographed, in CSS pixels.
//
Expand Down Expand Up @@ -194,20 +206,47 @@ func DefaultProbeClient() *http.Client {
// is a 400 before any work happens.
type Allowlist struct {
hosts map[string]bool
sites map[string]bool
}

// NewAllowlist builds an allowlist from a comma-separated host list.
func NewAllowlist(spec string) *Allowlist {
a := &Allowlist{hosts: map[string]bool{}}
for _, h := range strings.Split(spec, ",") {
h = strings.ToLower(strings.TrimSpace(h))
if h != "" {
a.hosts[h] = true
// NewAllowlist builds an allowlist from two comma-separated host lists.
//
// The split is not cosmetic. A gnoweb host is photographed through the selector
// chain, which is a set of gnoweb's own class names; a site host has no such
// structure and is photographed whole. Running the chain against a host it was
// never written for is how every third-party app came out classified as an
// error page. Keeping the two lists apart means the operator states which kind
// of page a host serves, rather than the service guessing from what it found.
func NewAllowlist(spec, siteSpec string) *Allowlist {
a := &Allowlist{hosts: map[string]bool{}, sites: map[string]bool{}}
fill := func(dst map[string]bool, spec string) {
for _, h := range strings.Split(spec, ",") {
h = strings.ToLower(strings.TrimSpace(h))
if h != "" {
dst[h] = true
}
}
}
fill(a.hosts, spec)
fill(a.sites, siteSpec)
// A host named in both is a configuration mistake with a quiet failure
// mode, so gnoweb wins and the site entry is dropped: the selector chain
// produces a better picture wherever it applies.
for h := range a.hosts {
delete(a.sites, h)
}
return a
}

// IsSite reports whether a URL is on a host photographed whole.
func (a *Allowlist) IsSite(raw string) bool {
u, err := url.Parse(raw)
if err != nil {
return false
}
return a.sites[strings.ToLower(u.Hostname())]
}

// DefaultAllowHosts is the production set: mainnet and the two public networks
// that currently serve a gnoweb.
//
Expand All @@ -225,7 +264,8 @@ func (a *Allowlist) Check(raw string) (string, error) {
if u.Scheme != "https" && u.Scheme != "http" {
return "", fmt.Errorf("scheme %q not allowed", u.Scheme)
}
if !a.hosts[strings.ToLower(u.Hostname())] {
h := strings.ToLower(u.Hostname())
if !a.hosts[h] && !a.sites[h] {
return "", fmt.Errorf("host %q not on the allowlist", u.Hostname())
}
u.Fragment = ""
Expand Down
15 changes: 12 additions & 3 deletions pkg/shot/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ import (
type Config struct {
Root string // storage root
Workers int // browsers, each ~1.3 GB of RSS
AllowHosts string // comma-separated allowlist
AllowHosts string // comma-separated gnoweb host allowlist
AllowSites string // comma-separated non-gnoweb host allowlist
RefreshAfter time.Duration // how old a probe may be before a read re-probes
SweepEvery time.Duration // background sweep interval, 0 to disable
SweepSource string // mygnoscan-compatible base URL to enumerate paths from
Expand Down Expand Up @@ -90,7 +91,7 @@ func New(cfg Config) (*Service, error) {
return &Service{
cfg: cfg,
store: st,
allow: NewAllowlist(cfg.AllowHosts),
allow: NewAllowlist(cfg.AllowHosts, cfg.AllowSites),
probe: DefaultProbeClient(),
interactive: make(chan job, cfg.QueueDepth),
background: make(chan job, cfg.QueueDepth),
Expand Down Expand Up @@ -202,9 +203,17 @@ func (s *Service) done(j job) {
// is the whole pipeline, and the CLI runs the identical code path so the
// service is not the only way to reproduce a capture.
func (s *Service) Refresh(ctx context.Context, br *Browser, pageURL string, theme Theme, modes ...Mode) error {
// Whether this is gnoweb is a property of the host, not of the request, so
// it is read here rather than carried through the queue: a caller asking
// for a render of a site host is asking for something that does not exist,
// and the honest answer is the page, not an error.
site := s.allow.IsSite(pageURL)
if len(modes) == 0 {
modes = []Mode{ModeRender}
}
if site {
modes = []Mode{ModePage}
}
p, err := DoProbe(ctx, s.probe, pageURL)
if err != nil {
return fmt.Errorf("probe: %w", err)
Expand Down Expand Up @@ -244,7 +253,7 @@ func (s *Service) Refresh(ctx context.Context, br *Browser, pageURL string, them
if prev != nil && prev.Freshness == p.Freshness {
modes = union(modes, capturedModes(prev))
}
res, err := br.Capture(ctx, pageURL, theme, modes)
res, err := br.Capture(ctx, pageURL, theme, modes, site)
if err != nil {
return fmt.Errorf("capture: %w", err)
}
Expand Down
Loading
Loading