fix: read the trust key table's sources and report dates from a committed capture of the public records

pkg/patterns/internal/trustgen/main.go

@@ -2,13 +2,16 @@
 // SPDX-License-Identifier: Apache-2.0
 
 // Command trustgen writes the trust key table, pkg/patterns/trust_keys.json,
-// from the certificates in pkg/patterns/testdata/trust, and the key table of
-// each finding page under docs/findings. Run it with
+// from the certificates in pkg/patterns/testdata/trust and the records in
+// pkg/patterns/testdata/trust-captured/source-records.txt, and the key table
+// of each finding page under docs/findings. Run it with
 //
 //	go generate ./pkg/patterns
 //
-// after adding or changing a certificate there. No value in the table is
-// typed: every hash, key and name is read from a certificate file.
+// after adding or changing a certificate or a record there. No hash, key,
+// name, source or reported date in the table is typed: each hash, key and
+// name is read from a certificate file, and each source and reported date
+// from the records file.
 //
 // A certificate's key use is the directory it sits in: tls, codesigning or
 // root. The command refuses a TLS certificate without the serverAuth extended
@@ -19,10 +22,19 @@
 // and a key whose certificates carry two names, are refused. Each file holds
 // one certificate and is named by its SHA-256.
 //
+// The records file holds one line per public record the table cites: its
+// URL, the UTC time it states for itself, its role and the keys it lists, in
+// columns separated by single spaces; its header states the rules. A row's
+// source is the sorted URLs of the records that list it, and its reported
+// date is the UTC date of the earliest of those whose role is status. The
+// command refuses a record that lists no row, a row that no status record
+// lists, and a row with a record dated before the earliest that reports its
+// status, so that reported is also the earliest date among the row's sources.
+//
 // The table is written in the key table file format: schemaVersion 1, then
 // the rows sorted by SPKI hash, each row's certificate hashes and sources
 // sorted, in the bytes a json.Encoder writes with SetEscapeHTML(false) and
-// SetIndent("", "  "). The same certificates and sources always give the same
+// SetIndent("", "  "). The same certificates and records always give the same
 // bytes.
 package main
 
@@ -41,17 +53,9 @@ import (
 	"path/filepath"
 	"slices"
 	"strings"
+	"time"
 )
 
-// The listing every row of the current table comes from: the Mozilla bug
-// that reported the eight keys, and the Chrome root program's announcement.
-var sources = []string{
-	"https://bugzilla.mozilla.org/show_bug.cgi?id=2083449",
-	"https://blog.google/security/protecting-chrome-and-android-users-from-compromised-emudhra-intermediate-ca-certificates/",
-}
-
-const reported = "2026-10-17"
-
 // schemaVersion is the version of the key table file format written.
 const schemaVersion = 1
 
@@ -66,6 +70,9 @@ var useDirs = []struct {
 }
 
 // Key is one row of trust_keys.json, its fields in the file format's order.
+// Its source is the https URL of each public record that reports the key's
+// status (for the root, its removal from a trust store), and its reported is
+// the earliest UTC date those records state for themselves.
 type Key struct {
 	SPKISHA256   string   `json:"spkiSha256"`
 	Name         string   `json:"name"`
@@ -92,11 +99,12 @@ var pages = map[string]string{
 
 func main() {
 	fixtures := flag.String("fixtures", "testdata/trust", "directory holding tls, codesigning and root")
+	records := flag.String("records", "testdata/trust-captured/source-records.txt", "the public records the table cites")
 	keysPath := flag.String("keys", "trust_keys.json", "key table to write")
 	pagesDir := flag.String("pages", "", "directory of the finding pages whose tables to rewrite")
 	flag.Parse()
 
-	keys, err := buildKeys(*fixtures)
+	keys, err := buildKeys(*fixtures, *records)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "trustgen:", err)
 		os.Exit(1)
@@ -137,10 +145,13 @@ func encodeTable(keys []*Key) ([]byte, error) {
 }
 
 // buildKeys reads every certificate under fixtures and returns one row per
-// SubjectPublicKeyInfo, sorted by SPKI hash.
-func buildKeys(fixtures string) ([]*Key, error) {
-	source := slices.Clone(sources)
-	slices.Sort(source)
+// SubjectPublicKeyInfo, sorted by SPKI hash, each citing the records of the
+// records file that list it.
+func buildKeys(fixtures, recordsPath string) ([]*Key, error) {
+	records, err := readRecords(recordsPath)
+	if err != nil {
+		return nil, err
+	}
 	var keys []*Key
 	for _, d := range useDirs {
 		files, err := filepath.Glob(filepath.Join(fixtures, d.dir, "*.crt"))
@@ -171,8 +182,6 @@ func buildKeys(fixtures string) ([]*Key, error) {
 					Name:         cert.Subject.CommonName,
 					SPKIDER:      base64.StdEncoding.EncodeToString(cert.RawSubjectPublicKeyInfo),
 					Use:          d.use,
-					Source:       source,
-					Reported:     reported,
 					Status:       d.status,
 					DistrustDate: d.distrustDate,
 				}
@@ -196,9 +205,150 @@ func buildKeys(fixtures string) ([]*Key, error) {
 		}
 	}
 	slices.SortFunc(keys, func(a, b *Key) int { return strings.Compare(a.SPKISHA256, b.SPKISHA256) })
+	if err := cite(keys, records, filepath.Dir(recordsPath)); err != nil {
+		return nil, fmt.Errorf("%s: %w", recordsPath, err)
+	}
 	return keys, nil
 }
 
+// record is one line of the records file.
+type record struct {
+	url   string
+	date  string // the UTC date of the time the record states for itself
+	role  string // status when it reports the key's status, lists when it only lists the key
+	lists string // certs:<file>, pkh:<pubKeyHash> or root:<subject common name>
+}
+
+// readRecords returns the records of the records file in file order. A line
+// that starts with # is a comment; any other is a record, whose four columns
+// are separated by single spaces, the last, lists, holding any spaces.
+func readRecords(path string) ([]record, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var records []record
+	seen := map[string]bool{}
+	for i, line := range strings.Split(strings.TrimSuffix(string(data), "\n"), "\n") {
+		if strings.HasPrefix(line, "#") {
+			continue
+		}
+		f := strings.SplitN(line, " ", 4)
+		if len(f) != 4 {
+			return nil, fmt.Errorf("%s:%d: want url, published, role and lists", path, i+1)
+		}
+		url, published, role, lists := f[0], f[1], f[2], f[3]
+		if !strings.HasPrefix(url, "https://") {
+			return nil, fmt.Errorf("%s:%d: %q is not an https URL", path, i+1, url)
+		}
+		if seen[url] {
+			return nil, fmt.Errorf("%s:%d: %s is listed twice", path, i+1, url)
+		}
+		seen[url] = true
+		at, err := time.Parse(time.RFC3339, published)
+		if err != nil || !strings.HasSuffix(published, "Z") {
+			return nil, fmt.Errorf("%s:%d: published %q is not a UTC time", path, i+1, published)
+		}
+		if role != "status" && role != "lists" {
+			return nil, fmt.Errorf("%s:%d: role %q is not status or lists", path, i+1, role)
+		}
+		records = append(records, record{url: url, date: at.Format("2006-01-02"), role: role, lists: lists})
+	}
+	if len(records) == 0 {
+		return nil, fmt.Errorf("%s: holds no record", path)
+	}
+	return records, nil
+}
+
+// cite sets each row's source to the sorted URLs of the records that list
+// it, and its reported to the date of the earliest of those whose role is
+// status. A certs: file is read from dir.
+func cite(keys []*Key, records []record, dir string) error {
+	earliest := map[*Key]string{}
+	for _, r := range records {
+		lists, err := lister(r.lists, dir)
+		if err != nil {
+			return fmt.Errorf("%s: %w", r.url, err)
+		}
+		listed := false
+		for _, k := range keys {
+			if !lists(k) {
+				continue
+			}
+			listed = true
+			k.Source = append(k.Source, r.url)
+			if e, ok := earliest[k]; !ok || r.date < e {
+				earliest[k] = r.date
+			}
+			if r.role == "status" && (k.Reported == "" || r.date < k.Reported) {
+				k.Reported = r.date
+			}
+		}
+		if !listed {
+			return fmt.Errorf("%s lists no row of the table", r.url)
+		}
+	}
+	for _, k := range keys {
+		if k.Reported == "" {
+			return fmt.Errorf("key %s: no record that reports its status lists it", k.SPKISHA256)
+		}
+		if earliest[k] != k.Reported {
+			return fmt.Errorf("key %s: a record of %s lists it before the earliest that reports its status, of %s", k.SPKISHA256, earliest[k], k.Reported)
+		}
+		slices.Sort(k.Source)
+	}
+	return nil
+}
+
+// lister returns whether a record's lists column lists a row: certs:<file>
+// lists every row with a certificate in that captured file, pkh:<value> the
+// row whose SPKI digest that is, and root:<name> the root row of that
+// subject name.
+func lister(lists, dir string) (func(*Key) bool, error) {
+	kind, value, _ := strings.Cut(lists, ":")
+	switch kind {
+	case "certs":
+		if value == "" || value != filepath.Base(value) {
+			return nil, fmt.Errorf("certs:%s is not a file name", value)
+		}
+		certs, err := readFirstColumn(filepath.Join(dir, value))
+		if err != nil {
+			return nil, err
+		}
+		return func(k *Key) bool {
+			return slices.ContainsFunc(k.CertSHA256, func(c string) bool { return certs[c] })
+		}, nil
+	case "pkh":
+		digest, err := base64.StdEncoding.DecodeString(value)
+		if err != nil || len(digest) != sha256.Size {
+			return nil, fmt.Errorf("pkh:%s is not a base64 SHA-256", value)
+		}
+		spki := hex.EncodeToString(digest)
+		return func(k *Key) bool { return k.SPKISHA256 == spki }, nil
+	case "root":
+		return func(k *Key) bool { return k.Use == "root" && k.Name == value }, nil
+	}
+	return nil, fmt.Errorf("lists %q is not certs:, pkh: or root:", lists)
+}
+
+// readFirstColumn returns the first column of each line of a captured-values
+// file that is neither empty nor a comment.
+func readFirstColumn(path string) (map[string]bool, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	values := map[string]bool{}
+	for _, line := range strings.Split(string(data), "\n") {
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		value, _, _ := strings.Cut(line, " ")
+		values[value] = true
+	}
+	return values, nil
+}
+
 // readCertificate returns the one certificate a PEM file holds.
 func readCertificate(file string) (*x509.Certificate, []byte, error) {
 	data, err := os.ReadFile(file)

pkg/patterns/testdata/trust-captured/source-records.txt

@@ -0,0 +1,23 @@
+# The public records the key table cites, one line per record. Columns:
+# url, published, role, lists. published is the UTC time the record was
+# published, read from the record itself: a bug's creation_time, a comment's
+# creation_time, a post's datePublished, a OneCRL record's last_modified.
+# role is status when the record reports the key's status (a compromised
+# key or a removed root), and lists when it only lists the key. lists names
+# the keys the record lists: certs:<capture file> for every row with a
+# certificate in that file, pkh:<pubKeyHash> for the row whose SPKI digest
+# it is, root:<subject common name> for the root row of that name.
+# A row's source is the sorted urls of the records that list it; its
+# reported is the UTC date of the earliest of those whose role is status.
+# The root's two records name it by subject alone: no record captured on
+# 2026-10-26 lists its certificate SHA-256, so its binding to the committed
+# certificate is by name until a root-store record that lists it is captured.
+https://bugzilla.mozilla.org/show_bug.cgi?id=2083449 2026-10-18T12:06:03Z status certs:bugzilla-2083449-certificate-sha256.txt
+https://bugzilla.mozilla.org/show_bug.cgi?id=2083449#c18 2026-10-20T13:06:02Z status root:emSign ECC Root CA - G3
+https://blog.google/security/protecting-chrome-and-android-users-from-compromised-emudhra-intermediate-ca-certificates/ 2026-10-20T19:49:00Z status root:emSign ECC Root CA - G3
+https://firefox.settings.services.mozilla.com/v1/buckets/security-state/collections/onecrl/records/5856b9af-5c14-46e7-b660-04d1cc3f2819 2026-10-21T22:34:01Z lists pkh:4CVMf83CZeLw+KQo2rt1q0rgObSHLWL17lfwh31aikE=
+https://firefox.settings.services.mozilla.com/v1/buckets/security-state/collections/onecrl/records/3f07ab6b-0d52-41a3-9a7b-de945502a3b8 2026-10-21T22:34:01Z lists pkh:Q7/6GdbvZRRR9oQeAZkx6W9vLq7/KKtLbxzkV23icXs=
+https://firefox.settings.services.mozilla.com/v1/buckets/security-state/collections/onecrl/records/5ed2b441-a5bf-4f8e-89af-d01fa292a9b8 2026-10-21T22:34:01Z lists pkh:6+1kdcLC2A/LGZ8/6N1uazzmC3frFdTkLD3sDG9bUQU=
+https://firefox.settings.services.mozilla.com/v1/buckets/security-state/collections/onecrl/records/adaa1e8d-57cb-4cfb-b647-305aa20f1b04 2026-10-21T22:34:01Z lists pkh:XAZgorFkkQ2P9YnBjhRVLp4Owjfc7+LLc21BpIEfgaA=
+https://firefox.settings.services.mozilla.com/v1/buckets/security-state/collections/onecrl/records/f67aff65-7a13-42ef-b818-74bb69053273 2026-10-21T22:34:01Z lists pkh:Y+zv7kMdSVI7wRNsTnLQIGpHcIbK9gbNXPAlaGwQnPg=
+https://firefox.settings.services.mozilla.com/v1/buckets/security-state/collections/onecrl/records/fb4be101-8280-4d4e-866e-fd0213bc3a02 2026-10-21T22:34:01Z lists pkh:Sd01PcK5GaN3BkURfQ5gwWEV59G8JZ7v5zSwDqQ0/l8=

pkg/patterns/trust.go

@@ -94,17 +94,18 @@ var trustKeysJSON []byte
 const trustSchemaVersion = 1
 
 // trustKey is one row of the key table: a public key a published advisory
-// lists, the certificates known for it and the listing. The table is
-// generated from the certificates in testdata/trust by internal/trustgen, and
-// a test recomputes every row from them.
+// lists, the certificates known for it and the public records that report its
+// status. The table is generated from the certificates in testdata/trust and
+// the records in testdata/trust-captured by internal/trustgen, and a test
+// recomputes every row from them.
 type trustKey struct {
 	SPKISHA256   string   `json:"spkiSha256"`             // SHA-256 of the DER SubjectPublicKeyInfo
 	Name         string   `json:"name"`                   // the subject common name of the key's certificates
 	CertSHA256   []string `json:"certSha256"`             // SHA-256 of the DER of each known certificate
 	SPKIDER      string   `json:"spkiDer"`                // the DER SubjectPublicKeyInfo, base64
 	Use          string   `json:"use"`                    // tls, codeSigning or root
-	Source       []string `json:"source"`                 // public pages that list the key
-	Reported     string   `json:"reported"`               // date the listing was reported
+	Source       []string `json:"source"`                 // https URL of each public record that reports the key's status (for the root, its removal from a trust store)
+	Reported     string   `json:"reported"`               // earliest UTC date the source records state for themselves
 	Status       string   `json:"status"`                 // compromised or distrusted
 	DistrustDate string   `json:"distrustDate,omitempty"` // for a distrusted root, the date browsers remove it