fix(global-tenders): give per-source tender health a 24h last-good budget

api/health.js

@@ -1304,59 +1304,69 @@ const SEED_META = {
   },
   spending:         { key: 'seed-meta:economic:spending',          maxStaleMin: 120 },
   globalTenders:    { key: 'seed-meta:economic:global-tenders',   maxStaleMin: 180 },
-  globalTendersSam:             { key: 'seed-meta:economic:global-tenders:sam',              maxStaleMin: 240 }, // 150min request pacing + hourly member gate yields ~180min publishes; 240min leaves one gate of scheduling jitter without raising the 10/day SAM budget.
-  globalTendersTed:             { key: 'seed-meta:economic:global-tenders:ted',              maxStaleMin: 180 },
+  // Per-source tender health: the producer keeps a failed source's last-good
+  // open tenders for 24h (TENDER_SOURCE_RETENTION_MS in scripts/_global-tenders.mjs),
+  // and seed-meta fetchedAt is the last SUCCESS, so a rate-limited, unpublished
+  // or weekend-quiet portal is OK until that success is a day old, then
+  // STALE_SEED. sourceFailureWithinBudgetOk keeps a failed attempt from
+  // reporting SEED_ERROR inside that budget only while the source still serves
+  // something true: retained rows ('stale') or a verified empty result. An
+  // 'error' with nothing retained still warns.
+  globalTendersSam:             { key: 'seed-meta:economic:global-tenders:sam',              maxStaleMin: 1440, sourceFailureWithinBudgetOk: true },
+  globalTendersTed:             { key: 'seed-meta:economic:global-tenders:ted',              maxStaleMin: 1440, sourceFailureWithinBudgetOk: true },
+  // Contracts Finder keeps its bespoke first-failure containment instead
+  // (composeContractsFinderHealth), which recounts usable canonical rows.
   globalTendersContractsFinder: { key: 'seed-meta:economic:global-tenders:contracts-finder', maxStaleMin: 180 },
-  globalTendersCanadaBuys:      { key: 'seed-meta:economic:global-tenders:canada-buys',      maxStaleMin: 180 },
-  globalTendersGets:            { key: 'seed-meta:economic:global-tenders:gets',             maxStaleMin: 180 },
-  globalTendersWorldBank:       { key: 'seed-meta:economic:global-tenders:world-bank',       maxStaleMin: 180 },
+  globalTendersCanadaBuys:      { key: 'seed-meta:economic:global-tenders:canada-buys',      maxStaleMin: 1440, sourceFailureWithinBudgetOk: true },
+  globalTendersGets:            { key: 'seed-meta:economic:global-tenders:gets',             maxStaleMin: 1440, sourceFailureWithinBudgetOk: true },
+  globalTendersWorldBank:       { key: 'seed-meta:economic:global-tenders:world-bank',       maxStaleMin: 1440, sourceFailureWithinBudgetOk: true },
   globalTendersColombiaSecop: {
-    key: 'seed-meta:economic:global-tenders:colombia-secop', maxStaleMin: 180,
+    key: 'seed-meta:economic:global-tenders:colombia-secop', maxStaleMin: 1440, sourceFailureWithinBudgetOk: true,
     activationKey: 'seed-activated:economic:global-tenders:colombia-secop',
     cutover: { mode: 'activation-marker', fromKey: null, issue: 9840, activationKey: 'seed-activated:economic:global-tenders:colombia-secop' },
   },
   globalTendersKenyaPpip: {
-    key: 'seed-meta:economic:global-tenders:kenya-ppip', maxStaleMin: 180,
+    key: 'seed-meta:economic:global-tenders:kenya-ppip', maxStaleMin: 1440, sourceFailureWithinBudgetOk: true,
     activationKey: 'seed-activated:economic:global-tenders:kenya-ppip',
     cutover: { mode: 'activation-marker', fromKey: null, issue: 9840, activationKey: 'seed-activated:economic:global-tenders:kenya-ppip' },
   },
   globalTendersSouthAfricaEtenders: {
-    key: 'seed-meta:economic:global-tenders:south-africa-etenders', maxStaleMin: 180,
+    key: 'seed-meta:economic:global-tenders:south-africa-etenders', maxStaleMin: 1440, sourceFailureWithinBudgetOk: true,
     activationKey: 'seed-activated:economic:global-tenders:south-africa-etenders',
     cutover: { mode: 'activation-marker', fromKey: null, issue: 9840, activationKey: 'seed-activated:economic:global-tenders:south-africa-etenders' },
   },
   globalTendersChileMercadoPublico: {
-    key: 'seed-meta:economic:global-tenders:chile-mercado-publico', maxStaleMin: 180,
+    key: 'seed-meta:economic:global-tenders:chile-mercado-publico', maxStaleMin: 1440, sourceFailureWithinBudgetOk: true,
     activationKey: 'seed-activated:economic:global-tenders:chile-mercado-publico',
     cutover: { mode: 'activation-marker', fromKey: null, issue: 9840, activationKey: 'seed-activated:economic:global-tenders:chile-mercado-publico' },
   },
   globalTendersBrazilPncp: {
-    key: 'seed-meta:economic:global-tenders:brazil-pncp', maxStaleMin: 180,
+    key: 'seed-meta:economic:global-tenders:brazil-pncp', maxStaleMin: 1440, sourceFailureWithinBudgetOk: true,
     activationKey: 'seed-activated:economic:global-tenders:brazil-pncp',
     cutover: { mode: 'activation-marker', fromKey: null, issue: 9840, activationKey: 'seed-activated:economic:global-tenders:brazil-pncp' },
   },
   globalTendersUkraineProzorro: {
-    key: 'seed-meta:economic:global-tenders:ukraine-prozorro', maxStaleMin: 180,
+    key: 'seed-meta:economic:global-tenders:ukraine-prozorro', maxStaleMin: 1440, sourceFailureWithinBudgetOk: true,
     activationKey: 'seed-activated:economic:global-tenders:ukraine-prozorro',
     cutover: { mode: 'activation-marker', fromKey: null, issue: 9840, activationKey: 'seed-activated:economic:global-tenders:ukraine-prozorro' },
   },
   globalTendersBulgariaEop: {
-    key: 'seed-meta:economic:global-tenders:bulgaria-eop', maxStaleMin: 180,
+    key: 'seed-meta:economic:global-tenders:bulgaria-eop', maxStaleMin: 1440, sourceFailureWithinBudgetOk: true,
     activationKey: 'seed-activated:economic:global-tenders:bulgaria-eop',
     cutover: { mode: 'activation-marker', fromKey: null, issue: 9840, activationKey: 'seed-activated:economic:global-tenders:bulgaria-eop' },
   },
   globalTendersLatviaIub: {
-    key: 'seed-meta:economic:global-tenders:latvia-iub', maxStaleMin: 180,
+    key: 'seed-meta:economic:global-tenders:latvia-iub', maxStaleMin: 1440, sourceFailureWithinBudgetOk: true,
     activationKey: 'seed-activated:economic:global-tenders:latvia-iub',
     cutover: { mode: 'activation-marker', fromKey: null, issue: 9840, activationKey: 'seed-activated:economic:global-tenders:latvia-iub' },
   },
   globalTendersPolandBzp: {
-    key: 'seed-meta:economic:global-tenders:poland-bzp', maxStaleMin: 180,
+    key: 'seed-meta:economic:global-tenders:poland-bzp', maxStaleMin: 1440, sourceFailureWithinBudgetOk: true,
     activationKey: 'seed-activated:economic:global-tenders:poland-bzp',
     cutover: { mode: 'activation-marker', fromKey: null, issue: 9840, activationKey: 'seed-activated:economic:global-tenders:poland-bzp' },
   },
   globalTendersPakistanPpra: {
-    key: 'seed-meta:economic:global-tenders:pakistan-ppra', maxStaleMin: 180,
+    key: 'seed-meta:economic:global-tenders:pakistan-ppra', maxStaleMin: 1440, sourceFailureWithinBudgetOk: true,
     activationKey: 'seed-activated:economic:global-tenders:pakistan-ppra',
     cutover: { mode: 'activation-marker', fromKey: null, issue: 9840, activationKey: 'seed-activated:economic:global-tenders:pakistan-ppra' },
   },
@@ -3400,7 +3410,10 @@ function readSeedMeta(seedCfg, keyMetaValues, keyMetaErrors, now) {
     && meta.sourceState !== 'ok'
     && !sourceUnavailable
     && !sourceBlocked
-    && sourceFailure?.pending !== true);
+    && sourceFailure?.pending !== true
+    && !(seedCfg.sourceFailureWithinBudgetOk === true && seedStale === false
+      && ((meta.sourceState === 'stale' && meta.recordCount > 0)
+        || (meta.sourceState === 'error' && meta.confirmedEmpty === true))));
   // Content-age trio (2026-05-04 health-readiness plan). Presence of
   // maxContentAgeMin is the opt-in signal — legacy seeders without it
   // get contentAge: null and skip the STALE_CONTENT branch in classifyKey.

scripts/_global-tender-feeds.mjs

@@ -202,8 +202,18 @@ export async function fetchBulgaria({ now = Date.now(), fetchJsonFn, fetchTextFn
   const source = 'bulgaria-eop';
   const read = reader(fetchJsonFn);
   const readText = reader(fetchTextFn);
-  const base = `https://storage.eop.bg/open-data-${isoDay(now - DAY)}/`;
-  const listing = new XMLParser().parse(await readText(base));
+  // Day D's bucket appears around 06:00 UTC on D+1 and answers 403 until then,
+  // so the early-UTC runs read the day before instead of failing.
+  let base = `https://storage.eop.bg/open-data-${isoDay(now - DAY)}/`;
+  let listingXml;
+  try {
+    listingXml = await readText(base);
+  } catch (error) {
+    if (![403, 404].includes(error?.status)) throw error;
+    base = `https://storage.eop.bg/open-data-${isoDay(now - 2 * DAY)}/`;
+    listingXml = await readText(base);
+  }
+  const listing = new XMLParser().parse(listingXml);
   const contents = listing?.ListBucketResult?.Contents;
   const files = contents ? (Array.isArray(contents) ? contents : [contents]) : [];
   const key = files.map((entry) => entry.Key).find((name) => typeof name === 'string' && name.includes('OCDS') && name.endsWith('.json'));

scripts/_global-tenders.mjs

@@ -30,6 +30,11 @@ const OFFICIAL_SOURCE_HOSTS = {
   'pakistan-ppra': ['epms.ppra.gov.pk'],
 };
 
+// A failed source keeps its last-good open tenders for a day: procurement
+// portals rate-limit, publish daily, and go quiet over weekends, and a
+// day-old open tender is still actionable until its own deadline.
+export const TENDER_SOURCE_RETENTION_MS = 24 * 60 * 60_000;
+
 export function tracksTenderFreshness(source) {
   return ['contracts-finder', 'world-bank', 'colombia-secop', 'kenya-ppip', 'south-africa-etenders',
     'chile-mercado-publico', 'brazil-pncp', 'ukraine-prozorro', 'bulgaria-eop', 'latvia-iub',
@@ -278,7 +283,7 @@ export function mergeTenderSourceResults({ settled, sourceNames, previousSnapsho
       const lastSuccessfulAt = firstString(priorStatus?.lastSuccessfulAt,
         priorStatus?.state === 'ok' ? priorStatus.fetchedAt : '');
       const successMs = Date.parse(lastSuccessfulAt);
-      const fresh = successMs > 0 && successMs <= attemptedAtMs && attemptedAtMs < successMs + 180 * 60_000;
+      const fresh = successMs > 0 && successMs <= attemptedAtMs && attemptedAtMs < successMs + TENDER_SOURCE_RETENTION_MS;
       const retained = fresh ? priorRecords.filter((tender) =>
         string(tender.id) && string(tender.title) && safeOfficialUrl(tender.officialUrl, source)
         && ['active', 'open'].includes(tender.status)

scripts/seed-global-tenders.mjs

@@ -24,7 +24,10 @@ import {
 } from './_global-tenders.mjs';
 
 const CACHE_TTL_SECONDS = 10_800; // 3h, safely beyond the hourly Railway cadence.
-const SOURCE_STATUS_TTL_SECONDS = CACHE_TTL_SECONDS;
+// Per-source status must outlive the 24h last-good budget health grades it
+// against; 26h leaves two hourly ticks for the key to age into STALE_SEED
+// before it expires into EMPTY.
+const SOURCE_STATUS_TTL_SECONDS = 26 * 60 * 60;
 const MAX_PER_SOURCE = 100;
 const GETS_FEED_URL = 'https://www.gets.govt.nz/ExternalRSSFeed.htm';
 const CANADA_BUYS_OPEN_CSV_URL = 'https://canadabuys.canada.ca/opendata/pub/openTenderNotice-ouvertAvisAppelOffres.csv';

tests/global-tender-health.test.mjs

@@ -22,7 +22,8 @@ test('new tender sources become strict after their first successful publication'
     assert.equal(classify(false, false).status, 'EMPTY_ON_DEMAND', name);
     assert.equal(classify(true, false).status, 'EMPTY', name);
     assert.equal(classify(true, true).status, 'OK', name);
-    assert.equal(classify(true, true, 181).status, 'STALE_SEED', name);
+    assert.equal(classify(true, true, 1439).status, 'OK', name);
+    assert.equal(classify(true, true, 1441).status, 'STALE_SEED', name);
   }
 });
 
@@ -56,37 +57,38 @@ test('health registers and classifies per-source global tender freshness', () =>
     now,
   });
 
-  assert.equal(entry.status, 'SEED_ERROR');
+  assert.equal(entry.status, 'OK', 'retained rows inside the 24h source budget are not an alarm');
   assert.equal(entry.records, 12);
 });
 
-test('SAM health allows the effective paced cadence plus one hourly jitter gate', () => {
+test('every per-source tender check grades a failed attempt against a 24h last-success budget', () => {
   const { classifyKey, SEED_META, STANDALONE_KEYS } = __testing__;
-  const name = 'globalTendersSam';
-  const dataKey = STANDALONE_KEYS[name];
-  const metaKey = SEED_META[name].key;
   const now = Date.parse('2026-07-22T12:00:00Z');
+  // Contracts Finder keeps its bespoke 180min first-failure containment (tests below).
+  assert.equal(SEED_META.globalTendersContractsFinder.maxStaleMin, 180);
+  assert.equal(SEED_META.globalTendersContractsFinder.sourceFailureWithinBudgetOk, undefined);
+  const sources = ['Sam', 'Ted', 'CanadaBuys', 'Gets', 'WorldBank',
+    'ColombiaSecop', 'KenyaPpip', 'SouthAfricaEtenders', 'ChileMercadoPublico', 'BrazilPncp',
+    'UkraineProzorro', 'BulgariaEop', 'LatviaIub', 'PolandBzp', 'PakistanPpra'];
 
-  assert.equal(SEED_META.globalTendersSam.maxStaleMin, 240);
-  for (const source of ['Ted', 'ContractsFinder', 'CanadaBuys', 'Gets', 'WorldBank']) {
-    assert.equal(SEED_META[`globalTenders${source}`].maxStaleMin, 180, `${source} keeps the hourly source SLA`);
+  for (const source of sources) {
+    const name = `globalTenders${source}`;
+    assert.equal(SEED_META[name].maxStaleMin, 1440, name);
+    assert.equal(SEED_META[name].sourceFailureWithinBudgetOk, true, name);
+    const dataKey = STANDALONE_KEYS[name];
+    const classify = (ageMin, meta) => classifyKey(name, dataKey, { allowOnDemand: true }, {
+      now, activationStates: new Map([[name, true]]),
+      keyStrens: new Map([[dataKey, 128]]), keyErrors: new Map(), keyMetaErrors: new Map(),
+      keyMetaValues: new Map([[SEED_META[name].key, JSON.stringify({ fetchedAt: now - ageMin * 60_000, ...meta })]]),
+    });
+    const retained = { recordCount: 3, sourceState: 'stale', stale: true, error: 'HTTP 429' };
+    assert.equal(classify(1439, retained).status, 'OK', `${name}: rate-limited source still serving last-good rows`);
+    assert.notEqual(classify(1441, retained).status, 'OK', `${name}: a day without a success warns`);
+    assert.equal(classify(60, { recordCount: 0, sourceState: 'error', stale: true, confirmedEmpty: true }).status, 'OK',
+      `${name}: a verified empty result stays true while the next attempt fails`);
+    assert.notEqual(classify(60, { recordCount: 0, sourceState: 'error', stale: true, error: 'HTTP 403' }).status, 'OK',
+      `${name}: a failure serving nothing still warns`);
   }
-
-  const classifyAtAge = (ageMin) => classifyKey(name, dataKey, { allowOnDemand: true }, {
-    keyStrens: new Map([[dataKey, 128]]),
-    keyErrors: new Map(),
-    keyMetaValues: new Map([[metaKey, JSON.stringify({
-      fetchedAt: now - ageMin * 60_000,
-      recordCount: 1,
-      sourceState: 'ok',
-      stale: false,
-    })]]),
-    keyMetaErrors: new Map(),
-    now,
-  });
-
-  assert.equal(classifyAtAge(181).status, 'OK', 'normal post-180min scheduler jitter must not false-alarm');
-  assert.equal(classifyAtAge(241).status, 'STALE_SEED', 'a genuinely missed paced refresh must still warn');
 });
 
 // An adapter the deployment never opted into is not a fault. fetchSam writes

tests/global-tenders-additional-feeds.test.mjs

@@ -181,6 +181,30 @@ test('Bulgaria reads the previous daily OCDS file and keeps lot deadlines separa
   assert.equal(result.records[0].deadline, '2026-11-09T10:00:00.000Z');
 });
 
+test('Bulgaria falls back one day while the previous daily bucket is unpublished', async () => {
+  // storage.eop.bg publishes day D's bucket around 06:00 UTC on D+1 and answers
+  // 403 AccessDenied until then, so every UTC morning would otherwise fail.
+  const calls = [];
+  const result = await fetchBulgaria({ now: NOW, fetchTextFn: async (url) => {
+    calls.push(String(url));
+    if (String(url).includes('open-data-2026-10-28/')) throw Object.assign(new Error('HTTP 403'), { status: 403 });
+    return '<ListBucketResult><Contents><Key>notices OCDS.json</Key></Contents></ListBucketResult>';
+  }, fetchJsonFn: async (url) => {
+    calls.push(String(url));
+    return { releases: [release({ tender: { id: '1', title: 'Water supply', status: 'active', lots: [
+      { id: 'LOT-1', tenderPeriod: { endDate: END } },
+    ] } })] };
+  } });
+  assert.match(calls[0], /open-data-2026-10-28\/$/);
+  assert.match(calls[1], /open-data-2026-10-27\/$/);
+  assert.match(calls[2], /open-data-2026-10-27\/notices%20OCDS.json$/);
+  assert.equal(result.records.length, 1);
+
+  await assert.rejects(fetchBulgaria({ now: NOW, fetchTextFn: async () => {
+    throw Object.assign(new Error('HTTP 500'), { status: 500 });
+  }, fetchJsonFn: async () => assert.fail('no listing, no file read') }), /HTTP 500/);
+});
+
 test('Latvia keeps competition lots, resolves Riga winter time, and drops contact points', async () => {
   const result = await fetchLatvia({ now: NOW, fetchJsonFn: async () => [{
     identifier: 'notice-1', name: 'Water supply', formType: 'competition',
@@ -241,7 +265,7 @@ test('each added source accepts valid empty responses and has a strict official
   }
 });
 
-test('added sources retain last-good records only within their own three-hour success clock', async () => {
+test('added sources retain last-good records only within their own 24-hour success clock', async () => {
   const record = (await fetchKenya({ now: NOW, fetchJsonFn: async () => ({ data: [{ id: 1, title: 'Water supply',
     status_id: 1, terminated: 0, published: 1, close_at: '2026-11-09 10:00:00' }] }) })).records[0];
   const prior = { fetchedAt: NOW, dataAvailable: true, tenders: [record], sourceStatuses: [{
@@ -251,7 +275,10 @@ test('added sources retain last-good records only within their own three-hour su
     settled: [{ status: 'rejected', reason: new Error('HTTP 429') }], attemptedAt: new Date(now).toISOString() });
   const retained = failed(NOW + 60_000, prior);
   assert.equal(retained.sourceStatuses[0].state, 'stale');
-  const expired = failed(NOW + 180 * 60_000, retained);
+  const lateDay = failed(NOW + (24 * 60 - 1) * 60_000, retained);
+  assert.equal(lateDay.sourceStatuses[0].state, 'stale');
+  assert.equal(lateDay.tenders.length, 1);
+  const expired = failed(NOW + 24 * 60 * 60_000, lateDay);
   assert.equal(expired.tenders.length, 0);
   assert.equal(expired.sourceStatuses[0].state, 'error');
   assert.equal(expired.sourceStatuses[0].lastSuccessfulAt, new Date(NOW).toISOString());