From 9e9a9fe50bfe5f3893c11e85f30c4022206a2565 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 25 Aug 2025 15:03:33 +0000 Subject: [PATCH 1/9] Initial plan From de9e40f7f00f569c0c78c9ff61de2464d3343edc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 25 Aug 2025 15:11:12 +0000 Subject: [PATCH 2/9] Implement daily_miner_deals_checked table and functionality Co-authored-by: bajtos <1140553+bajtos@users.noreply.github.com> --- lib/public-stats.js | 64 +++++++++ .../026.do.daily-miner-deals-checked.sql | 6 + test/public-stats.test.js | 128 ++++++++++++++++++ 3 files changed, 198 insertions(+) create mode 100644 migrations/026.do.daily-miner-deals-checked.sql diff --git a/lib/public-stats.js b/lib/public-stats.js index f595c142..47c42f59 100644 --- a/lib/public-stats.js +++ b/lib/public-stats.js @@ -29,6 +29,7 @@ export const updatePublicStats = async ({ createPgClient, committees, allMeasure await updateRetrievalTimings(pgClient, committees) await updateDailyClientRetrievalStats(pgClient, committees, findDealClients) await updateDailyAllocatorRetrievalStats(pgClient, committees, findDealAllocators) + await updateDailyMinerDealsChecked(pgClient, committees) } finally { await pgClient.end() } @@ -394,3 +395,66 @@ function buildPerPartyStats (committees, perDealParty, partyName) { ) return flatStats } + +/** + * @param {pg.Client} pgClient + * @param {Iterable} committees + */ +const updateDailyMinerDealsChecked = async (pgClient, committees) => { + /** @type {Map>} */ + const minerPayloadCids = new Map() + + for (const c of committees) { + const { minerId, cid } = c.retrievalTask + + let payloadCids = minerPayloadCids.get(minerId) + if (!payloadCids) { + payloadCids = new Set() + minerPayloadCids.set(minerId, payloadCids) + } + + payloadCids.add(cid) + } + + // Convert the map to arrays for the query + const flatStats = Array.from(minerPayloadCids.entries()).map( + ([miner_id, payloadCids]) => ({ + miner_id, + payload_cids: Array.from(payloadCids) + }) + ) + + if (debug.enabled) { + debug( + 'Updating public stats - daily miner deals checked: miners count = %s, total payload CIDs = %s', + flatStats.length, + flatStats.reduce((sum, stat) => sum + stat.payload_cids.length, 0) + ) + } + + if (flatStats.length === 0) { + debug('No miner deals to record') + return + } + + await pgClient.query(` + INSERT INTO daily_miner_deals_checked ( + day, + miner_id, + payload_cids + ) + SELECT + now(), + miner_id, + payload_cids + FROM jsonb_to_recordset($1::jsonb) AS t (miner_id text, payload_cids text[]) + ON CONFLICT(day, miner_id) DO UPDATE SET + payload_cids = array( + SELECT DISTINCT unnest( + array_cat(daily_miner_deals_checked.payload_cids, EXCLUDED.payload_cids) + ) + ) + `, [ + JSON.stringify(flatStats) + ]) +} diff --git a/migrations/026.do.daily-miner-deals-checked.sql b/migrations/026.do.daily-miner-deals-checked.sql new file mode 100644 index 00000000..ec84a9eb --- /dev/null +++ b/migrations/026.do.daily-miner-deals-checked.sql @@ -0,0 +1,6 @@ +CREATE TABLE daily_miner_deals_checked ( + day DATE NOT NULL, + miner_id TEXT NOT NULL, + payload_cids TEXT[] NOT NULL, + PRIMARY KEY (day, miner_id) +); \ No newline at end of file diff --git a/test/public-stats.test.js b/test/public-stats.test.js index 2b61d693..f965c274 100644 --- a/test/public-stats.test.js +++ b/test/public-stats.test.js @@ -31,6 +31,7 @@ describe('public-stats', () => { await pgClient.query('DELETE FROM retrieval_timings') await pgClient.query('DELETE FROM daily_client_retrieval_stats') await pgClient.query('DELETE FROM daily_allocator_retrieval_stats') + await pgClient.query('DELETE FROM daily_miner_deals_checked') // Run all tests inside a transaction to ensure `now()` always returns the same value // See https://dba.stackexchange.com/a/63549/125312 @@ -1170,6 +1171,133 @@ describe('public-stats', () => { }) }) + describe('updateDailyMinerDealsChecked', () => { + it('collects payload CIDs per miner', async () => { + /** @type {Measurement[]} */ + const honestMeasurements = [ + { ...VALID_MEASUREMENT, minerId: 'f1first', cid: 'cidone' }, + { ...VALID_MEASUREMENT, minerId: 'f1first', cid: 'cidtwo' }, + { ...VALID_MEASUREMENT, minerId: 'f1second', cid: 'cidone' }, + { ...VALID_MEASUREMENT, minerId: 'f1second', cid: 'cidthree' } + ] + const allMeasurements = honestMeasurements + const committees = buildEvaluatedCommitteesFromMeasurements(honestMeasurements) + + await updatePublicStats({ + createPgClient, + committees, + allMeasurements, + findDealClients: (_minerId, _cid) => ['f0client'], + findDealAllocators: (_minerId, _cid) => ['f0allocator'] + }) + + const { rows: created } = await pgClient.query( + 'SELECT day::TEXT, miner_id, payload_cids FROM daily_miner_deals_checked ORDER BY miner_id' + ) + assert.deepStrictEqual(created, [ + { day: today, miner_id: 'f1first', payload_cids: ['cidone', 'cidtwo'] }, + { day: today, miner_id: 'f1second', payload_cids: ['cidone', 'cidthree'] } + ]) + }) + + it('handles duplicate CIDs correctly', async () => { + /** @type {Measurement[]} */ + const honestMeasurements = [ + { ...VALID_MEASUREMENT, minerId: 'f1miner', cid: 'cidone' }, + { ...VALID_MEASUREMENT, minerId: 'f1miner', cid: 'cidone' }, // duplicate + { ...VALID_MEASUREMENT, minerId: 'f1miner', cid: 'cidtwo' } + ] + const allMeasurements = honestMeasurements + const committees = buildEvaluatedCommitteesFromMeasurements(honestMeasurements) + + await updatePublicStats({ + createPgClient, + committees, + allMeasurements, + findDealClients: (_minerId, _cid) => ['f0client'], + findDealAllocators: (_minerId, _cid) => ['f0allocator'] + }) + + const { rows: created } = await pgClient.query( + 'SELECT day::TEXT, miner_id, payload_cids FROM daily_miner_deals_checked' + ) + assert.deepStrictEqual(created, [ + { day: today, miner_id: 'f1miner', payload_cids: ['cidone', 'cidtwo'] } + ]) + }) + + it('updates existing records by merging CID arrays', async () => { + // First, create an initial record + { + /** @type {Measurement[]} */ + const honestMeasurements = [ + { ...VALID_MEASUREMENT, minerId: 'f1miner', cid: 'cidone' } + ] + const allMeasurements = honestMeasurements + const committees = buildEvaluatedCommitteesFromMeasurements(honestMeasurements) + + await updatePublicStats({ + createPgClient, + committees, + allMeasurements, + findDealClients: (_minerId, _cid) => ['f0client'], + findDealAllocators: (_minerId, _cid) => ['f0allocator'] + }) + + const { rows: created } = await pgClient.query( + 'SELECT day::TEXT, miner_id, payload_cids FROM daily_miner_deals_checked' + ) + assert.deepStrictEqual(created, [ + { day: today, miner_id: 'f1miner', payload_cids: ['cidone'] } + ]) + } + + // Now update with additional CIDs + { + /** @type {Measurement[]} */ + const honestMeasurements = [ + { ...VALID_MEASUREMENT, minerId: 'f1miner', cid: 'cidone' }, // duplicate - should be ignored + { ...VALID_MEASUREMENT, minerId: 'f1miner', cid: 'cidtwo' } // new CID + ] + const allMeasurements = honestMeasurements + const committees = buildEvaluatedCommitteesFromMeasurements(honestMeasurements) + + await updatePublicStats({ + createPgClient, + committees, + allMeasurements, + findDealClients: (_minerId, _cid) => ['f0client'], + findDealAllocators: (_minerId, _cid) => ['f0allocator'] + }) + + const { rows: updated } = await pgClient.query( + 'SELECT day::TEXT, miner_id, payload_cids FROM daily_miner_deals_checked' + ) + assert.deepStrictEqual(updated, [ + { day: today, miner_id: 'f1miner', payload_cids: ['cidone', 'cidtwo'] } + ]) + } + }) + + it('handles empty committees gracefully', async () => { + const committees = [] + const allMeasurements = [] + + await updatePublicStats({ + createPgClient, + committees, + allMeasurements, + findDealClients: (_minerId, _cid) => ['f0client'], + findDealAllocators: (_minerId, _cid) => ['f0allocator'] + }) + + const { rows: created } = await pgClient.query( + 'SELECT * FROM daily_miner_deals_checked' + ) + assert.deepStrictEqual(created, []) + }) + }) + const getCurrentDate = async () => { const { rows: [{ today }] } = await pgClient.query('SELECT now()::DATE::TEXT as today') return today From 5fb661de82d52d34cfc2ebf9d48e0bb37ef1f559 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 25 Aug 2025 15:13:05 +0000 Subject: [PATCH 3/9] Fix linting issues in public-stats.js Co-authored-by: bajtos <1140553+bajtos@users.noreply.github.com> --- lib/public-stats.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/public-stats.js b/lib/public-stats.js index 47c42f59..1964cac5 100644 --- a/lib/public-stats.js +++ b/lib/public-stats.js @@ -406,20 +406,21 @@ const updateDailyMinerDealsChecked = async (pgClient, committees) => { for (const c of committees) { const { minerId, cid } = c.retrievalTask - + let payloadCids = minerPayloadCids.get(minerId) if (!payloadCids) { payloadCids = new Set() minerPayloadCids.set(minerId, payloadCids) } - + payloadCids.add(cid) } // Convert the map to arrays for the query const flatStats = Array.from(minerPayloadCids.entries()).map( - ([miner_id, payloadCids]) => ({ - miner_id, + // eslint-disable-next-line camelcase + ([minerId, payloadCids]) => ({ + miner_id: minerId, payload_cids: Array.from(payloadCids) }) ) From d0326615c91fa6bd378d68bcc60b15d921981bce Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 25 Aug 2025 15:24:52 +0000 Subject: [PATCH 4/9] Use unnest approach instead of jsonb_to_recordset in updateDailyMinerDealsChecked Co-authored-by: bajtos <1140553+bajtos@users.noreply.github.com> --- lib/public-stats.js | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/public-stats.js b/lib/public-stats.js index 1964cac5..526b3f68 100644 --- a/lib/public-stats.js +++ b/lib/public-stats.js @@ -417,23 +417,23 @@ const updateDailyMinerDealsChecked = async (pgClient, committees) => { } // Convert the map to arrays for the query - const flatStats = Array.from(minerPayloadCids.entries()).map( - // eslint-disable-next-line camelcase - ([minerId, payloadCids]) => ({ - miner_id: minerId, - payload_cids: Array.from(payloadCids) - }) - ) + const minerIds = [] + const payloadCidsArrays = [] + + for (const [minerId, payloadCids] of minerPayloadCids.entries()) { + minerIds.push(minerId) + payloadCidsArrays.push(Array.from(payloadCids)) + } if (debug.enabled) { debug( 'Updating public stats - daily miner deals checked: miners count = %s, total payload CIDs = %s', - flatStats.length, - flatStats.reduce((sum, stat) => sum + stat.payload_cids.length, 0) + minerIds.length, + payloadCidsArrays.reduce((sum, cids) => sum + cids.length, 0) ) } - if (flatStats.length === 0) { + if (minerIds.length === 0) { debug('No miner deals to record') return } @@ -443,12 +443,11 @@ const updateDailyMinerDealsChecked = async (pgClient, committees) => { day, miner_id, payload_cids - ) - SELECT + ) VALUES ( now(), - miner_id, - payload_cids - FROM jsonb_to_recordset($1::jsonb) AS t (miner_id text, payload_cids text[]) + unnest($1::text[]), + unnest($2::text[][]) + ) ON CONFLICT(day, miner_id) DO UPDATE SET payload_cids = array( SELECT DISTINCT unnest( @@ -456,6 +455,7 @@ const updateDailyMinerDealsChecked = async (pgClient, committees) => { ) ) `, [ - JSON.stringify(flatStats) + minerIds, + payloadCidsArrays ]) } From 7c71e9867a19e1fbb6f84597f25dcb077309d2d7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 26 Aug 2025 17:15:01 +0000 Subject: [PATCH 5/9] Fix type error in updateDailyMinerDealsChecked function Co-authored-by: bajtos <1140553+bajtos@users.noreply.github.com> --- lib/public-stats.js | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/lib/public-stats.js b/lib/public-stats.js index 526b3f68..edc47f94 100644 --- a/lib/public-stats.js +++ b/lib/public-stats.js @@ -438,24 +438,26 @@ const updateDailyMinerDealsChecked = async (pgClient, committees) => { return } - await pgClient.query(` - INSERT INTO daily_miner_deals_checked ( - day, - miner_id, - payload_cids - ) VALUES ( - now(), - unnest($1::text[]), - unnest($2::text[][]) - ) - ON CONFLICT(day, miner_id) DO UPDATE SET - payload_cids = array( - SELECT DISTINCT unnest( - array_cat(daily_miner_deals_checked.payload_cids, EXCLUDED.payload_cids) - ) + for (let i = 0; i < minerIds.length; i++) { + const minerId = minerIds[i] + const payloadCids = payloadCidsArrays[i] + + await pgClient.query(` + INSERT INTO daily_miner_deals_checked ( + day, + miner_id, + payload_cids + ) VALUES ( + now(), + $1, + $2 ) - `, [ - minerIds, - payloadCidsArrays - ]) + ON CONFLICT(day, miner_id) DO UPDATE SET + payload_cids = array( + SELECT DISTINCT unnest( + array_cat(daily_miner_deals_checked.payload_cids, EXCLUDED.payload_cids) + ) + ) + `, [minerId, payloadCids]) + } } From 040177a0d22d5872cecb322d9465c7640eebe80f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 26 Aug 2025 17:29:47 +0000 Subject: [PATCH 6/9] Fix updateDailyMinerDealsChecked to use single query with unnest approach Co-authored-by: bajtos <1140553+bajtos@users.noreply.github.com> --- lib/public-stats.js | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/lib/public-stats.js b/lib/public-stats.js index edc47f94..6744c59a 100644 --- a/lib/public-stats.js +++ b/lib/public-stats.js @@ -438,26 +438,26 @@ const updateDailyMinerDealsChecked = async (pgClient, committees) => { return } - for (let i = 0; i < minerIds.length; i++) { - const minerId = minerIds[i] - const payloadCids = payloadCidsArrays[i] - - await pgClient.query(` - INSERT INTO daily_miner_deals_checked ( - day, - miner_id, - payload_cids - ) VALUES ( - now(), - $1, - $2 - ) - ON CONFLICT(day, miner_id) DO UPDATE SET - payload_cids = array( - SELECT DISTINCT unnest( - array_cat(daily_miner_deals_checked.payload_cids, EXCLUDED.payload_cids) - ) + // Use unnest with ORDINALITY to match arrays by position + await pgClient.query(` + INSERT INTO daily_miner_deals_checked ( + day, + miner_id, + payload_cids + ) + SELECT + now(), + miners.miner_id, + arrays.payload_cids + FROM + unnest($1::text[]) WITH ORDINALITY AS miners(miner_id, pos), + unnest($2::text[][]) WITH ORDINALITY AS arrays(payload_cids, pos) + WHERE miners.pos = arrays.pos + ON CONFLICT(day, miner_id) DO UPDATE SET + payload_cids = array( + SELECT DISTINCT unnest( + array_cat(daily_miner_deals_checked.payload_cids, EXCLUDED.payload_cids) ) - `, [minerId, payloadCids]) - } + ) + `, [minerIds, payloadCidsArrays]) } From 4ae7c17e31b89d0eedae4526d93e22b2efd232fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Fri, 29 Aug 2025 17:07:25 +0200 Subject: [PATCH 7/9] Revert "Fix updateDailyMinerDealsChecked to use single query with unnest approach" This reverts commit 040177a0d22d5872cecb322d9465c7640eebe80f. --- lib/public-stats.js | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/lib/public-stats.js b/lib/public-stats.js index 6744c59a..edc47f94 100644 --- a/lib/public-stats.js +++ b/lib/public-stats.js @@ -438,26 +438,26 @@ const updateDailyMinerDealsChecked = async (pgClient, committees) => { return } - // Use unnest with ORDINALITY to match arrays by position - await pgClient.query(` - INSERT INTO daily_miner_deals_checked ( - day, - miner_id, - payload_cids - ) - SELECT - now(), - miners.miner_id, - arrays.payload_cids - FROM - unnest($1::text[]) WITH ORDINALITY AS miners(miner_id, pos), - unnest($2::text[][]) WITH ORDINALITY AS arrays(payload_cids, pos) - WHERE miners.pos = arrays.pos - ON CONFLICT(day, miner_id) DO UPDATE SET - payload_cids = array( - SELECT DISTINCT unnest( - array_cat(daily_miner_deals_checked.payload_cids, EXCLUDED.payload_cids) - ) + for (let i = 0; i < minerIds.length; i++) { + const minerId = minerIds[i] + const payloadCids = payloadCidsArrays[i] + + await pgClient.query(` + INSERT INTO daily_miner_deals_checked ( + day, + miner_id, + payload_cids + ) VALUES ( + now(), + $1, + $2 ) - `, [minerIds, payloadCidsArrays]) + ON CONFLICT(day, miner_id) DO UPDATE SET + payload_cids = array( + SELECT DISTINCT unnest( + array_cat(daily_miner_deals_checked.payload_cids, EXCLUDED.payload_cids) + ) + ) + `, [minerId, payloadCids]) + } } From b4d0c942c5a95d976a6b54e446f3fc2851cf4d3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Fri, 29 Aug 2025 17:07:27 +0200 Subject: [PATCH 8/9] Revert "Fix type error in updateDailyMinerDealsChecked function" This reverts commit 7c71e9867a19e1fbb6f84597f25dcb077309d2d7. --- lib/public-stats.js | 40 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/lib/public-stats.js b/lib/public-stats.js index edc47f94..526b3f68 100644 --- a/lib/public-stats.js +++ b/lib/public-stats.js @@ -438,26 +438,24 @@ const updateDailyMinerDealsChecked = async (pgClient, committees) => { return } - for (let i = 0; i < minerIds.length; i++) { - const minerId = minerIds[i] - const payloadCids = payloadCidsArrays[i] - - await pgClient.query(` - INSERT INTO daily_miner_deals_checked ( - day, - miner_id, - payload_cids - ) VALUES ( - now(), - $1, - $2 - ) - ON CONFLICT(day, miner_id) DO UPDATE SET - payload_cids = array( - SELECT DISTINCT unnest( - array_cat(daily_miner_deals_checked.payload_cids, EXCLUDED.payload_cids) - ) + await pgClient.query(` + INSERT INTO daily_miner_deals_checked ( + day, + miner_id, + payload_cids + ) VALUES ( + now(), + unnest($1::text[]), + unnest($2::text[][]) + ) + ON CONFLICT(day, miner_id) DO UPDATE SET + payload_cids = array( + SELECT DISTINCT unnest( + array_cat(daily_miner_deals_checked.payload_cids, EXCLUDED.payload_cids) ) - `, [minerId, payloadCids]) - } + ) + `, [ + minerIds, + payloadCidsArrays + ]) } From 33f7221b0ba9f098d9d1c6cab6d51e894c951d1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Fri, 29 Aug 2025 17:07:29 +0200 Subject: [PATCH 9/9] Revert "Use unnest approach instead of jsonb_to_recordset in updateDailyMinerDealsChecked" This reverts commit d0326615c91fa6bd378d68bcc60b15d921981bce. --- lib/public-stats.js | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/public-stats.js b/lib/public-stats.js index 526b3f68..1964cac5 100644 --- a/lib/public-stats.js +++ b/lib/public-stats.js @@ -417,23 +417,23 @@ const updateDailyMinerDealsChecked = async (pgClient, committees) => { } // Convert the map to arrays for the query - const minerIds = [] - const payloadCidsArrays = [] - - for (const [minerId, payloadCids] of minerPayloadCids.entries()) { - minerIds.push(minerId) - payloadCidsArrays.push(Array.from(payloadCids)) - } + const flatStats = Array.from(minerPayloadCids.entries()).map( + // eslint-disable-next-line camelcase + ([minerId, payloadCids]) => ({ + miner_id: minerId, + payload_cids: Array.from(payloadCids) + }) + ) if (debug.enabled) { debug( 'Updating public stats - daily miner deals checked: miners count = %s, total payload CIDs = %s', - minerIds.length, - payloadCidsArrays.reduce((sum, cids) => sum + cids.length, 0) + flatStats.length, + flatStats.reduce((sum, stat) => sum + stat.payload_cids.length, 0) ) } - if (minerIds.length === 0) { + if (flatStats.length === 0) { debug('No miner deals to record') return } @@ -443,11 +443,12 @@ const updateDailyMinerDealsChecked = async (pgClient, committees) => { day, miner_id, payload_cids - ) VALUES ( - now(), - unnest($1::text[]), - unnest($2::text[][]) ) + SELECT + now(), + miner_id, + payload_cids + FROM jsonb_to_recordset($1::jsonb) AS t (miner_id text, payload_cids text[]) ON CONFLICT(day, miner_id) DO UPDATE SET payload_cids = array( SELECT DISTINCT unnest( @@ -455,7 +456,6 @@ const updateDailyMinerDealsChecked = async (pgClient, committees) => { ) ) `, [ - minerIds, - payloadCidsArrays + JSON.stringify(flatStats) ]) }