From 2ce18adb3b0551d765d8be3479bd3e9e9541b1dc Mon Sep 17 00:00:00 2001 From: Alex Anderson <191496+alxndrsn@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:05:17 +0300 Subject: [PATCH 1/7] nginx: buffer responses in nginx, not application layer (#2036) Closes #2029 --- files/nginx/odk.conf.template | 3 +- test/nginx/mock-http-server/index.js | 52 ++++++++++++++++++++ test/nginx/src/lib.js | 5 ++ test/nginx/src/mocha/nginx.spec.js | 73 ++++++++++++++++++++++++++++ 4 files changed, 131 insertions(+), 2 deletions(-) diff --git a/files/nginx/odk.conf.template b/files/nginx/odk.conf.template index 702a4c407..d4e476033 100644 --- a/files/nginx/odk.conf.template +++ b/files/nginx/odk.conf.template @@ -209,9 +209,8 @@ server { proxy_pass http://service:8383; proxy_redirect off; - # buffer requests, but not responses, so streaming out works. proxy_request_buffering on; - proxy_buffering off; + proxy_buffering on; proxy_read_timeout 2m; } diff --git a/test/nginx/mock-http-server/index.js b/test/nginx/mock-http-server/index.js index 053a6e105..707878f2e 100644 --- a/test/nginx/mock-http-server/index.js +++ b/test/nginx/mock-http-server/index.js @@ -1,9 +1,13 @@ +const { Readable } = require('node:stream'); + const express = require('express'); const port = process.env.PORT || 80; const log = (...args) => console.log('[mock-http-server]', ...args); const requests = []; +let openProcessorCount = 0; +let completedProcessorCount = 0; const app = express(); app.set('case sensitive routing', true); @@ -29,9 +33,57 @@ app.get('/__mock_http_server/health', (req, res) => res.send('OK')); app.get('/__mock_http_server/request-log', (req, res) => res.json(requests)); app.get('/__mock_http_server/reset', (req, res) => { requests.length = 0; + openProcessorCount = 0; + completedProcessorCount = 0; res.json('OK'); }); +app.get(new RegExp('^/v1/.*/100MB\\.csv$'), (req, res) => { + const csvSizeBytes = 100_000_000; + + res.set('Content-Disposition', `attachment; filename="100MB.csv"; filename*=UTF-8''100MB.csv`); + res.set('Content-Type', 'text/csv; charset=utf-8'); + + ++openProcessorCount; + + async function* generateCsv(targetByteLength) { + let rowCount = 0; + let totalWritten = 0; + + const batchSize = Math.pow(2, 18); + + const header = Buffer.from('row_number,timestamp,random-number\n', 'utf8'); + totalWritten += header.byteLength; + yield header; + + while(totalWritten < targetByteLength) { + await new Promise(resolve => setTimeout(resolve, 1)); + + const batch = Buffer.allocUnsafe(Math.min(batchSize, targetByteLength - totalWritten)); + let bufpos = 0; + while(bufpos < batch.length) { + const line = `${++rowCount},${new Date().toISOString()},${Math.random()}\n`; + const bytesWritten = batch.write(line, bufpos, batch.length-bufpos, 'utf8'); + bufpos += bytesWritten; + totalWritten += bytesWritten; + } + yield batch; + } + + ++completedProcessorCount; + } + + const randomStream = Readable.from(generateCsv(csvSizeBytes)); + randomStream.pipe(res); + req.on('close', () => { + randomStream.destroy(); + --openProcessorCount; + }); +}); +app.get('/__mock_http_server/open-processor-count', (req, res) => { + res.send({ openProcessorCount, completedProcessorCount }); +}); + app.get('/v1/reflect-headers', (req, res) => res.json(req.headers)); // Central-Backend can set Cache headers and those should have highest precedence diff --git a/test/nginx/src/lib.js b/test/nginx/src/lib.js index 147db139e..d96a2de2d 100644 --- a/test/nginx/src/lib.js +++ b/test/nginx/src/lib.js @@ -10,6 +10,7 @@ module.exports = { assertSentryReceived, requestSentryMock, resetSentryMock, + sleep, }; async function assertSentryReceived(...expectedRequests) { @@ -56,3 +57,7 @@ function requestSentryMock(opts) { req.end(); }); } + +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} diff --git a/test/nginx/src/mocha/nginx.spec.js b/test/nginx/src/mocha/nginx.spec.js index e5be18274..29cb93eb1 100644 --- a/test/nginx/src/mocha/nginx.spec.js +++ b/test/nginx/src/mocha/nginx.spec.js @@ -5,6 +5,7 @@ const { assertSentryReceived, requestSentryMock, resetSentryMock, + sleep, } = require('../lib'); const request = require('./request'); @@ -437,6 +438,78 @@ function standardTestSuite({ fetchHttp, fetchHttp6, apiFetch, apiFetch6, forward }); }); + describe('response buffering', () => { + it('should buffer responses in nginx, not backend services', async function() { + const testTimeout = 5_000; + this.timeout(testTimeout); + + let controller; + + try { + // given + controller = new AbortController(); + const { signal } = controller; + + // when + const res = await apiFetch('/v1/projects/123/forms/some_form_id/attachments/100MB.csv', { signal }); + // then + assert.equal(res.status, 200); + + // when + const reader = res.body.getReader(); + const initialRead = await reader.read(); + let bytesRead = initialRead.value.length; + // then + assert.isFalse(initialRead.done); + assert.isAtMost(bytesRead, 16_384); + assert.equal(new TextDecoder('utf8').decode(initialRead.value).split('\n', 1)[0], 'row_number,timestamp,random-number'); + // and + assert.deepEqual(await getOpenProcessorCount(), { openProcessorCount:1, completedProcessorCount:0 }); + + // when + await untilOpenProcessorCountIs({ timeout:testTimeout, openProcessorCount:0, completedProcessorCount:1 }); + // and + while(true) { + const { done, value } = await reader.read(); + if(done) break; + bytesRead += value.length; + } + // then + assert.equal(bytesRead, 100_000_000); + } finally { + controller.abort(); + } + }); + + async function getOpenProcessorCount() { + const res = await request(`http://localhost:8383/__mock_http_server/open-processor-count`); + assert.isTrue(res.ok); + return await res.json(); + } + + async function untilOpenProcessorCountIs({ timeout, ...expected }) { + let timeoutId; + try { + let timedOut; + timeoutId = setTimeout(() => { timedOut = true; }, timeout); + + while(true) { + const { openProcessorCount, completedProcessorCount } = await getOpenProcessorCount(); + if(openProcessorCount === expected.openProcessorCount && + completedProcessorCount === expected.completedProcessorCount) { + break; + } + + if(timedOut) throw new Error(`Timeout of ${timeout} ms exceeded.`); + + await sleep(100); + } + } finally { + clearTimeout(timeoutId); + } + } + }); + it('should serve generated client-config.json', async () => { // when const res = await apiFetch('/client-config.json'); From 097bb4fb3264f4d339ca12b29d5a873f0957600e Mon Sep 17 00:00:00 2001 From: Alex Anderson <191496+alxndrsn@users.noreply.github.com> Date: Wed, 5 Aug 2026 06:35:42 +0300 Subject: [PATCH 2/7] nginx: make selfsign cert generation more robust (#2097) Protect against failure between privkey generation and fullchain generation. --- files/nginx/setup-odk.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/files/nginx/setup-odk.sh b/files/nginx/setup-odk.sh index cd839113a..6c81fa56e 100755 --- a/files/nginx/setup-odk.sh +++ b/files/nginx/setup-odk.sh @@ -28,7 +28,10 @@ if [ "$SSL_TYPE" != "upstream" ] && [ ! -s "$DH_PATH" ]; then fi SELFSIGN_PATH="/etc/selfsign/live/$DOMAIN" -if [ "$SSL_TYPE" = "selfsign" ] && [ ! -s "$SELFSIGN_PATH/privkey.pem" ]; then +if [ "$SSL_TYPE" = "selfsign" ] && { + ! [ -s "$SELFSIGN_PATH/privkey.pem" ] || + ! [ -s "$SELFSIGN_PATH/fullchain.pem" ]; +}; then mkdir -p "$SELFSIGN_PATH" openssl req -x509 -newkey rsa:4096 \ -subj "/C=XX/ST=XXXX/L=XXXX/O=XXXX/CN=localhost" \ From 736344e02b7a4bf408874e1c6bf5c3f70dfc3cde Mon Sep 17 00:00:00 2001 From: Alex Anderson <191496+alxndrsn@users.noreply.github.com> Date: Wed, 5 Aug 2026 09:14:02 +0300 Subject: [PATCH 3/7] nginx: don't regenerate bad host SSL cert if exists (#2096) * consistent with surrounding code: other keys are only generated if they don't exist * generation is fast, but repeating every startup is still a waste of effort --- files/nginx/setup-odk.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/files/nginx/setup-odk.sh b/files/nginx/setup-odk.sh index 6c81fa56e..abfc632ab 100755 --- a/files/nginx/setup-odk.sh +++ b/files/nginx/setup-odk.sh @@ -15,12 +15,15 @@ fi # Generate self-signed keys for the incorrect (catch-all) HTTPS listener. This # cert should never be seen by legitimate users, so it's not a big deal that # it's self-signed and won't expire for 1,000 years. -mkdir -p /etc/nginx/ssl -openssl req -x509 -nodes -newkey rsa:2048 \ - -subj "/CN=invalid.local" \ - -keyout /etc/nginx/ssl/nginx.default.key \ - -out /etc/nginx/ssl/nginx.default.crt \ - -days 365000 +BADHOST_DH_PATH=/etc/nginx/ssl/nginx.default +if ! [ -s "$BADHOST_DH_PATH.key" ] || ! [ -s "$BADHOST_DH_PATH.crt" ]; then + mkdir -p /etc/nginx/ssl + openssl req -x509 -nodes -newkey rsa:2048 \ + -subj "/CN=invalid.local" \ + -keyout "$BADHOST_DH_PATH.key" \ + -out "$BADHOST_DH_PATH.crt" \ + -days 365000 +fi DH_PATH=/etc/dh/nginx.pem if [ "$SSL_TYPE" != "upstream" ] && [ ! -s "$DH_PATH" ]; then From 60503c4eb3648c9807d8f79744ee24676e707735 Mon Sep 17 00:00:00 2001 From: Alex Anderson <191496+alxndrsn@users.noreply.github.com> Date: Wed, 5 Aug 2026 10:29:35 +0300 Subject: [PATCH 4/7] nginx: remove ssl_trusted_certificate (#2033) This setting is only relevant if either `ssl_verify_client` or `ssl_stapling` is enabled. Neither setting is. See: https://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_trusted_certificate Any deployer who does require client certificate verification is likely sophisticated enough to re-enable themselves. Closes #831 --- files/nginx/odk.conf.template | 1 - 1 file changed, 1 deletion(-) diff --git a/files/nginx/odk.conf.template b/files/nginx/odk.conf.template index d4e476033..b741529b7 100644 --- a/files/nginx/odk.conf.template +++ b/files/nginx/odk.conf.template @@ -121,7 +121,6 @@ server { ssl_certificate /etc/${SSL_TYPE}/live/${CERT_DOMAIN}/fullchain.pem; ssl_certificate_key /etc/${SSL_TYPE}/live/${CERT_DOMAIN}/privkey.pem; - ssl_trusted_certificate /etc/${SSL_TYPE}/live/${CERT_DOMAIN}/fullchain.pem; ssl_protocols TLSv1.2 TLSv1.3; ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; From 4b24698ed79f57f2a9a014ebfb80dd58402829f2 Mon Sep 17 00:00:00 2001 From: Gareth Bowen Date: Tue, 11 Aug 2026 17:55:18 +1200 Subject: [PATCH 5/7] chore: update client and server for v2026.2.2 --- docker-compose.yml | 2 +- server | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 72f755977..5707b8076 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -97,7 +97,7 @@ services: dockerfile: nginx.dockerfile args: FRONTEND_BUILD_MODE: ${FRONTEND_BUILD_MODE:-fetch} - FRONTEND_VERSION: v2026.2.1 + FRONTEND_VERSION: v2026.2.2 depends_on: - service - enketo diff --git a/server b/server index bee93414a..7aa74c5ec 160000 --- a/server +++ b/server @@ -1 +1 @@ -Subproject commit bee93414a2215bd73d7f371113c6640c66d90c95 +Subproject commit 7aa74c5ec5d90189aa2482f57c041ea4ce9efbfd From 75ea809242794d1c0608947f4ec1c895aad90894 Mon Sep 17 00:00:00 2001 From: Alex Anderson <191496+alxndrsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 10:02:29 +0300 Subject: [PATCH 6/7] test/nginx/setup-odk: increase global teardown timeout (#2130) Seen to timeout after 2.1 seconds: https://github.com/getodk/central/actions/runs/31661302771/job/94326512906?pr=2104 Ref #2104 --- test/nginx/src/mocha/setup-odk.spec.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/nginx/src/mocha/setup-odk.spec.js b/test/nginx/src/mocha/setup-odk.spec.js index 221eaff0f..ce2b27107 100644 --- a/test/nginx/src/mocha/setup-odk.spec.js +++ b/test/nginx/src/mocha/setup-odk.spec.js @@ -13,7 +13,8 @@ describe('setup-odk.sh', function() { dockerCompose({}, `logs --timestamps ${service}`); log('--- END CONTAINER LOGS ---'); }); - after(() => { + after(function() { + this.timeout(5000); dockerCompose({}, `down --remove-orphans --volumes`); }); From 11ee00b6e3b87add36f19c3cc97b732d0b3fbec7 Mon Sep 17 00:00:00 2001 From: Alex Anderson <191496+alxndrsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 10:07:36 +0300 Subject: [PATCH 7/7] test/nginx/setup-odk: fix typo in test name (#2129) --- test/nginx/src/mocha/setup-odk.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/nginx/src/mocha/setup-odk.spec.js b/test/nginx/src/mocha/setup-odk.spec.js index ce2b27107..a05c85b67 100644 --- a/test/nginx/src/mocha/setup-odk.spec.js +++ b/test/nginx/src/mocha/setup-odk.spec.js @@ -28,7 +28,7 @@ describe('setup-odk.sh', function() { [ 'bad-format', '' ], [ 'https://abcdef0123456789abcdef0123456789@some-dsn.ingest.sentry.io/', '' ], ].forEach(([ SENTRY_DSN_FRONTEND, expectedCspEntry ]) => { - it(`should generated expected CSP for SENTRY_DSN_FRONTEND='${SENTRY_DSN_FRONTEND}'`, withNginx({ + it(`should generate expected CSP for SENTRY_DSN_FRONTEND='${SENTRY_DSN_FRONTEND}'`, withNginx({ SENTRY_DSN_FRONTEND, }, async () => { // when