From 8fe5c4ea66b9b8187600e6d5ec9b1b6781f44009 Mon Sep 17 00:00:00 2001 From: Ciffelia Date: Fri, 5 Nov 2021 22:42:53 +0900 Subject: [PATCH 01/19] 2.x: Specify encoding as an optional peer dependency in package.json (#1310) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Specify `encoding` as an optional peer dependency * Update package.json Co-authored-by: Linus Unnebäck Co-authored-by: Linus Unnebäck --- package.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/package.json b/package.json index ec0510513..6f0ac4302 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,14 @@ "dependencies": { "whatwg-url": "^5.0.0" }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + }, "devDependencies": { "@ungap/url-search-params": "^0.1.2", "abort-controller": "^1.1.0", From 1ef4b560a17e644a02a3bfdea7631ffeee578b35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jimmy=20W=C3=A4rting?= Date: Sun, 16 Jan 2022 12:45:33 +0100 Subject: [PATCH 02/19] backport of #1449 (#1453) * backport of #1449 * bump patch version --- package.json | 2 +- src/index.js | 49 ++++++++++++++++++++++++++++++++++++++++--------- test/server.js | 7 ++++++- test/test.js | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 94 insertions(+), 11 deletions(-) diff --git a/package.json b/package.json index 6f0ac4302..3c1bd8da7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.6.6", + "version": "2.6.7", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", "browser": "./browser.js", diff --git a/src/index.js b/src/index.js index 03b56f733..b210d28e4 100644 --- a/src/index.js +++ b/src/index.js @@ -13,16 +13,29 @@ import https from 'https'; import zlib from 'zlib'; import Stream from 'stream'; -import Body, { writeToStream, getTotalBytes } from './body'; -import Response from './response'; -import Headers, { createHeadersLenient } from './headers'; -import Request, { getNodeRequestOptions } from './request'; -import FetchError from './fetch-error'; -import AbortError from './abort-error'; +import Body, { writeToStream, getTotalBytes } from './body.js'; +import Response from './response.js'; +import Headers, { createHeadersLenient } from './headers.js'; +import Request, { getNodeRequestOptions } from './request.js'; +import FetchError from './fetch-error.js'; +import AbortError from './abort-error.js'; + +import whatwgUrl from 'whatwg-url'; + +const URL = Url.URL || whatwgUrl.URL; // fix an issue where "PassThrough", "resolve" aren't a named export for node <10 const PassThrough = Stream.PassThrough; -const resolve_url = Url.resolve; + +const isDomainOrSubdomain = (destination, original) => { + const orig = new URL(original).hostname; + const dest = new URL(destination).hostname; + + return orig === dest || ( + orig[orig.length - dest.length - 1] === '.' && orig.endsWith(dest) + ); +}; + /** * Fetch function @@ -109,7 +122,19 @@ export default function fetch(url, opts) { const location = headers.get('Location'); // HTTP fetch step 5.3 - const locationURL = location === null ? null : resolve_url(request.url, location); + let locationURL = null; + try { + locationURL = location === null ? null : new URL(location, request.url).toString(); + } catch (err) { + // error here can only be invalid URL in Location: header + // do not throw when options.redirect == manual + // let the user extract the errorneous redirect URL + if (request.redirect !== 'manual') { + reject(new FetchError(`uri requested responds with an invalid redirect URL: ${location}`, 'invalid-redirect')); + finalize(); + return; + } + } // HTTP fetch step 5.5 switch (request.redirect) { @@ -154,9 +179,15 @@ export default function fetch(url, opts) { body: request.body, signal: request.signal, timeout: request.timeout, - size: request.size + size: request.size }; + if (!isDomainOrSubdomain(request.url, locationURL)) { + for (const name of ['authorization', 'www-authenticate', 'cookie', 'cookie2']) { + requestOpts.headers.delete(name); + } + } + // HTTP-redirect fetch step 9 if (res.statusCode !== 303 && request.body && getTotalBytes(request) === null) { reject(new FetchError('Cannot follow redirect with body being a readable stream', 'unsupported-redirect')); diff --git a/test/server.js b/test/server.js index ebd311d9c..2f0baf8cd 100644 --- a/test/server.js +++ b/test/server.js @@ -1,7 +1,6 @@ import * as http from 'http'; import { parse } from 'url'; import * as zlib from 'zlib'; -import * as stream from 'stream'; import { multipart as Multipart } from 'parted'; let convert; @@ -66,6 +65,12 @@ export default class TestServer { })); } + if (p.startsWith('/redirect-to/3')) { + res.statusCode = p.slice(13, 16); + res.setHeader('Location', p.slice(17)); + res.end(); + } + if (p === '/gzip') { res.statusCode = 200; res.setHeader('Content-Type', 'text/plain'); diff --git a/test/test.js b/test/test.js index 6427ae21e..cdeb51f00 100644 --- a/test/test.js +++ b/test/test.js @@ -1569,6 +1569,53 @@ describe('node-fetch', () => { }); }); + it('should not forward secure headers to 3th party', () => { + return fetch(`${base}redirect-to/302/https://httpbin.org/get`, { + headers: new Headers({ + cookie: 'gets=removed', + cookie2: 'gets=removed', + authorization: 'gets=removed', + 'www-authenticate': 'gets=removed', + 'other-safe-headers': 'stays', + 'x-foo': 'bar' + }) + }).then(res => res.json()).then(json => { + const headers = new Headers(json.headers); + // Safe headers are not removed + expect(headers.get('other-safe-headers')).to.equal('stays'); + expect(headers.get('x-foo')).to.equal('bar'); + // Unsafe headers should not have been sent to httpbin + expect(headers.get('cookie')).to.equal(null); + expect(headers.get('cookie2')).to.equal(null); + expect(headers.get('www-authenticate')).to.equal(null); + expect(headers.get('authorization')).to.equal(null); + }); + }); + + it('should forward secure headers to same host', () => { + return fetch(`${base}redirect-to/302/${base}inspect`, { + headers: new Headers({ + cookie: 'is=cookie', + cookie2: 'is=cookie2', + authorization: 'is=authorization', + 'other-safe-headers': 'stays', + 'www-authenticate': 'is=www-authenticate', + 'x-foo': 'bar' + }) + }).then(res => res.json().then(json => { + const headers = new Headers(json.headers); + // Safe headers are not removed + expect(res.url).to.equal(`${base}inspect`); + expect(headers.get('other-safe-headers')).to.equal('stays'); + expect(headers.get('x-foo')).to.equal('bar'); + // Unsafe headers should not have been sent to httpbin + expect(headers.get('cookie')).to.equal('is=cookie'); + expect(headers.get('cookie2')).to.equal('is=cookie2'); + expect(headers.get('www-authenticate')).to.equal('is=www-authenticate'); + expect(headers.get('authorization')).to.equal('is=authorization'); + })); + }); + it('should allow PATCH request', function() { const url = `${base}inspect`; const opts = { From 838d9713ef5e673bbd86768fd22ba98ec461ed9d Mon Sep 17 00:00:00 2001 From: Maciej Goszczycki Date: Mon, 17 Jan 2022 00:40:12 +0100 Subject: [PATCH 03/19] Handle zero-length OK deflate responses (#903) --- src/index.js | 7 +++++++ test/server.js | 7 +++++++ test/test.js | 11 +++++++++++ 3 files changed, 25 insertions(+) diff --git a/src/index.js b/src/index.js index b210d28e4..93945f6d8 100644 --- a/src/index.js +++ b/src/index.js @@ -275,6 +275,13 @@ export default function fetch(url, opts) { response = new Response(body, response_options); resolve(response); }); + raw.on('end', () => { + // some old IIS servers return zero-length OK deflate responses, so 'data' is never emitted. + if (!response) { + response = new Response(body, response_options); + resolve(response); + } + }) return; } diff --git a/test/server.js b/test/server.js index 2f0baf8cd..ffe4733ff 100644 --- a/test/server.js +++ b/test/server.js @@ -120,6 +120,13 @@ export default class TestServer { }); } + if (p === '/empty/deflate') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/plain'); + res.setHeader('Content-Encoding', 'deflate'); + res.end(); + } + if (p === '/sdch') { res.statusCode = 200; res.setHeader('Content-Type', 'text/plain'); diff --git a/test/test.js b/test/test.js index cdeb51f00..719b5b960 100644 --- a/test/test.js +++ b/test/test.js @@ -679,6 +679,17 @@ describe('node-fetch', () => { }); }); + it('should handle empty deflate response', function() { + const url = `${base}empty/deflate`; + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain'); + return res.text().then(result => { + expect(result).to.be.a('string'); + expect(result).to.be.empty; + }); + }); + }); + it('should decompress brotli response', function() { if(typeof zlib.createBrotliDecompress !== 'function') this.skip(); const url = `${base}brotli`; From 50536d1e02ad42bdf262381034805378b98bfa53 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Sat, 16 Jul 2022 13:16:51 +0000 Subject: [PATCH 04/19] fix: premature close with chunked transfer encoding and for async iterators in Node 12 (#1172) * fix: premature close with chunked transfer encoding and for async iterators in Node 12 This PR backports the fix from #1064 to the `2.x.x` branch following the [comment here](https://github.com/node-fetch/node-fetch/pull/1064#issuecomment-849167400). I had to add some extra babel config to allow using the `for await..of` syntax in the tests. The config is only needed for the tests as this syntax is not used in the implementation. * chore: fix up tests for node 6+ * chore: codecov dropped support for node < 8 without shipping major * chore: npm7 strips empty dependencies hash during install * chore: pin deps to versions that work on node 4 * chore: do not emit close error after aborting a request * chore: test on node 4-16 * chore: simplify chunked transer encoding bad ending * chore: avoid calling .destroy as it is not in every node.js release * chore: listen for response close as socket is reused and shows warnings --- .babelrc | 6 ++-- .travis.yml | 2 ++ README.md | 43 +++++++++++++++++++++++++++++ package.json | 4 ++- src/index.js | 68 +++++++++++++++++++++++++++++++++++++++++++++- test/server.js | 28 +++++++++++++++++++ test/test.js | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 221 insertions(+), 4 deletions(-) diff --git a/.babelrc b/.babelrc index 6a95c25e7..901200bce 100644 --- a/.babelrc +++ b/.babelrc @@ -14,7 +14,8 @@ } ] ], plugins: [ - './build/babel-plugin' + './build/babel-plugin', + 'transform-async-generator-functions' ] }, coverage: { @@ -31,7 +32,8 @@ ], plugins: [ [ 'istanbul', { exclude: [ 'src/blob.js', 'build', 'test' ] } ], - './build/babel-plugin' + './build/babel-plugin', + 'transform-async-generator-functions' ] }, rollup: { diff --git a/.travis.yml b/.travis.yml index 3bb109e15..7d7081b33 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,8 @@ node_js: - "6" - "8" - "10" + - "12" + - "14" - "node" env: - FORMDATA_VERSION=1.0.0 diff --git a/README.md b/README.md index 2dde74289..4f87a59a0 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,49 @@ fetch('https://assets-cdn.github.com/images/modules/logos_page/Octocat.png') }); ``` +In Node.js 14 you can also use async iterators to read `body`; however, be careful to catch +errors -- the longer a response runs, the more likely it is to encounter an error. + +```js +const fetch = require('node-fetch'); +const response = await fetch('https://httpbin.org/stream/3'); +try { + for await (const chunk of response.body) { + console.dir(JSON.parse(chunk.toString())); + } +} catch (err) { + console.error(err.stack); +} +``` + +In Node.js 12 you can also use async iterators to read `body`; however, async iterators with streams +did not mature until Node.js 14, so you need to do some extra work to ensure you handle errors +directly from the stream and wait on it response to fully close. + +```js +const fetch = require('node-fetch'); +const read = async body => { + let error; + body.on('error', err => { + error = err; + }); + for await (const chunk of body) { + console.dir(JSON.parse(chunk.toString())); + } + return new Promise((resolve, reject) => { + body.on('close', () => { + error ? reject(error) : resolve(); + }); + }); +}; +try { + const response = await fetch('https://httpbin.org/stream/3'); + await read(response.body); +} catch (err) { + console.error(err.stack); +} +``` + #### Buffer If you prefer to cache binary data in full, use buffer(). (NOTE: `buffer()` is a `node-fetch`-only API) diff --git a/package.json b/package.json index 3c1bd8da7..ace7d8c4f 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,9 @@ "abortcontroller-polyfill": "^1.3.0", "babel-core": "^6.26.3", "babel-plugin-istanbul": "^4.1.6", - "babel-preset-env": "^1.6.1", + "babel-plugin-transform-async-generator-functions": "^6.24.1", + "babel-polyfill": "^6.26.0", + "babel-preset-env": "1.4.0", "babel-register": "^6.16.3", "chai": "^3.5.0", "chai-as-promised": "^7.1.1", diff --git a/src/index.js b/src/index.js index 93945f6d8..1a25e800d 100644 --- a/src/index.js +++ b/src/index.js @@ -67,7 +67,7 @@ export default function fetch(url, opts) { let error = new AbortError('The user aborted a request.'); reject(error); if (request.body && request.body instanceof Stream.Readable) { - request.body.destroy(error); + destroyStream(request.body, error); } if (!response || !response.body) return; response.body.emit('error', error); @@ -108,9 +108,41 @@ export default function fetch(url, opts) { req.on('error', err => { reject(new FetchError(`request to ${request.url} failed, reason: ${err.message}`, 'system', err)); + + if (response && response.body) { + destroyStream(response.body, err); + } + finalize(); }); + fixResponseChunkedTransferBadEnding(req, err => { + if (signal && signal.aborted) { + return + } + + destroyStream(response.body, err); + }); + + /* c8 ignore next 18 */ + if (parseInt(process.version.substring(1)) < 14) { + // Before Node.js 14, pipeline() does not fully support async iterators and does not always + // properly handle when the socket close/end events are out of order. + req.on('socket', s => { + s.addListener('close', hadError => { + // if a data listener is still present we didn't end cleanly + const hasDataListener = s.listenerCount('data') > 0 + + // if end happened before close but the socket didn't emit an error, do it now + if (response && hasDataListener && !hadError && !(signal && signal.aborted)) { + const err = new Error('Premature close'); + err.code = 'ERR_STREAM_PREMATURE_CLOSE'; + response.body.emit('error', err); + } + }); + }); + } + req.on('response', res => { clearTimeout(reqTimeout); @@ -303,6 +335,40 @@ export default function fetch(url, opts) { }; +function fixResponseChunkedTransferBadEnding(request, errorCallback) { + let socket; + + request.on('socket', s => { + socket = s; + }); + + request.on('response', response => { + const {headers} = response; + if (headers['transfer-encoding'] === 'chunked' && !headers['content-length']) { + response.once('close', hadError => { + // if a data listener is still present we didn't end cleanly + const hasDataListener = socket.listenerCount('data') > 0; + + if (hasDataListener && !hadError) { + const err = new Error('Premature close'); + err.code = 'ERR_STREAM_PREMATURE_CLOSE'; + errorCallback(err); + } + }); + } + }); +} + +function destroyStream (stream, err) { + if (stream.destroy) { + stream.destroy(err); + } else { + // node < 8 + stream.emit('error', err); + stream.end(); + } +} + /** * Redirect code matching * diff --git a/test/server.js b/test/server.js index ffe4733ff..ffff88877 100644 --- a/test/server.js +++ b/test/server.js @@ -329,6 +329,34 @@ export default class TestServer { res.destroy(); } + if (p === '/error/premature/chunked') { + res.writeHead(200, { + 'Content-Type': 'application/json', + 'Transfer-Encoding': 'chunked' + }); + + // Transfer-Encoding: 'chunked' sends chunk sizes followed by the + // chunks - https://en.wikipedia.org/wiki/Chunked_transfer_encoding + const sendChunk = (obj) => { + const data = JSON.stringify(obj) + + res.write(`${data.length}\r\n`) + res.write(`${data}\r\n`) + } + + sendChunk({data: 'hi'}) + + setTimeout(() => { + sendChunk({data: 'bye'}) + }, 200); + + setTimeout(() => { + // should send '0\r\n\r\n' to end the response properly but instead + // just close the connection + res.destroy(); + }, 400); + } + if (p === '/error/json') { res.statusCode = 200; res.setHeader('Content-Type', 'application/json'); diff --git a/test/test.js b/test/test.js index 719b5b960..36fe7703e 100644 --- a/test/test.js +++ b/test/test.js @@ -1,4 +1,7 @@ +import 'babel-core/register' +import 'babel-polyfill' + // test tools import chai from 'chai'; import chaiPromised from 'chai-as-promised'; @@ -552,6 +555,77 @@ describe('node-fetch', () => { .and.have.property('code', 'ECONNRESET'); }); + it('should handle network-error in chunked response', () => { + const url = `${base}error/premature/chunked`; + return fetch(url).then(res => { + expect(res.status).to.equal(200); + expect(res.ok).to.be.true; + + return expect(new Promise((resolve, reject) => { + res.body.on('error', reject); + res.body.on('close', resolve); + })).to.eventually.be.rejectedWith(Error, 'Premature close') + .and.have.property('code', 'ERR_STREAM_PREMATURE_CLOSE'); + }); + }); + + // Skip test if streams are not async iterators (node < 10) + const itAsyncIterator = Boolean(new stream.PassThrough()[Symbol.asyncIterator]) ? it : it.skip; + + itAsyncIterator('should handle network-error in chunked response async iterator', () => { + const url = `${base}error/premature/chunked`; + return fetch(url).then(res => { + expect(res.status).to.equal(200); + expect(res.ok).to.be.true; + + const read = async body => { + const chunks = []; + + if (process.version < 'v14') { + // In Node.js 12, some errors don't come out in the async iterator; we have to pick + // them up from the event-emitter and then throw them after the async iterator + let error; + body.on('error', err => { + error = err; + }); + + for await (const chunk of body) { + chunks.push(chunk); + } + + if (error) { + throw error; + } + + return new Promise(resolve => { + body.on('close', () => resolve(chunks)); + }); + } + + for await (const chunk of body) { + chunks.push(chunk); + } + + return chunks; + }; + + return expect(read(res.body)) + .to.eventually.be.rejectedWith(Error, 'Premature close') + .and.have.property('code', 'ERR_STREAM_PREMATURE_CLOSE'); + }); + }); + + it('should handle network-error in chunked response in consumeBody', () => { + const url = `${base}error/premature/chunked`; + return fetch(url).then(res => { + expect(res.status).to.equal(200); + expect(res.ok).to.be.true; + + return expect(res.text()) + .to.eventually.be.rejectedWith(Error, 'Premature close'); + }); + }); + it('should handle DNS-error response', function() { const url = 'http://domain.invalid'; return expect(fetch(url)).to.eventually.be.rejected From fddad0e7ea3fd6da01cc006fdf0ed304ccdd7990 Mon Sep 17 00:00:00 2001 From: victal Date: Tue, 19 Jul 2022 17:38:01 -0300 Subject: [PATCH 05/19] fix(headers): don't forward secure headers on protocol change (#1605) backport for #1599 to the 2.x branch Co-authored-by: Guilherme Victal --- src/index.js | 16 +++++++++++++++- test/test.js | 23 +++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/index.js b/src/index.js index 1a25e800d..f39ed3a93 100644 --- a/src/index.js +++ b/src/index.js @@ -36,6 +36,20 @@ const isDomainOrSubdomain = (destination, original) => { ); }; +/** + * isSameProtocol reports whether the two provided URLs use the same protocol. + * + * Both domains must already be in canonical form. + * @param {string|URL} original + * @param {string|URL} destination + */ +const isSameProtocol = (destination, original) => { + const orig = new URL(original).protocol; + const dest = new URL(destination).protocol; + + return orig === dest; +}; + /** * Fetch function @@ -214,7 +228,7 @@ export default function fetch(url, opts) { size: request.size }; - if (!isDomainOrSubdomain(request.url, locationURL)) { + if (!isDomainOrSubdomain(request.url, locationURL) || !isSameProtocol(request.url, locationURL)) { for (const name of ['authorization', 'www-authenticate', 'cookie', 'cookie2']) { requestOpts.headers.delete(name); } diff --git a/test/test.js b/test/test.js index 36fe7703e..21cf24055 100644 --- a/test/test.js +++ b/test/test.js @@ -1677,6 +1677,29 @@ describe('node-fetch', () => { }); }); + it('should not forward secure headers to changed protocol', async () => { + const res = await fetch('https://httpbin.org/redirect-to?url=http%3A%2F%2Fhttpbin.org%2Fget&status_code=302', { + headers: new Headers({ + cookie: 'gets=removed', + cookie2: 'gets=removed', + authorization: 'gets=removed', + 'www-authenticate': 'gets=removed', + 'other-safe-headers': 'stays', + 'x-foo': 'bar' + }) + }); + + const headers = new Headers((await res.json()).headers); + // Safe headers are not removed + expect(headers.get('other-safe-headers')).to.equal('stays'); + expect(headers.get('x-foo')).to.equal('bar'); + // Unsafe headers should not have been sent to downgraded http + expect(headers.get('cookie')).to.equal(null); + expect(headers.get('cookie2')).to.equal(null); + expect(headers.get('www-authenticate')).to.equal(null); + expect(headers.get('authorization')).to.equal(null); + }); + it('should forward secure headers to same host', () => { return fetch(`${base}redirect-to/302/${base}inspect`, { headers: new Headers({ From e218f8d5b7c6ad48b3a6c8e85bc65948ed295b26 Mon Sep 17 00:00:00 2001 From: Seth Westphal Date: Sun, 31 Jul 2022 10:01:53 -0500 Subject: [PATCH 06/19] Add missing changelog entries. (#1613) --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29d168cf7..3d98f588f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ Changelog # 2.x release +## v2.6.7 + +- Fix: don't forward secure headers to 3th party + +## v2.6.6 + +- Fix: prefer built in URL version when available and fallback to whatwg + ## v2.6.5 - Fix: import `whatwg-url` in a way compatible with ESM From 8bb6e317c866c4134e7d67e90a5596a8c67e3965 Mon Sep 17 00:00:00 2001 From: Val Date: Tue, 8 Nov 2022 07:46:25 -0500 Subject: [PATCH 07/19] fix: prevent hoisting of the undefined `global` variable in `browser.js` (#1534) Because of JS hoisting `var global` to the top of the file, `typeof global` in `getGlobal()` will always be `undefined`. By using a different variable name like `globalObject`, we are able to read the "real" `typeof global` and get access to the global object that way. --- browser.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/browser.js b/browser.js index 83c54c584..7035edbed 100644 --- a/browser.js +++ b/browser.js @@ -11,15 +11,15 @@ var getGlobal = function () { throw new Error('unable to locate global object'); } -var global = getGlobal(); +var globalObject = getGlobal(); -module.exports = exports = global.fetch; +module.exports = exports = globalObject.fetch; // Needed for TypeScript and Webpack. -if (global.fetch) { - exports.default = global.fetch.bind(global); +if (globalObject.fetch) { + exports.default = globalObject.fetch.bind(global); } -exports.Headers = global.Headers; -exports.Request = global.Request; -exports.Response = global.Response; \ No newline at end of file +exports.Headers = globalObject.Headers; +exports.Request = globalObject.Request; +exports.Response = globalObject.Response; From 1768eaa7dcc51adc0038cb07e2cdfd6d44b2164a Mon Sep 17 00:00:00 2001 From: Gregor Martynus <39992+gr2m@users.noreply.github.com> Date: Thu, 12 Jan 2023 16:55:10 -0800 Subject: [PATCH 08/19] ci(release): initial version --- .github/workflows/release.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..881b4cb69 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,22 @@ +name: Release +on: + push: + branches: + - main + - next + - beta + - "*.x" # maintenance releases such as 2.x + +jobs: + release: + name: release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: 16 + - run: npx semantic-release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} From ce37bcd93e869e2c0a05d4a913ad08ce94399e88 Mon Sep 17 00:00:00 2001 From: Gregor Martynus <39992+gr2m@users.noreply.github.com> Date: Thu, 12 Jan 2023 16:56:34 -0800 Subject: [PATCH 09/19] ci(semantic-release): config --- package.json | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/package.json b/package.json index ace7d8c4f..66f59adc6 100644 --- a/package.json +++ b/package.json @@ -74,5 +74,16 @@ "rollup-plugin-babel": "^3.0.7", "string-to-arraybuffer": "^1.0.2", "teeny-request": "3.7.0" + }, + "release": { + "branches": [ + "+([0-9]).x", + "main", + "next", + { + "name": "beta", + "prerelease": true + } + ] } } From 49bef02a2f630bb083d1920cb40ff09363479ef2 Mon Sep 17 00:00:00 2001 From: Gregor Martynus <39992+gr2m@users.noreply.github.com> Date: Thu, 12 Jan 2023 16:57:50 -0800 Subject: [PATCH 10/19] ci(release): use latest Node LTS --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 881b4cb69..236ef0ae3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: - node-version: 16 + node-version: "lts/*" - run: npx semantic-release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From dd2a0ba0fb1ed0d321fcde46562e824d9f40fea1 Mon Sep 17 00:00:00 2001 From: Gregor Martynus <39992+gr2m@users.noreply.github.com> Date: Thu, 12 Jan 2023 17:01:08 -0800 Subject: [PATCH 11/19] ci(release): install dependencies --- .github/workflows/release.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 236ef0ae3..e116bd611 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,10 +12,11 @@ jobs: name: release runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 with: node-version: "lts/*" + - run: npm ci - run: npx semantic-release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 6e9464d7e34dc323edf4dabad7615dd94ab847bd Mon Sep 17 00:00:00 2001 From: Gregor Martynus <39992+gr2m@users.noreply.github.com> Date: Thu, 12 Jan 2023 17:02:06 -0800 Subject: [PATCH 12/19] ci(release): install dependencies --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e116bd611..315e6c813 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/setup-node@v3 with: node-version: "lts/*" - - run: npm ci + - run: npm install - run: npx semantic-release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 0f1ebb0d9d9726351a83a50eaaccf66342f04e06 Mon Sep 17 00:00:00 2001 From: Amit Agarwal <1344071+labnol@users.noreply.github.com> Date: Mon, 23 Jan 2023 21:08:22 +0530 Subject: [PATCH 13/19] Prevent error when response is null (#1699) Fix for this error. TypeError: Cannot read properties of null (reading 'body') --- src/index.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/index.js b/src/index.js index f39ed3a93..9013f1de2 100644 --- a/src/index.js +++ b/src/index.js @@ -135,7 +135,9 @@ export default function fetch(url, opts) { return } - destroyStream(response.body, err); + if (response && response.body) { + destroyStream(response.body, err); + } }); /* c8 ignore next 18 */ From 70f592d9d2da959df1cebc2dd2314286a4bcf345 Mon Sep 17 00:00:00 2001 From: Segev Finer Date: Mon, 30 Jan 2023 23:59:17 +0200 Subject: [PATCH 14/19] fix: "global is not defined" (#1704) --- browser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/browser.js b/browser.js index 7035edbed..ee86265ae 100644 --- a/browser.js +++ b/browser.js @@ -17,7 +17,7 @@ module.exports = exports = globalObject.fetch; // Needed for TypeScript and Webpack. if (globalObject.fetch) { - exports.default = globalObject.fetch.bind(global); + exports.default = globalObject.fetch.bind(globalObject); } exports.Headers = globalObject.Headers; From 29909d75c62d51e0d1c23758e526dba74bfd463d Mon Sep 17 00:00:00 2001 From: Alexis Clarembeau Date: Mon, 8 May 2023 18:19:42 +0200 Subject: [PATCH 15/19] fix: handle bom in text and json (#1739) * fix: handle bom in text and json * add unit tests --- package.json | 2 +- src/body.js | 8 ++++---- test/test.js | 15 +++++++++++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 66f59adc6..92f958869 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "dependencies": { "whatwg-url": "^5.0.0" }, - "peerDependencies": { + "peerDependencies": { "encoding": "^0.1.0" }, "peerDependenciesMeta": { diff --git a/src/body.js b/src/body.js index a9d2e7973..273044efe 100644 --- a/src/body.js +++ b/src/body.js @@ -114,9 +114,9 @@ Body.prototype = { * @return Promise */ json() { - return consumeBody.call(this).then((buffer) => { - try { - return JSON.parse(buffer.toString()); + return this.text().then((text) => { + try{ + return JSON.parse(text); } catch (err) { return Body.Promise.reject(new FetchError(`invalid json response body at ${this.url} reason: ${err.message}`, 'invalid-json')); } @@ -129,7 +129,7 @@ Body.prototype = { * @return Promise */ text() { - return consumeBody.call(this).then(buffer => buffer.toString()); + return consumeBody.call(this).then(buffer => new TextDecoder().decode(buffer)); }, /** diff --git a/test/test.js b/test/test.js index 21cf24055..0939e0f1f 100644 --- a/test/test.js +++ b/test/test.js @@ -2479,6 +2479,21 @@ describe('Response', function () { expect(res.headers.get('a')).to.equal('1'); }); + it('should decode responses containing BOM to json', async () => { + const json = await new Response('\uFEFF{"a":1}').json(); + expect(json.a).to.equal(1); + }); + + it('should decode responses containing BOM to text', async () => { + const text = await new Response('\uFEFF{"a":1}').text(); + expect(text).to.equal('{"a":1}'); + }); + + it('should keep BOM when getting raw bytes', async () => { + const ab = await new Response('\uFEFF{"a":1}').arrayBuffer(); + expect(ab.byteLength).to.equal(10); + }); + it('should support text() method', function() { const res = new Response('a=1'); return res.text().then(result => { From afb36f6c178342488d71947dfc87e7ddd19fab9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jimmy=20W=C3=A4rting?= Date: Tue, 9 May 2023 13:05:46 +0200 Subject: [PATCH 16/19] Revert "fix: handle bom in text and json (#1739)" (#1741) This reverts commit 29909d75c62d51e0d1c23758e526dba74bfd463d. --- package.json | 2 +- src/body.js | 8 ++++---- test/test.js | 15 --------------- 3 files changed, 5 insertions(+), 20 deletions(-) diff --git a/package.json b/package.json index 92f958869..66f59adc6 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "dependencies": { "whatwg-url": "^5.0.0" }, - "peerDependencies": { + "peerDependencies": { "encoding": "^0.1.0" }, "peerDependenciesMeta": { diff --git a/src/body.js b/src/body.js index 273044efe..a9d2e7973 100644 --- a/src/body.js +++ b/src/body.js @@ -114,9 +114,9 @@ Body.prototype = { * @return Promise */ json() { - return this.text().then((text) => { - try{ - return JSON.parse(text); + return consumeBody.call(this).then((buffer) => { + try { + return JSON.parse(buffer.toString()); } catch (err) { return Body.Promise.reject(new FetchError(`invalid json response body at ${this.url} reason: ${err.message}`, 'invalid-json')); } @@ -129,7 +129,7 @@ Body.prototype = { * @return Promise */ text() { - return consumeBody.call(this).then(buffer => new TextDecoder().decode(buffer)); + return consumeBody.call(this).then(buffer => buffer.toString()); }, /** diff --git a/test/test.js b/test/test.js index 0939e0f1f..21cf24055 100644 --- a/test/test.js +++ b/test/test.js @@ -2479,21 +2479,6 @@ describe('Response', function () { expect(res.headers.get('a')).to.equal('1'); }); - it('should decode responses containing BOM to json', async () => { - const json = await new Response('\uFEFF{"a":1}').json(); - expect(json.a).to.equal(1); - }); - - it('should decode responses containing BOM to text', async () => { - const text = await new Response('\uFEFF{"a":1}').text(); - expect(text).to.equal('{"a":1}'); - }); - - it('should keep BOM when getting raw bytes', async () => { - const ab = await new Response('\uFEFF{"a":1}').arrayBuffer(); - expect(ab.byteLength).to.equal(10); - }); - it('should support text() method', function() { const res = new Response('a=1'); return res.text().then(result => { From 8bc3a7c85f67fb81bb3d71c8254e68f3b88e9169 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Magalh=C3=A3es?= Date: Thu, 29 Jun 2023 20:15:38 +0100 Subject: [PATCH 17/19] fix: socket variable testing for undefined (#1726) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: socket variable testing for undefined Avoid issue where socket event was not triggered and local socket vraible is not defined. This issue is reproducible only using deno. * chore: made socket test capture simpler Only controls the execution of the section that uses socket. * Update src/index.js Co-authored-by: Linus Unnebäck * Update index.js --------- Co-authored-by: Linus Unnebäck Co-authored-by: Jimmy Wärting --- src/index.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/index.js b/src/index.js index 9013f1de2..3f6f5f004 100644 --- a/src/index.js +++ b/src/index.js @@ -362,8 +362,11 @@ function fixResponseChunkedTransferBadEnding(request, errorCallback) { const {headers} = response; if (headers['transfer-encoding'] === 'chunked' && !headers['content-length']) { response.once('close', hadError => { - // if a data listener is still present we didn't end cleanly - const hasDataListener = socket.listenerCount('data') > 0; + // tests for socket presence, as in some situations the + // the 'socket' event is not triggered for the request + // (happens in deno), avoids `TypeError` + // if a data listener is still present we didn't end cleanly + const hasDataListener = socket && socket.listenerCount('data') > 0; if (hasDataListener && !hadError) { const err = new Error('Premature close'); From 65ae25a1da2834b046c218685f2085a06f679492 Mon Sep 17 00:00:00 2001 From: David Edey Date: Fri, 18 Aug 2023 21:23:31 +0100 Subject: [PATCH 18/19] fix: Remove the default connection close header (#1765) Instead, we rely on the underlying http implementation in Node.js to handle this, as per the documentation at https://nodejs.org/api/http.html#new-agentoptions This fixes #1735 and likely replaces #1473 The original change introducing this provided no clear motivation for the override, and the implementation has since been changed to disable this header when an agent is provided, so I think there is sufficient evidence that removing this is the correct behaviour. https://github.com/node-fetch/node-fetch/commit/af21ae6c1c964cd40e48e974af56ea2e1361304f https://github.com/node-fetch/node-fetch/commit/7f68577de44c7d1efe7009b212f7c54a0b4a709b This commit is backported to the v2 branch from #1736 against v3. --- README.md | 3 ++- src/request.js | 4 ---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4f87a59a0..55f09b7ff 100644 --- a/README.md +++ b/README.md @@ -387,7 +387,6 @@ Header | Value ------------------- | -------------------------------------------------------- `Accept-Encoding` | `gzip,deflate` _(when `options.compress === true`)_ `Accept` | `*/*` -`Connection` | `close` _(when no `options.agent` is present)_ `Content-Length` | _(automatically calculated, if possible)_ `Transfer-Encoding` | `chunked` _(when `req.body` is a stream)_ `User-Agent` | `node-fetch/1.0 (+https://github.com/bitinn/node-fetch)` @@ -404,6 +403,8 @@ The `agent` option allows you to specify networking related options which are ou See [`http.Agent`](https://nodejs.org/api/http.html#http_new_agent_options) for more information. +If no agent is specified, the default agent provided by Node.js is used. Note that [this changed in Node.js 19](https://github.com/nodejs/node/blob/4267b92604ad78584244488e7f7508a690cb80d0/lib/_http_agent.js#L564) to have `keepalive` true by default. If you wish to enable `keepalive` in an earlier version of Node.js, you can override the agent as per the following code sample. + In addition, the `agent` option accepts a function that returns `http`(s)`.Agent` instance given current [URL](https://nodejs.org/api/url.html), this is useful during a redirection chain across HTTP and HTTPS protocol. ```js diff --git a/src/request.js b/src/request.js index 739ba9071..e55c07d4d 100644 --- a/src/request.js +++ b/src/request.js @@ -258,10 +258,6 @@ export function getNodeRequestOptions(request) { agent = agent(parsedURL); } - if (!headers.has('Connection') && !agent) { - headers.set('Connection', 'close'); - } - // HTTP-network fetch step 4.2 // chunked encoding is handled by Node.js From 9b9d45881e5ca68757077726b3c0ecf8fdca1f29 Mon Sep 17 00:00:00 2001 From: Frederik Prijck Date: Wed, 23 Aug 2023 19:17:49 +0200 Subject: [PATCH 19/19] feat: `AbortError` (#1744) --- src/index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/index.js b/src/index.js index 3f6f5f004..b2d5d2dcc 100644 --- a/src/index.js +++ b/src/index.js @@ -402,5 +402,6 @@ export { Headers, Request, Response, - FetchError + FetchError, + AbortError };