From 2358a6c2563d1730a0cdaccc197c611949f6a334 Mon Sep 17 00:00:00 2001 From: Antoni Kepinski Date: Sat, 5 Sep 2020 14:55:39 +0200 Subject: [PATCH 01/23] Honor the `size` option after following a redirect and revert data uri support Co-authored-by: Richie Bendall --- CHANGELOG.md | 6 ++++++ src/index.js | 14 ++------------ test/test.js | 25 ------------------------- 3 files changed, 8 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 188fcd399..543d3d947 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ Changelog # 2.x release +## v2.6.1 + +**This is an important security release. It is strongly recommended to update as soon as possible.** + +- Fix: honor the `size` option after following a redirect. + ## v2.6.0 - Enhance: `options.agent`, it now accepts a function that returns custom http(s).Agent instance based on current URL, see readme for more information. diff --git a/src/index.js b/src/index.js index 8bf9248fd..03b56f733 100644 --- a/src/index.js +++ b/src/index.js @@ -38,17 +38,6 @@ export default function fetch(url, opts) { throw new Error('native promise missing, set fetch.Promise to your favorite alternative'); } - if (/^data:/.test(url)) { - const request = new Request(url, opts); - try { - const data = Buffer.from(url.split(',')[1], 'base64') - const res = new Response(data.body, { headers: { 'Content-Type': data.mimeType || url.match(/^data:(.+);base64,.*$/)[1] } }); - return fetch.Promise.resolve(res); - } catch (err) { - return fetch.Promise.reject(new FetchError(`[${request.method}] ${request.url} invalid URL, ${err.message}`, 'system', err)); - } - } - Body.Promise = fetch.Promise; // wrap http.request into fetch @@ -164,7 +153,8 @@ export default function fetch(url, opts) { method: request.method, body: request.body, signal: request.signal, - timeout: request.timeout + timeout: request.timeout, + size: request.size }; // HTTP-redirect fetch step 9 diff --git a/test/test.js b/test/test.js index c5d61c72a..d3cf2fc97 100644 --- a/test/test.js +++ b/test/test.js @@ -2844,29 +2844,4 @@ describe('external encoding', () => { }); }); }); - - describe('data uri', function() { - const dataUrl = 'data:image/gif;base64,R0lGODlhAQABAIAAAAUEBAAAACwAAAAAAQABAAACAkQBADs='; - - const invalidDataUrl = 'data:@@@@'; - - it('should accept data uri', function() { - return fetch(dataUrl).then(r => { - console.assert(r.status == 200); - console.assert(r.headers.get('Content-Type') == 'image/gif'); - - return r.buffer().then(b => { - console.assert(b instanceof Buffer); - }); - }); - }); - - it('should reject invalid data uri', function() { - return fetch(invalidDataUrl) - .catch(e => { - console.assert(e); - console.assert(e.message.includes('invalid URL')); - }); - }); - }); }); From b5e2e41b2b50bf2997720d6125accaf0dd68c0ab Mon Sep 17 00:00:00 2001 From: Antoni Kepinski Date: Sat, 5 Sep 2020 14:58:33 +0200 Subject: [PATCH 02/23] update version number --- package.json | 128 +++++++++++++++++++++++++-------------------------- 1 file changed, 64 insertions(+), 64 deletions(-) diff --git a/package.json b/package.json index 8e5c883b2..216046916 100644 --- a/package.json +++ b/package.json @@ -1,66 +1,66 @@ { - "name": "node-fetch", - "version": "2.6.0", - "description": "A light-weight module that brings window.fetch to node.js", - "main": "lib/index", - "browser": "./browser.js", - "module": "lib/index.mjs", - "files": [ - "lib/index.js", - "lib/index.mjs", - "lib/index.es.js", - "browser.js" - ], - "engines": { - "node": "4.x || >=6.0.0" - }, - "scripts": { - "build": "cross-env BABEL_ENV=rollup rollup -c", - "prepare": "npm run build", - "test": "cross-env BABEL_ENV=test mocha --require babel-register --throw-deprecation test/test.js", - "report": "cross-env BABEL_ENV=coverage nyc --reporter lcov --reporter text mocha -R spec test/test.js", - "coverage": "cross-env BABEL_ENV=coverage nyc --reporter json --reporter text mocha -R spec test/test.js && codecov -f coverage/coverage-final.json" - }, - "repository": { - "type": "git", - "url": "https://github.com/bitinn/node-fetch.git" - }, - "keywords": [ - "fetch", - "http", - "promise" - ], - "author": "David Frank", - "license": "MIT", - "bugs": { - "url": "https://github.com/bitinn/node-fetch/issues" - }, - "homepage": "https://github.com/bitinn/node-fetch", - "devDependencies": { - "@ungap/url-search-params": "^0.1.2", - "abort-controller": "^1.1.0", - "abortcontroller-polyfill": "^1.3.0", - "babel-core": "^6.26.3", - "babel-plugin-istanbul": "^4.1.6", - "babel-preset-env": "^1.6.1", - "babel-register": "^6.16.3", - "chai": "^3.5.0", - "chai-as-promised": "^7.1.1", - "chai-iterator": "^1.1.1", - "chai-string": "~1.3.0", - "codecov": "^3.3.0", - "cross-env": "^5.2.0", - "form-data": "^2.3.3", - "is-builtin-module": "^1.0.0", - "mocha": "^5.0.0", - "nyc": "11.9.0", - "parted": "^0.1.1", - "promise": "^8.0.3", - "resumer": "0.0.0", - "rollup": "^0.63.4", - "rollup-plugin-babel": "^3.0.7", - "string-to-arraybuffer": "^1.0.2", - "whatwg-url": "^5.0.0" - }, - "dependencies": {} + "name": "node-fetch", + "version": "2.6.1", + "description": "A light-weight module that brings window.fetch to node.js", + "main": "lib/index", + "browser": "./browser.js", + "module": "lib/index.mjs", + "files": [ + "lib/index.js", + "lib/index.mjs", + "lib/index.es.js", + "browser.js" + ], + "engines": { + "node": "4.x || >=6.0.0" + }, + "scripts": { + "build": "cross-env BABEL_ENV=rollup rollup -c", + "prepare": "npm run build", + "test": "cross-env BABEL_ENV=test mocha --require babel-register --throw-deprecation test/test.js", + "report": "cross-env BABEL_ENV=coverage nyc --reporter lcov --reporter text mocha -R spec test/test.js", + "coverage": "cross-env BABEL_ENV=coverage nyc --reporter json --reporter text mocha -R spec test/test.js && codecov -f coverage/coverage-final.json" + }, + "repository": { + "type": "git", + "url": "https://github.com/bitinn/node-fetch.git" + }, + "keywords": [ + "fetch", + "http", + "promise" + ], + "author": "David Frank", + "license": "MIT", + "bugs": { + "url": "https://github.com/bitinn/node-fetch/issues" + }, + "homepage": "https://github.com/bitinn/node-fetch", + "devDependencies": { + "@ungap/url-search-params": "^0.1.2", + "abort-controller": "^1.1.0", + "abortcontroller-polyfill": "^1.3.0", + "babel-core": "^6.26.3", + "babel-plugin-istanbul": "^4.1.6", + "babel-preset-env": "^1.6.1", + "babel-register": "^6.16.3", + "chai": "^3.5.0", + "chai-as-promised": "^7.1.1", + "chai-iterator": "^1.1.1", + "chai-string": "~1.3.0", + "codecov": "^3.3.0", + "cross-env": "^5.2.0", + "form-data": "^2.3.3", + "is-builtin-module": "^1.0.0", + "mocha": "^5.0.0", + "nyc": "11.9.0", + "parted": "^0.1.1", + "promise": "^8.0.3", + "resumer": "0.0.0", + "rollup": "^0.63.4", + "rollup-plugin-babel": "^3.0.7", + "string-to-arraybuffer": "^1.0.2", + "whatwg-url": "^5.0.0" + }, + "dependencies": {} } From 152214ca2f6e2a5a17d71e4638114625d3be30c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jimmy=20W=C3=A4rting?= Date: Mon, 6 Sep 2021 14:58:28 +0200 Subject: [PATCH 03/23] Fix(package.json): Corrected main file path in package.json (#1274) * fix main configuration in package.json * pinned a breaking change in codecov & teeny-request --- CHANGELOG.md | 5 +++++ package.json | 10 +++++----- rollup.config.js | 6 +----- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 543d3d947..812a96309 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ Changelog # 2.x release +## v2.6.2 + +- Fix: used full filename for main in package.json +- Other: pinned codecov & teeny-request (had one breaking change with spread operators) + ## v2.6.1 **This is an important security release. It is strongly recommended to update as soon as possible.** diff --git a/package.json b/package.json index 216046916..4bb7d9640 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,8 @@ { "name": "node-fetch", - "version": "2.6.1", + "version": "2.6.2", "description": "A light-weight module that brings window.fetch to node.js", - "main": "lib/index", + "main": "lib/index.js", "browser": "./browser.js", "module": "lib/index.mjs", "files": [ @@ -48,7 +48,7 @@ "chai-as-promised": "^7.1.1", "chai-iterator": "^1.1.1", "chai-string": "~1.3.0", - "codecov": "^3.3.0", + "codecov": "3.3.0", "cross-env": "^5.2.0", "form-data": "^2.3.3", "is-builtin-module": "^1.0.0", @@ -60,7 +60,7 @@ "rollup": "^0.63.4", "rollup-plugin-babel": "^3.0.7", "string-to-arraybuffer": "^1.0.2", + "teeny-request": "3.7.0", "whatwg-url": "^5.0.0" - }, - "dependencies": {} + } } diff --git a/rollup.config.js b/rollup.config.js index a201ee455..1bc88f8db 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -18,10 +18,6 @@ export default { tweakDefault() ], external: function (id) { - if (isBuiltin(id)) { - return true; - } - id = id.split('/').slice(0, id[0] === '@' ? 2 : 1).join('/'); - return !!require('./package.json').dependencies[id]; + return isBuiltin(id); } }; From ace7536c955556be742d9910566738630cc3c2a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20Unneb=C3=A4ck?= Date: Mon, 20 Sep 2021 16:09:10 +0200 Subject: [PATCH 04/23] fix: properly encode url with unicode characters (#1291) * fix: properly encode url with unicode characters * release: 2.6.3 --- CHANGELOG.md | 4 ++++ package.json | 8 +++++--- rollup.config.js | 5 ++++- src/request.js | 40 +++++++++++++++++++++++++++++++++++++--- test/server.js | 8 +++++++- test/test.js | 22 ++++++++++++++++++++++ 6 files changed, 79 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 812a96309..671da7654 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ Changelog # 2.x release +## v2.6.3 + +- Fix: properly encode url with unicode characters + ## v2.6.2 - Fix: used full filename for main in package.json diff --git a/package.json b/package.json index 4bb7d9640..c5edc7991 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.6.2", + "version": "2.6.3", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", "browser": "./browser.js", @@ -36,6 +36,9 @@ "url": "https://github.com/bitinn/node-fetch/issues" }, "homepage": "https://github.com/bitinn/node-fetch", + "dependencies": { + "whatwg-url": "^5.0.0" + }, "devDependencies": { "@ungap/url-search-params": "^0.1.2", "abort-controller": "^1.1.0", @@ -60,7 +63,6 @@ "rollup": "^0.63.4", "rollup-plugin-babel": "^3.0.7", "string-to-arraybuffer": "^1.0.2", - "teeny-request": "3.7.0", - "whatwg-url": "^5.0.0" + "teeny-request": "3.7.0" } } diff --git a/rollup.config.js b/rollup.config.js index 1bc88f8db..d5951bd2e 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -1,9 +1,12 @@ import isBuiltin from 'is-builtin-module'; import babel from 'rollup-plugin-babel'; +import packageJson from './package.json'; import tweakDefault from './build/rollup-plugin'; process.env.BABEL_ENV = 'rollup'; +const dependencies = Object.keys(packageJson.dependencies); + export default { input: 'src/index.js', output: [ @@ -18,6 +21,6 @@ export default { tweakDefault() ], external: function (id) { - return isBuiltin(id); + return dependencies.includes(id) || isBuiltin(id); } }; diff --git a/src/request.js b/src/request.js index 45a7eb7e4..6fa8e77b6 100644 --- a/src/request.js +++ b/src/request.js @@ -9,6 +9,7 @@ import Url from 'url'; import Stream from 'stream'; +import {URL} from 'whatwg-url'; import Headers, { exportNodeCompatibleHeaders } from './headers.js'; import Body, { clone, extractContentType, getTotalBytes } from './body'; @@ -18,6 +19,39 @@ const INTERNALS = Symbol('Request internals'); const parse_url = Url.parse; const format_url = Url.format; +/** + * Wrapper around `new URL` to handle arbitrary URLs + * + * @param {string} urlStr + * @return {void} + */ +function parseURL(urlStr) { + /* + Check whether the URL is absolute or not + + Scheme: https://tools.ietf.org/html/rfc3986#section-3.1 + Absolute URL: https://tools.ietf.org/html/rfc3986#section-4.3 + */ + if (/^[a-zA-Z][a-zA-Z\d+\-.]*:/.exec(urlStr)) { + const url = new URL(urlStr); + + return { + path: url.pathname, + pathname: url.pathname, + hostname: url.hostname, + protocol: url.protocol, + port: url.port, + hash: url.hash, + search: url.search, + query: url.query, + href: url.href, + } + } + + // Fallback to old implementation for arbitrary URLs + return parse_url(urlStr); +} + const streamDestructionSupported = 'destroy' in Stream.Readable.prototype; /** @@ -59,14 +93,14 @@ export default class Request { // in order to support Node.js' Url objects; though WHATWG's URL objects // will fall into this branch also (since their `toString()` will return // `href` property anyway) - parsedURL = parse_url(input.href); + parsedURL = parseURL(input.href); } else { // coerce input to a string before attempting to parse - parsedURL = parse_url(`${input}`); + parsedURL = parseURL(`${input}`); } input = {}; } else { - parsedURL = parse_url(input.url); + parsedURL = parseURL(input.url); } let method = init.method || input.method || 'GET'; diff --git a/test/server.js b/test/server.js index 06c715d65..ebd311d9c 100644 --- a/test/server.js +++ b/test/server.js @@ -32,7 +32,7 @@ export default class TestServer { } router(req, res) { - let p = parse(req.url).pathname; + let p = decodeURIComponent(parse(req.url).pathname); if (p === '/hello') { res.statusCode = 200; @@ -384,6 +384,12 @@ export default class TestServer { }); req.pipe(parser); } + + if (p === '/issues/1290/ひらがな') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/plain'); + res.end('Success'); + } } } diff --git a/test/test.js b/test/test.js index d3cf2fc97..9220cbd4a 100644 --- a/test/test.js +++ b/test/test.js @@ -2845,3 +2845,25 @@ describe('external encoding', () => { }); }); }); + +describe('issue #1290', function() { + it('should handle escaped unicode in URLs', () => { + const url = `${base}issues/1290/%E3%81%B2%E3%82%89%E3%81%8C%E3%81%AA`; + return fetch(url).then((res) => { + expect(res.status).to.equal(200); + return res.text().then(result => { + expect(result).to.equal('Success'); + }); + }); + }); + + it('should handle unicode in URLs', () => { + const url = `${base}issues/1290/ひらがな`; + return fetch(url).then((res) => { + expect(res.status).to.equal(200); + return res.text().then(result => { + expect(result).to.equal('Success'); + }); + }); + }); +}); From 18193c5922c64046b922e18faf41821290535f06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jimmy=20W=C3=A4rting?= Date: Tue, 21 Sep 2021 16:42:50 +0200 Subject: [PATCH 05/23] fix v2.6.3 that did not sending query params (#1301) --- CHANGELOG.md | 4 ++++ package.json | 2 +- src/request.js | 14 +------------- test/test.js | 9 +++++++++ 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 671da7654..46eef0ff0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ Changelog # 2.x release +## v2.6.4 + +- Hotfix: fix v2.6.3 that did not sending query params + ## v2.6.3 - Fix: properly encode url with unicode characters diff --git a/package.json b/package.json index c5edc7991..98cf5f4c5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.6.3", + "version": "2.6.4", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", "browser": "./browser.js", diff --git a/src/request.js b/src/request.js index 6fa8e77b6..3c27c81cf 100644 --- a/src/request.js +++ b/src/request.js @@ -33,19 +33,7 @@ function parseURL(urlStr) { Absolute URL: https://tools.ietf.org/html/rfc3986#section-4.3 */ if (/^[a-zA-Z][a-zA-Z\d+\-.]*:/.exec(urlStr)) { - const url = new URL(urlStr); - - return { - path: url.pathname, - pathname: url.pathname, - hostname: url.hostname, - protocol: url.protocol, - port: url.port, - hash: url.hash, - search: url.search, - query: url.query, - href: url.href, - } + urlStr = new URL(urlStr).toString() } // Fallback to old implementation for arbitrary URLs diff --git a/test/test.js b/test/test.js index 9220cbd4a..9568489e3 100644 --- a/test/test.js +++ b/test/test.js @@ -2847,6 +2847,15 @@ describe('external encoding', () => { }); describe('issue #1290', function() { + + it('should keep query params', function() { + return fetch(`${base}inspect?month=2021-09`) + .then(res => res.json()) + .then(json => { + expect(json.url).to.equal('/inspect?month=2021-09') + }) + }) + it('should handle escaped unicode in URLs', () => { const url = `${base}issues/1290/%E3%81%B2%E3%82%89%E3%81%8C%E3%81%AA`; return fetch(url).then((res) => { From b5417aea6a3275932283a200214522e6ab53f1ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20Unneb=C3=A4ck?= Date: Wed, 22 Sep 2021 11:16:53 +0200 Subject: [PATCH 06/23] fix: import whatwg-url in a way compatible with ESM Node (#1303) * fix: import whatwg-url in a way compatible with ESM Node * release: 2.6.5 --- CHANGELOG.md | 4 ++++ package.json | 2 +- src/request.js | 3 ++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46eef0ff0..29d168cf7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ Changelog # 2.x release +## v2.6.5 + +- Fix: import `whatwg-url` in a way compatible with ESM + ## v2.6.4 - Hotfix: fix v2.6.3 that did not sending query params diff --git a/package.json b/package.json index 98cf5f4c5..4178c1a32 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.6.4", + "version": "2.6.5", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", "browser": "./browser.js", diff --git a/src/request.js b/src/request.js index 3c27c81cf..59850fd95 100644 --- a/src/request.js +++ b/src/request.js @@ -9,11 +9,12 @@ import Url from 'url'; import Stream from 'stream'; -import {URL} from 'whatwg-url'; +import whatwgUrl from 'whatwg-url'; import Headers, { exportNodeCompatibleHeaders } from './headers.js'; import Body, { clone, extractContentType, getTotalBytes } from './body'; const INTERNALS = Symbol('Request internals'); +const URL = whatwgUrl.URL; // fix an issue where "format", "parse" aren't a named export for node <10 const parse_url = Url.parse; From f56b0c66d3dd2ef185436de1f2fd40f66bfea8f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jimmy=20W=C3=A4rting?= Date: Sun, 31 Oct 2021 16:40:17 +0100 Subject: [PATCH 07/23] fix(URL): prefer built in URL version when available and fallback to whatwg (#1352) * fix(URL): prefer built in URL version when available and fallback to whatwg * bump minor --- package.json | 2 +- src/request.js | 2 +- test/test.js | 7 +++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 4178c1a32..ec0510513 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.6.5", + "version": "2.6.6", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", "browser": "./browser.js", diff --git a/src/request.js b/src/request.js index 59850fd95..739ba9071 100644 --- a/src/request.js +++ b/src/request.js @@ -14,7 +14,7 @@ import Headers, { exportNodeCompatibleHeaders } from './headers.js'; import Body, { clone, extractContentType, getTotalBytes } from './body'; const INTERNALS = Symbol('Request internals'); -const URL = whatwgUrl.URL; +const URL = Url.URL || whatwgUrl.URL; // fix an issue where "format", "parse" aren't a named export for node <10 const parse_url = Url.parse; diff --git a/test/test.js b/test/test.js index 9568489e3..6427ae21e 100644 --- a/test/test.js +++ b/test/test.js @@ -2875,4 +2875,11 @@ describe('issue #1290', function() { }); }); }); + + // #1342 + it('should not throw with a valid URL', () => { + const url = 'https://r2---sn-n4v7sney.example.com'; + new Request(url); + }); + }); From 8fe5c4ea66b9b8187600e6d5ec9b1b6781f44009 Mon Sep 17 00:00:00 2001 From: Ciffelia Date: Fri, 5 Nov 2021 22:42:53 +0900 Subject: [PATCH 08/23] 2.x: Specify encoding as an optional peer dependency in package.json (#1310) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Specify `encoding` as an optional peer dependency * Update package.json Co-authored-by: Linus Unnebäck Co-authored-by: Linus Unnebäck --- package.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/package.json b/package.json index ec0510513..6f0ac4302 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,14 @@ "dependencies": { "whatwg-url": "^5.0.0" }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + }, "devDependencies": { "@ungap/url-search-params": "^0.1.2", "abort-controller": "^1.1.0", From 1ef4b560a17e644a02a3bfdea7631ffeee578b35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jimmy=20W=C3=A4rting?= Date: Sun, 16 Jan 2022 12:45:33 +0100 Subject: [PATCH 09/23] backport of #1449 (#1453) * backport of #1449 * bump patch version --- package.json | 2 +- src/index.js | 49 ++++++++++++++++++++++++++++++++++++++++--------- test/server.js | 7 ++++++- test/test.js | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 94 insertions(+), 11 deletions(-) diff --git a/package.json b/package.json index 6f0ac4302..3c1bd8da7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.6.6", + "version": "2.6.7", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", "browser": "./browser.js", diff --git a/src/index.js b/src/index.js index 03b56f733..b210d28e4 100644 --- a/src/index.js +++ b/src/index.js @@ -13,16 +13,29 @@ import https from 'https'; import zlib from 'zlib'; import Stream from 'stream'; -import Body, { writeToStream, getTotalBytes } from './body'; -import Response from './response'; -import Headers, { createHeadersLenient } from './headers'; -import Request, { getNodeRequestOptions } from './request'; -import FetchError from './fetch-error'; -import AbortError from './abort-error'; +import Body, { writeToStream, getTotalBytes } from './body.js'; +import Response from './response.js'; +import Headers, { createHeadersLenient } from './headers.js'; +import Request, { getNodeRequestOptions } from './request.js'; +import FetchError from './fetch-error.js'; +import AbortError from './abort-error.js'; + +import whatwgUrl from 'whatwg-url'; + +const URL = Url.URL || whatwgUrl.URL; // fix an issue where "PassThrough", "resolve" aren't a named export for node <10 const PassThrough = Stream.PassThrough; -const resolve_url = Url.resolve; + +const isDomainOrSubdomain = (destination, original) => { + const orig = new URL(original).hostname; + const dest = new URL(destination).hostname; + + return orig === dest || ( + orig[orig.length - dest.length - 1] === '.' && orig.endsWith(dest) + ); +}; + /** * Fetch function @@ -109,7 +122,19 @@ export default function fetch(url, opts) { const location = headers.get('Location'); // HTTP fetch step 5.3 - const locationURL = location === null ? null : resolve_url(request.url, location); + let locationURL = null; + try { + locationURL = location === null ? null : new URL(location, request.url).toString(); + } catch (err) { + // error here can only be invalid URL in Location: header + // do not throw when options.redirect == manual + // let the user extract the errorneous redirect URL + if (request.redirect !== 'manual') { + reject(new FetchError(`uri requested responds with an invalid redirect URL: ${location}`, 'invalid-redirect')); + finalize(); + return; + } + } // HTTP fetch step 5.5 switch (request.redirect) { @@ -154,9 +179,15 @@ export default function fetch(url, opts) { body: request.body, signal: request.signal, timeout: request.timeout, - size: request.size + size: request.size }; + if (!isDomainOrSubdomain(request.url, locationURL)) { + for (const name of ['authorization', 'www-authenticate', 'cookie', 'cookie2']) { + requestOpts.headers.delete(name); + } + } + // HTTP-redirect fetch step 9 if (res.statusCode !== 303 && request.body && getTotalBytes(request) === null) { reject(new FetchError('Cannot follow redirect with body being a readable stream', 'unsupported-redirect')); diff --git a/test/server.js b/test/server.js index ebd311d9c..2f0baf8cd 100644 --- a/test/server.js +++ b/test/server.js @@ -1,7 +1,6 @@ import * as http from 'http'; import { parse } from 'url'; import * as zlib from 'zlib'; -import * as stream from 'stream'; import { multipart as Multipart } from 'parted'; let convert; @@ -66,6 +65,12 @@ export default class TestServer { })); } + if (p.startsWith('/redirect-to/3')) { + res.statusCode = p.slice(13, 16); + res.setHeader('Location', p.slice(17)); + res.end(); + } + if (p === '/gzip') { res.statusCode = 200; res.setHeader('Content-Type', 'text/plain'); diff --git a/test/test.js b/test/test.js index 6427ae21e..cdeb51f00 100644 --- a/test/test.js +++ b/test/test.js @@ -1569,6 +1569,53 @@ describe('node-fetch', () => { }); }); + it('should not forward secure headers to 3th party', () => { + return fetch(`${base}redirect-to/302/https://httpbin.org/get`, { + headers: new Headers({ + cookie: 'gets=removed', + cookie2: 'gets=removed', + authorization: 'gets=removed', + 'www-authenticate': 'gets=removed', + 'other-safe-headers': 'stays', + 'x-foo': 'bar' + }) + }).then(res => res.json()).then(json => { + const headers = new Headers(json.headers); + // Safe headers are not removed + expect(headers.get('other-safe-headers')).to.equal('stays'); + expect(headers.get('x-foo')).to.equal('bar'); + // Unsafe headers should not have been sent to httpbin + expect(headers.get('cookie')).to.equal(null); + expect(headers.get('cookie2')).to.equal(null); + expect(headers.get('www-authenticate')).to.equal(null); + expect(headers.get('authorization')).to.equal(null); + }); + }); + + it('should forward secure headers to same host', () => { + return fetch(`${base}redirect-to/302/${base}inspect`, { + headers: new Headers({ + cookie: 'is=cookie', + cookie2: 'is=cookie2', + authorization: 'is=authorization', + 'other-safe-headers': 'stays', + 'www-authenticate': 'is=www-authenticate', + 'x-foo': 'bar' + }) + }).then(res => res.json().then(json => { + const headers = new Headers(json.headers); + // Safe headers are not removed + expect(res.url).to.equal(`${base}inspect`); + expect(headers.get('other-safe-headers')).to.equal('stays'); + expect(headers.get('x-foo')).to.equal('bar'); + // Unsafe headers should not have been sent to httpbin + expect(headers.get('cookie')).to.equal('is=cookie'); + expect(headers.get('cookie2')).to.equal('is=cookie2'); + expect(headers.get('www-authenticate')).to.equal('is=www-authenticate'); + expect(headers.get('authorization')).to.equal('is=authorization'); + })); + }); + it('should allow PATCH request', function() { const url = `${base}inspect`; const opts = { From 838d9713ef5e673bbd86768fd22ba98ec461ed9d Mon Sep 17 00:00:00 2001 From: Maciej Goszczycki Date: Mon, 17 Jan 2022 00:40:12 +0100 Subject: [PATCH 10/23] Handle zero-length OK deflate responses (#903) --- src/index.js | 7 +++++++ test/server.js | 7 +++++++ test/test.js | 11 +++++++++++ 3 files changed, 25 insertions(+) diff --git a/src/index.js b/src/index.js index b210d28e4..93945f6d8 100644 --- a/src/index.js +++ b/src/index.js @@ -275,6 +275,13 @@ export default function fetch(url, opts) { response = new Response(body, response_options); resolve(response); }); + raw.on('end', () => { + // some old IIS servers return zero-length OK deflate responses, so 'data' is never emitted. + if (!response) { + response = new Response(body, response_options); + resolve(response); + } + }) return; } diff --git a/test/server.js b/test/server.js index 2f0baf8cd..ffe4733ff 100644 --- a/test/server.js +++ b/test/server.js @@ -120,6 +120,13 @@ export default class TestServer { }); } + if (p === '/empty/deflate') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/plain'); + res.setHeader('Content-Encoding', 'deflate'); + res.end(); + } + if (p === '/sdch') { res.statusCode = 200; res.setHeader('Content-Type', 'text/plain'); diff --git a/test/test.js b/test/test.js index cdeb51f00..719b5b960 100644 --- a/test/test.js +++ b/test/test.js @@ -679,6 +679,17 @@ describe('node-fetch', () => { }); }); + it('should handle empty deflate response', function() { + const url = `${base}empty/deflate`; + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain'); + return res.text().then(result => { + expect(result).to.be.a('string'); + expect(result).to.be.empty; + }); + }); + }); + it('should decompress brotli response', function() { if(typeof zlib.createBrotliDecompress !== 'function') this.skip(); const url = `${base}brotli`; From 50536d1e02ad42bdf262381034805378b98bfa53 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Sat, 16 Jul 2022 13:16:51 +0000 Subject: [PATCH 11/23] fix: premature close with chunked transfer encoding and for async iterators in Node 12 (#1172) * fix: premature close with chunked transfer encoding and for async iterators in Node 12 This PR backports the fix from #1064 to the `2.x.x` branch following the [comment here](https://github.com/node-fetch/node-fetch/pull/1064#issuecomment-849167400). I had to add some extra babel config to allow using the `for await..of` syntax in the tests. The config is only needed for the tests as this syntax is not used in the implementation. * chore: fix up tests for node 6+ * chore: codecov dropped support for node < 8 without shipping major * chore: npm7 strips empty dependencies hash during install * chore: pin deps to versions that work on node 4 * chore: do not emit close error after aborting a request * chore: test on node 4-16 * chore: simplify chunked transer encoding bad ending * chore: avoid calling .destroy as it is not in every node.js release * chore: listen for response close as socket is reused and shows warnings --- .babelrc | 6 ++-- .travis.yml | 2 ++ README.md | 43 +++++++++++++++++++++++++++++ package.json | 4 ++- src/index.js | 68 +++++++++++++++++++++++++++++++++++++++++++++- test/server.js | 28 +++++++++++++++++++ test/test.js | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 221 insertions(+), 4 deletions(-) diff --git a/.babelrc b/.babelrc index 6a95c25e7..901200bce 100644 --- a/.babelrc +++ b/.babelrc @@ -14,7 +14,8 @@ } ] ], plugins: [ - './build/babel-plugin' + './build/babel-plugin', + 'transform-async-generator-functions' ] }, coverage: { @@ -31,7 +32,8 @@ ], plugins: [ [ 'istanbul', { exclude: [ 'src/blob.js', 'build', 'test' ] } ], - './build/babel-plugin' + './build/babel-plugin', + 'transform-async-generator-functions' ] }, rollup: { diff --git a/.travis.yml b/.travis.yml index 3bb109e15..7d7081b33 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,8 @@ node_js: - "6" - "8" - "10" + - "12" + - "14" - "node" env: - FORMDATA_VERSION=1.0.0 diff --git a/README.md b/README.md index 2dde74289..4f87a59a0 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,49 @@ fetch('https://assets-cdn.github.com/images/modules/logos_page/Octocat.png') }); ``` +In Node.js 14 you can also use async iterators to read `body`; however, be careful to catch +errors -- the longer a response runs, the more likely it is to encounter an error. + +```js +const fetch = require('node-fetch'); +const response = await fetch('https://httpbin.org/stream/3'); +try { + for await (const chunk of response.body) { + console.dir(JSON.parse(chunk.toString())); + } +} catch (err) { + console.error(err.stack); +} +``` + +In Node.js 12 you can also use async iterators to read `body`; however, async iterators with streams +did not mature until Node.js 14, so you need to do some extra work to ensure you handle errors +directly from the stream and wait on it response to fully close. + +```js +const fetch = require('node-fetch'); +const read = async body => { + let error; + body.on('error', err => { + error = err; + }); + for await (const chunk of body) { + console.dir(JSON.parse(chunk.toString())); + } + return new Promise((resolve, reject) => { + body.on('close', () => { + error ? reject(error) : resolve(); + }); + }); +}; +try { + const response = await fetch('https://httpbin.org/stream/3'); + await read(response.body); +} catch (err) { + console.error(err.stack); +} +``` + #### Buffer If you prefer to cache binary data in full, use buffer(). (NOTE: `buffer()` is a `node-fetch`-only API) diff --git a/package.json b/package.json index 3c1bd8da7..ace7d8c4f 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,9 @@ "abortcontroller-polyfill": "^1.3.0", "babel-core": "^6.26.3", "babel-plugin-istanbul": "^4.1.6", - "babel-preset-env": "^1.6.1", + "babel-plugin-transform-async-generator-functions": "^6.24.1", + "babel-polyfill": "^6.26.0", + "babel-preset-env": "1.4.0", "babel-register": "^6.16.3", "chai": "^3.5.0", "chai-as-promised": "^7.1.1", diff --git a/src/index.js b/src/index.js index 93945f6d8..1a25e800d 100644 --- a/src/index.js +++ b/src/index.js @@ -67,7 +67,7 @@ export default function fetch(url, opts) { let error = new AbortError('The user aborted a request.'); reject(error); if (request.body && request.body instanceof Stream.Readable) { - request.body.destroy(error); + destroyStream(request.body, error); } if (!response || !response.body) return; response.body.emit('error', error); @@ -108,9 +108,41 @@ export default function fetch(url, opts) { req.on('error', err => { reject(new FetchError(`request to ${request.url} failed, reason: ${err.message}`, 'system', err)); + + if (response && response.body) { + destroyStream(response.body, err); + } + finalize(); }); + fixResponseChunkedTransferBadEnding(req, err => { + if (signal && signal.aborted) { + return + } + + destroyStream(response.body, err); + }); + + /* c8 ignore next 18 */ + if (parseInt(process.version.substring(1)) < 14) { + // Before Node.js 14, pipeline() does not fully support async iterators and does not always + // properly handle when the socket close/end events are out of order. + req.on('socket', s => { + s.addListener('close', hadError => { + // if a data listener is still present we didn't end cleanly + const hasDataListener = s.listenerCount('data') > 0 + + // if end happened before close but the socket didn't emit an error, do it now + if (response && hasDataListener && !hadError && !(signal && signal.aborted)) { + const err = new Error('Premature close'); + err.code = 'ERR_STREAM_PREMATURE_CLOSE'; + response.body.emit('error', err); + } + }); + }); + } + req.on('response', res => { clearTimeout(reqTimeout); @@ -303,6 +335,40 @@ export default function fetch(url, opts) { }; +function fixResponseChunkedTransferBadEnding(request, errorCallback) { + let socket; + + request.on('socket', s => { + socket = s; + }); + + request.on('response', response => { + const {headers} = response; + if (headers['transfer-encoding'] === 'chunked' && !headers['content-length']) { + response.once('close', hadError => { + // if a data listener is still present we didn't end cleanly + const hasDataListener = socket.listenerCount('data') > 0; + + if (hasDataListener && !hadError) { + const err = new Error('Premature close'); + err.code = 'ERR_STREAM_PREMATURE_CLOSE'; + errorCallback(err); + } + }); + } + }); +} + +function destroyStream (stream, err) { + if (stream.destroy) { + stream.destroy(err); + } else { + // node < 8 + stream.emit('error', err); + stream.end(); + } +} + /** * Redirect code matching * diff --git a/test/server.js b/test/server.js index ffe4733ff..ffff88877 100644 --- a/test/server.js +++ b/test/server.js @@ -329,6 +329,34 @@ export default class TestServer { res.destroy(); } + if (p === '/error/premature/chunked') { + res.writeHead(200, { + 'Content-Type': 'application/json', + 'Transfer-Encoding': 'chunked' + }); + + // Transfer-Encoding: 'chunked' sends chunk sizes followed by the + // chunks - https://en.wikipedia.org/wiki/Chunked_transfer_encoding + const sendChunk = (obj) => { + const data = JSON.stringify(obj) + + res.write(`${data.length}\r\n`) + res.write(`${data}\r\n`) + } + + sendChunk({data: 'hi'}) + + setTimeout(() => { + sendChunk({data: 'bye'}) + }, 200); + + setTimeout(() => { + // should send '0\r\n\r\n' to end the response properly but instead + // just close the connection + res.destroy(); + }, 400); + } + if (p === '/error/json') { res.statusCode = 200; res.setHeader('Content-Type', 'application/json'); diff --git a/test/test.js b/test/test.js index 719b5b960..36fe7703e 100644 --- a/test/test.js +++ b/test/test.js @@ -1,4 +1,7 @@ +import 'babel-core/register' +import 'babel-polyfill' + // test tools import chai from 'chai'; import chaiPromised from 'chai-as-promised'; @@ -552,6 +555,77 @@ describe('node-fetch', () => { .and.have.property('code', 'ECONNRESET'); }); + it('should handle network-error in chunked response', () => { + const url = `${base}error/premature/chunked`; + return fetch(url).then(res => { + expect(res.status).to.equal(200); + expect(res.ok).to.be.true; + + return expect(new Promise((resolve, reject) => { + res.body.on('error', reject); + res.body.on('close', resolve); + })).to.eventually.be.rejectedWith(Error, 'Premature close') + .and.have.property('code', 'ERR_STREAM_PREMATURE_CLOSE'); + }); + }); + + // Skip test if streams are not async iterators (node < 10) + const itAsyncIterator = Boolean(new stream.PassThrough()[Symbol.asyncIterator]) ? it : it.skip; + + itAsyncIterator('should handle network-error in chunked response async iterator', () => { + const url = `${base}error/premature/chunked`; + return fetch(url).then(res => { + expect(res.status).to.equal(200); + expect(res.ok).to.be.true; + + const read = async body => { + const chunks = []; + + if (process.version < 'v14') { + // In Node.js 12, some errors don't come out in the async iterator; we have to pick + // them up from the event-emitter and then throw them after the async iterator + let error; + body.on('error', err => { + error = err; + }); + + for await (const chunk of body) { + chunks.push(chunk); + } + + if (error) { + throw error; + } + + return new Promise(resolve => { + body.on('close', () => resolve(chunks)); + }); + } + + for await (const chunk of body) { + chunks.push(chunk); + } + + return chunks; + }; + + return expect(read(res.body)) + .to.eventually.be.rejectedWith(Error, 'Premature close') + .and.have.property('code', 'ERR_STREAM_PREMATURE_CLOSE'); + }); + }); + + it('should handle network-error in chunked response in consumeBody', () => { + const url = `${base}error/premature/chunked`; + return fetch(url).then(res => { + expect(res.status).to.equal(200); + expect(res.ok).to.be.true; + + return expect(res.text()) + .to.eventually.be.rejectedWith(Error, 'Premature close'); + }); + }); + it('should handle DNS-error response', function() { const url = 'http://domain.invalid'; return expect(fetch(url)).to.eventually.be.rejected From fddad0e7ea3fd6da01cc006fdf0ed304ccdd7990 Mon Sep 17 00:00:00 2001 From: victal Date: Tue, 19 Jul 2022 17:38:01 -0300 Subject: [PATCH 12/23] fix(headers): don't forward secure headers on protocol change (#1605) backport for #1599 to the 2.x branch Co-authored-by: Guilherme Victal --- src/index.js | 16 +++++++++++++++- test/test.js | 23 +++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/index.js b/src/index.js index 1a25e800d..f39ed3a93 100644 --- a/src/index.js +++ b/src/index.js @@ -36,6 +36,20 @@ const isDomainOrSubdomain = (destination, original) => { ); }; +/** + * isSameProtocol reports whether the two provided URLs use the same protocol. + * + * Both domains must already be in canonical form. + * @param {string|URL} original + * @param {string|URL} destination + */ +const isSameProtocol = (destination, original) => { + const orig = new URL(original).protocol; + const dest = new URL(destination).protocol; + + return orig === dest; +}; + /** * Fetch function @@ -214,7 +228,7 @@ export default function fetch(url, opts) { size: request.size }; - if (!isDomainOrSubdomain(request.url, locationURL)) { + if (!isDomainOrSubdomain(request.url, locationURL) || !isSameProtocol(request.url, locationURL)) { for (const name of ['authorization', 'www-authenticate', 'cookie', 'cookie2']) { requestOpts.headers.delete(name); } diff --git a/test/test.js b/test/test.js index 36fe7703e..21cf24055 100644 --- a/test/test.js +++ b/test/test.js @@ -1677,6 +1677,29 @@ describe('node-fetch', () => { }); }); + it('should not forward secure headers to changed protocol', async () => { + const res = await fetch('https://httpbin.org/redirect-to?url=http%3A%2F%2Fhttpbin.org%2Fget&status_code=302', { + headers: new Headers({ + cookie: 'gets=removed', + cookie2: 'gets=removed', + authorization: 'gets=removed', + 'www-authenticate': 'gets=removed', + 'other-safe-headers': 'stays', + 'x-foo': 'bar' + }) + }); + + const headers = new Headers((await res.json()).headers); + // Safe headers are not removed + expect(headers.get('other-safe-headers')).to.equal('stays'); + expect(headers.get('x-foo')).to.equal('bar'); + // Unsafe headers should not have been sent to downgraded http + expect(headers.get('cookie')).to.equal(null); + expect(headers.get('cookie2')).to.equal(null); + expect(headers.get('www-authenticate')).to.equal(null); + expect(headers.get('authorization')).to.equal(null); + }); + it('should forward secure headers to same host', () => { return fetch(`${base}redirect-to/302/${base}inspect`, { headers: new Headers({ From e218f8d5b7c6ad48b3a6c8e85bc65948ed295b26 Mon Sep 17 00:00:00 2001 From: Seth Westphal Date: Sun, 31 Jul 2022 10:01:53 -0500 Subject: [PATCH 13/23] Add missing changelog entries. (#1613) --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29d168cf7..3d98f588f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ Changelog # 2.x release +## v2.6.7 + +- Fix: don't forward secure headers to 3th party + +## v2.6.6 + +- Fix: prefer built in URL version when available and fallback to whatwg + ## v2.6.5 - Fix: import `whatwg-url` in a way compatible with ESM From 8bb6e317c866c4134e7d67e90a5596a8c67e3965 Mon Sep 17 00:00:00 2001 From: Val Date: Tue, 8 Nov 2022 07:46:25 -0500 Subject: [PATCH 14/23] fix: prevent hoisting of the undefined `global` variable in `browser.js` (#1534) Because of JS hoisting `var global` to the top of the file, `typeof global` in `getGlobal()` will always be `undefined`. By using a different variable name like `globalObject`, we are able to read the "real" `typeof global` and get access to the global object that way. --- browser.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/browser.js b/browser.js index 83c54c584..7035edbed 100644 --- a/browser.js +++ b/browser.js @@ -11,15 +11,15 @@ var getGlobal = function () { throw new Error('unable to locate global object'); } -var global = getGlobal(); +var globalObject = getGlobal(); -module.exports = exports = global.fetch; +module.exports = exports = globalObject.fetch; // Needed for TypeScript and Webpack. -if (global.fetch) { - exports.default = global.fetch.bind(global); +if (globalObject.fetch) { + exports.default = globalObject.fetch.bind(global); } -exports.Headers = global.Headers; -exports.Request = global.Request; -exports.Response = global.Response; \ No newline at end of file +exports.Headers = globalObject.Headers; +exports.Request = globalObject.Request; +exports.Response = globalObject.Response; From 1768eaa7dcc51adc0038cb07e2cdfd6d44b2164a Mon Sep 17 00:00:00 2001 From: Gregor Martynus <39992+gr2m@users.noreply.github.com> Date: Thu, 12 Jan 2023 16:55:10 -0800 Subject: [PATCH 15/23] ci(release): initial version --- .github/workflows/release.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..881b4cb69 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,22 @@ +name: Release +on: + push: + branches: + - main + - next + - beta + - "*.x" # maintenance releases such as 2.x + +jobs: + release: + name: release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: 16 + - run: npx semantic-release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} From ce37bcd93e869e2c0a05d4a913ad08ce94399e88 Mon Sep 17 00:00:00 2001 From: Gregor Martynus <39992+gr2m@users.noreply.github.com> Date: Thu, 12 Jan 2023 16:56:34 -0800 Subject: [PATCH 16/23] ci(semantic-release): config --- package.json | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/package.json b/package.json index ace7d8c4f..66f59adc6 100644 --- a/package.json +++ b/package.json @@ -74,5 +74,16 @@ "rollup-plugin-babel": "^3.0.7", "string-to-arraybuffer": "^1.0.2", "teeny-request": "3.7.0" + }, + "release": { + "branches": [ + "+([0-9]).x", + "main", + "next", + { + "name": "beta", + "prerelease": true + } + ] } } From 49bef02a2f630bb083d1920cb40ff09363479ef2 Mon Sep 17 00:00:00 2001 From: Gregor Martynus <39992+gr2m@users.noreply.github.com> Date: Thu, 12 Jan 2023 16:57:50 -0800 Subject: [PATCH 17/23] ci(release): use latest Node LTS --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 881b4cb69..236ef0ae3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: - node-version: 16 + node-version: "lts/*" - run: npx semantic-release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From dd2a0ba0fb1ed0d321fcde46562e824d9f40fea1 Mon Sep 17 00:00:00 2001 From: Gregor Martynus <39992+gr2m@users.noreply.github.com> Date: Thu, 12 Jan 2023 17:01:08 -0800 Subject: [PATCH 18/23] ci(release): install dependencies --- .github/workflows/release.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 236ef0ae3..e116bd611 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,10 +12,11 @@ jobs: name: release runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 with: node-version: "lts/*" + - run: npm ci - run: npx semantic-release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 6e9464d7e34dc323edf4dabad7615dd94ab847bd Mon Sep 17 00:00:00 2001 From: Gregor Martynus <39992+gr2m@users.noreply.github.com> Date: Thu, 12 Jan 2023 17:02:06 -0800 Subject: [PATCH 19/23] ci(release): install dependencies --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e116bd611..315e6c813 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/setup-node@v3 with: node-version: "lts/*" - - run: npm ci + - run: npm install - run: npx semantic-release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 0f1ebb0d9d9726351a83a50eaaccf66342f04e06 Mon Sep 17 00:00:00 2001 From: Amit Agarwal <1344071+labnol@users.noreply.github.com> Date: Mon, 23 Jan 2023 21:08:22 +0530 Subject: [PATCH 20/23] Prevent error when response is null (#1699) Fix for this error. TypeError: Cannot read properties of null (reading 'body') --- src/index.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/index.js b/src/index.js index f39ed3a93..9013f1de2 100644 --- a/src/index.js +++ b/src/index.js @@ -135,7 +135,9 @@ export default function fetch(url, opts) { return } - destroyStream(response.body, err); + if (response && response.body) { + destroyStream(response.body, err); + } }); /* c8 ignore next 18 */ From 70f592d9d2da959df1cebc2dd2314286a4bcf345 Mon Sep 17 00:00:00 2001 From: Segev Finer Date: Mon, 30 Jan 2023 23:59:17 +0200 Subject: [PATCH 21/23] fix: "global is not defined" (#1704) --- browser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/browser.js b/browser.js index 7035edbed..ee86265ae 100644 --- a/browser.js +++ b/browser.js @@ -17,7 +17,7 @@ module.exports = exports = globalObject.fetch; // Needed for TypeScript and Webpack. if (globalObject.fetch) { - exports.default = globalObject.fetch.bind(global); + exports.default = globalObject.fetch.bind(globalObject); } exports.Headers = globalObject.Headers; From 29909d75c62d51e0d1c23758e526dba74bfd463d Mon Sep 17 00:00:00 2001 From: Alexis Clarembeau Date: Mon, 8 May 2023 18:19:42 +0200 Subject: [PATCH 22/23] fix: handle bom in text and json (#1739) * fix: handle bom in text and json * add unit tests --- package.json | 2 +- src/body.js | 8 ++++---- test/test.js | 15 +++++++++++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 66f59adc6..92f958869 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "dependencies": { "whatwg-url": "^5.0.0" }, - "peerDependencies": { + "peerDependencies": { "encoding": "^0.1.0" }, "peerDependenciesMeta": { diff --git a/src/body.js b/src/body.js index a9d2e7973..273044efe 100644 --- a/src/body.js +++ b/src/body.js @@ -114,9 +114,9 @@ Body.prototype = { * @return Promise */ json() { - return consumeBody.call(this).then((buffer) => { - try { - return JSON.parse(buffer.toString()); + return this.text().then((text) => { + try{ + return JSON.parse(text); } catch (err) { return Body.Promise.reject(new FetchError(`invalid json response body at ${this.url} reason: ${err.message}`, 'invalid-json')); } @@ -129,7 +129,7 @@ Body.prototype = { * @return Promise */ text() { - return consumeBody.call(this).then(buffer => buffer.toString()); + return consumeBody.call(this).then(buffer => new TextDecoder().decode(buffer)); }, /** diff --git a/test/test.js b/test/test.js index 21cf24055..0939e0f1f 100644 --- a/test/test.js +++ b/test/test.js @@ -2479,6 +2479,21 @@ describe('Response', function () { expect(res.headers.get('a')).to.equal('1'); }); + it('should decode responses containing BOM to json', async () => { + const json = await new Response('\uFEFF{"a":1}').json(); + expect(json.a).to.equal(1); + }); + + it('should decode responses containing BOM to text', async () => { + const text = await new Response('\uFEFF{"a":1}').text(); + expect(text).to.equal('{"a":1}'); + }); + + it('should keep BOM when getting raw bytes', async () => { + const ab = await new Response('\uFEFF{"a":1}').arrayBuffer(); + expect(ab.byteLength).to.equal(10); + }); + it('should support text() method', function() { const res = new Response('a=1'); return res.text().then(result => { From 921280de9d469f03e235a3cf3a31c12481787cf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jimmy=20W=C3=A4rting?= Date: Tue, 9 May 2023 07:09:50 +0200 Subject: [PATCH 23/23] Revert "fix: handle bom in text and json (#1739)" This reverts commit 29909d75c62d51e0d1c23758e526dba74bfd463d. --- package.json | 2 +- src/body.js | 8 ++++---- test/test.js | 15 --------------- 3 files changed, 5 insertions(+), 20 deletions(-) diff --git a/package.json b/package.json index 92f958869..66f59adc6 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "dependencies": { "whatwg-url": "^5.0.0" }, - "peerDependencies": { + "peerDependencies": { "encoding": "^0.1.0" }, "peerDependenciesMeta": { diff --git a/src/body.js b/src/body.js index 273044efe..a9d2e7973 100644 --- a/src/body.js +++ b/src/body.js @@ -114,9 +114,9 @@ Body.prototype = { * @return Promise */ json() { - return this.text().then((text) => { - try{ - return JSON.parse(text); + return consumeBody.call(this).then((buffer) => { + try { + return JSON.parse(buffer.toString()); } catch (err) { return Body.Promise.reject(new FetchError(`invalid json response body at ${this.url} reason: ${err.message}`, 'invalid-json')); } @@ -129,7 +129,7 @@ Body.prototype = { * @return Promise */ text() { - return consumeBody.call(this).then(buffer => new TextDecoder().decode(buffer)); + return consumeBody.call(this).then(buffer => buffer.toString()); }, /** diff --git a/test/test.js b/test/test.js index 0939e0f1f..21cf24055 100644 --- a/test/test.js +++ b/test/test.js @@ -2479,21 +2479,6 @@ describe('Response', function () { expect(res.headers.get('a')).to.equal('1'); }); - it('should decode responses containing BOM to json', async () => { - const json = await new Response('\uFEFF{"a":1}').json(); - expect(json.a).to.equal(1); - }); - - it('should decode responses containing BOM to text', async () => { - const text = await new Response('\uFEFF{"a":1}').text(); - expect(text).to.equal('{"a":1}'); - }); - - it('should keep BOM when getting raw bytes', async () => { - const ab = await new Response('\uFEFF{"a":1}').arrayBuffer(); - expect(ab.byteLength).to.equal(10); - }); - it('should support text() method', function() { const res = new Response('a=1'); return res.text().then(result => {