From 7c26fa94796687957a11f562802f2f24a838b57d Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Tue, 4 Oct 2016 02:37:49 -0700 Subject: [PATCH 001/223] Add Babel infrastructure No actual code has been changed yet. --- .gitignore | 6 +++++- package.json | 43 ++++++++++++++++++++++++++++++++----- {lib => src}/body.js | 0 {lib => src}/fetch-error.js | 0 {lib => src}/headers.js | 0 index.js => src/index.js | 10 ++++----- {lib => src}/request.js | 0 {lib => src}/response.js | 0 test/test.js | 14 ++++++------ 9 files changed, 55 insertions(+), 18 deletions(-) rename {lib => src}/body.js (100%) rename {lib => src}/fetch-error.js (100%) rename {lib => src}/headers.js (100%) rename index.js => src/index.js (97%) rename {lib => src}/request.js (100%) rename {lib => src}/response.js (100%) diff --git a/.gitignore b/.gitignore index a2234e079..99c8c2d57 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,8 @@ pids # Directory for instrumented libs generated by jscoverage/JSCover lib-cov -# Coverage directory used by tools like istanbul +# Coverage directory used by tools like nyc and istanbul +.nyc_output coverage # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) @@ -32,3 +33,6 @@ node_modules # Coveralls token files .coveralls.yml + +# Babel-compiled files +lib diff --git a/package.json b/package.json index 262708091..c1128264a 100644 --- a/package.json +++ b/package.json @@ -2,11 +2,13 @@ "name": "node-fetch", "version": "1.6.3", "description": "A light-weight module that brings window.fetch to node.js and io.js", - "main": "index.js", + "main": "lib/index.js", "scripts": { - "test": "mocha test/test.js", - "report": "istanbul cover _mocha -- -R spec test/test.js", - "coverage": "istanbul cover _mocha --report lcovonly -- -R spec test/test.js && cat ./coverage/lcov.info | coveralls" + "build": "babel -d lib src", + "prepublish": "npm run build", + "test": "mocha --compilers js:babel-register test/test.js", + "report": "cross-env BABEL_ENV=test nyc --reporter lcov --reporter text mocha -R spec test/test.js", + "coverage": "cross-env BABEL_ENV=test nyc --reporter lcovonly mocha -R spec test/test.js && cat ./coverage/lcov.info | coveralls" }, "repository": { "type": "git", @@ -24,13 +26,19 @@ }, "homepage": "https://github.com/bitinn/node-fetch", "devDependencies": { + "babel-cli": "^6.16.0", + "babel-plugin-istanbul": "^2.0.1", + "babel-plugin-transform-runtime": "^6.15.0", + "babel-preset-es2015": "^6.16.0", + "babel-register": "^6.16.3", "bluebird": "^3.3.4", "chai": "^3.5.0", "chai-as-promised": "^5.2.0", "coveralls": "^2.11.2", + "cross-env": "^3.0.0", "form-data": ">=1.0.0", - "istanbul": "^0.4.2", "mocha": "^2.1.0", + "nyc": "^8.3.0", "parted": "^0.1.1", "promise": "^7.1.1", "resumer": "0.0.0" @@ -38,5 +46,30 @@ "dependencies": { "encoding": "^0.1.11", "is-stream": "^1.0.1" + }, + "babel": { + "presets": [ + "es2015" + ], + "plugins": [ + "transform-runtime" + ], + "env": { + "test": { + "plugins": [ + "istanbul" + ] + } + } + }, + "nyc": { + "include": [ + "src/*.js" + ], + "require": [ + "babel-register" + ], + "sourceMap": false, + "instrument": false } } diff --git a/lib/body.js b/src/body.js similarity index 100% rename from lib/body.js rename to src/body.js diff --git a/lib/fetch-error.js b/src/fetch-error.js similarity index 100% rename from lib/fetch-error.js rename to src/fetch-error.js diff --git a/lib/headers.js b/src/headers.js similarity index 100% rename from lib/headers.js rename to src/headers.js diff --git a/index.js b/src/index.js similarity index 97% rename from index.js rename to src/index.js index df89c80c7..676b57a85 100644 --- a/index.js +++ b/src/index.js @@ -12,11 +12,11 @@ var https = require('https'); var zlib = require('zlib'); var stream = require('stream'); -var Body = require('./lib/body'); -var Response = require('./lib/response'); -var Headers = require('./lib/headers'); -var Request = require('./lib/request'); -var FetchError = require('./lib/fetch-error'); +var Body = require('./body'); +var Response = require('./response'); +var Headers = require('./headers'); +var Request = require('./request'); +var FetchError = require('./fetch-error'); // commonjs module.exports = Fetch; diff --git a/lib/request.js b/src/request.js similarity index 100% rename from lib/request.js rename to src/request.js diff --git a/lib/response.js b/src/response.js similarity index 100% rename from lib/response.js rename to src/response.js diff --git a/test/test.js b/test/test.js index 6067ccdb0..a7ce43e37 100644 --- a/test/test.js +++ b/test/test.js @@ -16,12 +16,12 @@ var fs = require('fs'); var TestServer = require('./server'); // test subjects -var fetch = require('../index.js'); -var Headers = require('../lib/headers.js'); -var Response = require('../lib/response.js'); -var Request = require('../lib/request.js'); -var Body = require('../lib/body.js'); -var FetchError = require('../lib/fetch-error.js'); +var fetch = require('../src/index.js'); +var Headers = require('../src/headers.js'); +var Response = require('../src/response.js'); +var Request = require('../src/request.js'); +var Body = require('../src/body.js'); +var FetchError = require('../src/fetch-error.js'); // test with native promise on node 0.11, and bluebird for node 0.10 fetch.Promise = fetch.Promise || bluebird; @@ -1122,7 +1122,7 @@ describe('node-fetch', function() { result.push([key, val]); }); - expected = [ + var expected = [ ["a", "1"] , ["b", "2"] , ["b", "3"] From 2874af4218c059e9990de47596d0d9da1872cdf9 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Tue, 4 Oct 2016 03:07:44 -0700 Subject: [PATCH 002/223] Use cross-env@2.0.1 to support Node.js < v4 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c1128264a..7014d4323 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "chai": "^3.5.0", "chai-as-promised": "^5.2.0", "coveralls": "^2.11.2", - "cross-env": "^3.0.0", + "cross-env": "2.0.1", "form-data": ">=1.0.0", "mocha": "^2.1.0", "nyc": "^8.3.0", From f30cce86c47432e3e8505ae127927e1926733eef Mon Sep 17 00:00:00 2001 From: Chris Veness Date: Sat, 8 Oct 2016 21:35:19 +0100 Subject: [PATCH 003/223] Add 'post JSON' example to readme (#173) --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 96d69f68f..1428f9273 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,20 @@ fetch('http://httpbin.org/post', { method: 'POST', body: stream }) console.log(json); }); +// post with JSON + +var body = { a: 1 }; +fetch('http://httpbin.org/post', { + method: 'POST', + body: JSON.stringify(body), + headers: { 'Content-Type': 'application/json' }, +}) + .then(function(res) { + return res.json(); + }).then(function(json) { + console.log(json); + }); + // post with form-data (detect multipart) var FormData = require('form-data'); From e67b3a8579e00b90f036c3e81f3083fac58ce1e2 Mon Sep 17 00:00:00 2001 From: Dan Dascalescu Date: Sat, 8 Oct 2016 13:37:49 -0700 Subject: [PATCH 004/223] Fix run-on sentence about Options (#168) [ci skip] --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1428f9273..5a4f380f3 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,7 @@ Should be an absolute url, eg `http://example.com/` ### Options -default values are shown, note that only `method`, `headers`, `redirect` and `body` are allowed in `window.fetch`, others are node.js extensions. +Note that only `method`, `headers`, `redirect` and `body` are allowed in `window.fetch`. Other options are node.js extensions. The default values are shown after each option key. ``` { From 993d4cdea1b63db4ec41a994873e6044f26cde85 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sat, 8 Oct 2016 20:51:01 -0700 Subject: [PATCH 005/223] Convert Headers to ES2015 and implement Iterable interface (#180) Closes #127, #174. --- src/headers.js | 296 ++++++++++++++++++++++++++++++------------------ src/index.js | 2 +- src/request.js | 2 +- src/response.js | 2 +- test/test.js | 113 ++++++++++++++---- 5 files changed, 276 insertions(+), 139 deletions(-) diff --git a/src/headers.js b/src/headers.js index fd7a14eaa..ad2e9e4aa 100644 --- a/src/headers.js +++ b/src/headers.js @@ -5,137 +5,211 @@ * Headers class offers convenient helpers */ -module.exports = Headers; - -/** - * Headers class - * - * @param Object headers Response headers - * @return Void - */ -function Headers(headers) { +export const MAP = Symbol('map'); + +export default class Headers { + /** + * Headers class + * + * @param Object headers Response headers + * @return Void + */ + constructor(headers) { + this[MAP] = {}; + + // Headers + if (headers instanceof Headers) { + headers = headers.raw(); + } - var self = this; - this._headers = {}; + // plain object + for (const prop in headers) { + if (!headers.hasOwnProperty(prop)) { + continue; + } + + if (typeof headers[prop] === 'string') { + this.set(prop, headers[prop]); + } else if (typeof headers[prop] === 'number' && !isNaN(headers[prop])) { + this.set(prop, headers[prop].toString()); + } else if (headers[prop] instanceof Array) { + headers[prop].forEach(item => { + this.append(prop, item.toString()); + }); + } + } + } - // Headers - if (headers instanceof Headers) { - headers = headers.raw(); + /** + * Return first header value given name + * + * @param String name Header name + * @return Mixed + */ + get(name) { + const list = this[MAP][name.toLowerCase()]; + return list ? list[0] : null; } - // plain object - for (var prop in headers) { - if (!headers.hasOwnProperty(prop)) { - continue; + /** + * Return all header values given name + * + * @param String name Header name + * @return Array + */ + getAll(name) { + if (!this.has(name)) { + return []; } - if (typeof headers[prop] === 'string') { - this.set(prop, headers[prop]); - - } else if (typeof headers[prop] === 'number' && !isNaN(headers[prop])) { - this.set(prop, headers[prop].toString()); + return this[MAP][name.toLowerCase()]; + } - } else if (headers[prop] instanceof Array) { - headers[prop].forEach(function(item) { - self.append(prop, item.toString()); + /** + * Iterate over all headers + * + * @param Function callback Executed for each item with parameters (value, name, thisArg) + * @param Boolean thisArg `this` context for callback function + * @return Void + */ + forEach(callback, thisArg) { + Object.getOwnPropertyNames(this[MAP]).forEach(name => { + this[MAP][name].forEach(value => { + callback.call(thisArg, value, name, this); }); + }); + } + + /** + * Overwrite header values given name + * + * @param String name Header name + * @param String value Header value + * @return Void + */ + set(name, value) { + this[MAP][name.toLowerCase()] = [value]; + } + + /** + * Append a value onto existing header + * + * @param String name Header name + * @param String value Header value + * @return Void + */ + append(name, value) { + if (!this.has(name)) { + this.set(name, value); + return; } + + this[MAP][name.toLowerCase()].push(value); } -} + /** + * Check for header name existence + * + * @param String name Header name + * @return Boolean + */ + has(name) { + return this[MAP].hasOwnProperty(name.toLowerCase()); + } -/** - * Return first header value given name - * - * @param String name Header name - * @return Mixed - */ -Headers.prototype.get = function(name) { - var list = this._headers[name.toLowerCase()]; - return list ? list[0] : null; -}; + /** + * Delete all header values given name + * + * @param String name Header name + * @return Void + */ + delete(name) { + delete this[MAP][name.toLowerCase()]; + }; + + /** + * Return raw headers (non-spec api) + * + * @return Object + */ + raw() { + return this[MAP]; + } -/** - * Return all header values given name - * - * @param String name Header name - * @return Array - */ -Headers.prototype.getAll = function(name) { - if (!this.has(name)) { - return []; + /** + * Get an iterator on keys. + * + * @return Iterator + */ + keys() { + const keys = []; + this.forEach((_, name) => keys.push(name)); + return new Iterator(keys); } - return this._headers[name.toLowerCase()]; -}; + /** + * Get an iterator on values. + * + * @return Iterator + */ + values() { + const values = []; + this.forEach(value => values.push(value)); + return new Iterator(values); + } -/** - * Iterate over all headers - * - * @param Function callback Executed for each item with parameters (value, name, thisArg) - * @param Boolean thisArg `this` context for callback function - * @return Void - */ -Headers.prototype.forEach = function(callback, thisArg) { - Object.getOwnPropertyNames(this._headers).forEach(function(name) { - this._headers[name].forEach(function(value) { - callback.call(thisArg, value, name, this) - }, this) - }, this) -} + /** + * Get an iterator on entries. + * + * @return Iterator + */ + entries() { + const entries = []; + this.forEach((value, name) => entries.push([name, value])); + return new Iterator(entries); + } -/** - * Overwrite header values given name - * - * @param String name Header name - * @param String value Header value - * @return Void - */ -Headers.prototype.set = function(name, value) { - this._headers[name.toLowerCase()] = [value]; -}; + /** + * Get an iterator on entries. + * + * This is the default iterator of the Headers object. + * + * @return Iterator + */ + [Symbol.iterator]() { + return this.entries(); + } -/** - * Append a value onto existing header - * - * @param String name Header name - * @param String value Header value - * @return Void - */ -Headers.prototype.append = function(name, value) { - if (!this.has(name)) { - this.set(name, value); - return; + /** + * Tag used by `Object.prototype.toString()`. + */ + get [Symbol.toStringTag]() { + return 'Headers'; } +} - this._headers[name.toLowerCase()].push(value); -}; +const ITEMS = Symbol('items'); +class Iterator { + constructor(items) { + this[ITEMS] = items; + } -/** - * Check for header name existence - * - * @param String name Header name - * @return Boolean - */ -Headers.prototype.has = function(name) { - return this._headers.hasOwnProperty(name.toLowerCase()); -}; + next() { + if (!this[ITEMS].length) { + return { + value: undefined, + done: true + }; + } -/** - * Delete all header values given name - * - * @param String name Header name - * @return Void - */ -Headers.prototype['delete'] = function(name) { - delete this._headers[name.toLowerCase()]; -}; + return { + value: this[ITEMS].shift(), + done: false + }; -/** - * Return raw headers (non-spec api) - * - * @return Object - */ -Headers.prototype.raw = function() { - return this._headers; -}; + } + + [Symbol.iterator]() { + return this; + } +} diff --git a/src/index.js b/src/index.js index 676b57a85..d30c2d826 100644 --- a/src/index.js +++ b/src/index.js @@ -14,7 +14,7 @@ var stream = require('stream'); var Body = require('./body'); var Response = require('./response'); -var Headers = require('./headers'); +import Headers from './headers'; var Request = require('./request'); var FetchError = require('./fetch-error'); diff --git a/src/request.js b/src/request.js index 1a29c29c6..ec80635af 100644 --- a/src/request.js +++ b/src/request.js @@ -6,7 +6,7 @@ */ var parse_url = require('url').parse; -var Headers = require('./headers'); +import Headers from './headers'; var Body = require('./body'); module.exports = Request; diff --git a/src/response.js b/src/response.js index f96aa85e8..b079bd071 100644 --- a/src/response.js +++ b/src/response.js @@ -6,7 +6,7 @@ */ var http = require('http'); -var Headers = require('./headers'); +import Headers from './headers'; var Body = require('./body'); module.exports = Response; diff --git a/test/test.js b/test/test.js index a7ce43e37..b59340712 100644 --- a/test/test.js +++ b/test/test.js @@ -17,7 +17,7 @@ var TestServer = require('./server'); // test subjects var fetch = require('../src/index.js'); -var Headers = require('../src/headers.js'); +import Headers from '../src/headers.js'; var Response = require('../src/response.js'); var Request = require('../src/request.js'); var Body = require('../src/body.js'); @@ -1131,6 +1131,65 @@ describe('node-fetch', function() { expect(result).to.deep.equal(expected); }); + it('should allow iterating through all headers', function() { + var headers = new Headers({ + a: 1 + , b: [2, 3] + , c: [4] + }); + expect(headers).to.have.property(Symbol.iterator); + expect(headers).to.have.property('keys'); + expect(headers).to.have.property('values'); + expect(headers).to.have.property('entries'); + + var result, expected; + + result = []; + for (let [key, val] of headers) { + result.push([key, val]); + } + + expected = [ + ["a", "1"] + , ["b", "2"] + , ["b", "3"] + , ["c", "4"] + ]; + expect(result).to.deep.equal(expected); + + result = []; + for (let [key, val] of headers.entries()) { + result.push([key, val]); + } + expect(result).to.deep.equal(expected); + + result = []; + for (let key of headers.keys()) { + result.push(key); + } + + expected = [ + "a" + , "b" + , "b" + , "c" + ]; + expect(result).to.deep.equal(expected); + + result = []; + for (let key of headers.values()) { + result.push(key); + } + + expected = [ + "1" + , "2" + , "3" + , "4" + ]; + expect(result).to.deep.equal(expected); + }); + it('should allow deleting header', function() { url = base + '/cookie'; return fetch(url).then(function(res) { @@ -1178,49 +1237,53 @@ describe('node-fetch', function() { res.m = new Buffer('test'); var h1 = new Headers(res); + var h1Raw = h1.raw(); - expect(h1._headers['a']).to.include('string'); - expect(h1._headers['b']).to.include('1'); - expect(h1._headers['b']).to.include('2'); - expect(h1._headers['c']).to.include(''); - expect(h1._headers['d']).to.be.undefined; + expect(h1Raw['a']).to.include('string'); + expect(h1Raw['b']).to.include('1'); + expect(h1Raw['b']).to.include('2'); + expect(h1Raw['c']).to.include(''); + expect(h1Raw['d']).to.be.undefined; - expect(h1._headers['e']).to.include('1'); - expect(h1._headers['f']).to.include('1'); - expect(h1._headers['f']).to.include('2'); + expect(h1Raw['e']).to.include('1'); + expect(h1Raw['f']).to.include('1'); + expect(h1Raw['f']).to.include('2'); - expect(h1._headers['g']).to.be.undefined; - expect(h1._headers['h']).to.be.undefined; - expect(h1._headers['i']).to.be.undefined; - expect(h1._headers['j']).to.be.undefined; - expect(h1._headers['k']).to.be.undefined; - expect(h1._headers['l']).to.be.undefined; - expect(h1._headers['m']).to.be.undefined; + expect(h1Raw['g']).to.be.undefined; + expect(h1Raw['h']).to.be.undefined; + expect(h1Raw['i']).to.be.undefined; + expect(h1Raw['j']).to.be.undefined; + expect(h1Raw['k']).to.be.undefined; + expect(h1Raw['l']).to.be.undefined; + expect(h1Raw['m']).to.be.undefined; - expect(h1._headers['z']).to.be.undefined; + expect(h1Raw['z']).to.be.undefined; }); it('should wrap headers', function() { var h1 = new Headers({ a: '1' }); + var h1Raw = h1.raw(); var h2 = new Headers(h1); h2.set('b', '1'); + var h2Raw = h2.raw(); var h3 = new Headers(h2); h3.append('a', '2'); + var h3Raw = h3.raw(); - expect(h1._headers['a']).to.include('1'); - expect(h1._headers['a']).to.not.include('2'); + expect(h1Raw['a']).to.include('1'); + expect(h1Raw['a']).to.not.include('2'); - expect(h2._headers['a']).to.include('1'); - expect(h2._headers['a']).to.not.include('2'); - expect(h2._headers['b']).to.include('1'); + expect(h2Raw['a']).to.include('1'); + expect(h2Raw['a']).to.not.include('2'); + expect(h2Raw['b']).to.include('1'); - expect(h3._headers['a']).to.include('1'); - expect(h3._headers['a']).to.include('2'); - expect(h3._headers['b']).to.include('1'); + expect(h3Raw['a']).to.include('1'); + expect(h3Raw['a']).to.include('2'); + expect(h3Raw['b']).to.include('1'); }); it('should support fetch with Request instance', function() { From 838071247d986a3f706392640ea299ec37f67073 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Mon, 10 Oct 2016 11:50:04 -0700 Subject: [PATCH 006/223] Convert all files to ES2015 (#182) Elements of this commit come from #140 by @gwicke. --- package.json | 5 +- src/body.js | 352 ++++++++++++----------- src/fetch-error.js | 4 +- src/index.js | 111 +++----- src/request.js | 108 +++---- src/response.js | 77 ++--- test/server.js | 566 ++++++++++++++++++------------------- test/test.js | 691 ++++++++++++++++++++++----------------------- 8 files changed, 948 insertions(+), 966 deletions(-) diff --git a/package.json b/package.json index 7014d4323..155270426 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "babel -d lib src", "prepublish": "npm run build", - "test": "mocha --compilers js:babel-register test/test.js", + "test": "mocha --compilers js:babel-polyfill --compilers js:babel-register test/test.js", "report": "cross-env BABEL_ENV=test nyc --reporter lcov --reporter text mocha -R spec test/test.js", "coverage": "cross-env BABEL_ENV=test nyc --reporter lcovonly mocha -R spec test/test.js && cat ./coverage/lcov.info | coveralls" }, @@ -29,6 +29,7 @@ "babel-cli": "^6.16.0", "babel-plugin-istanbul": "^2.0.1", "babel-plugin-transform-runtime": "^6.15.0", + "babel-polyfill": "^6.16.0", "babel-preset-es2015": "^6.16.0", "babel-register": "^6.16.3", "bluebird": "^3.3.4", @@ -44,6 +45,7 @@ "resumer": "0.0.0" }, "dependencies": { + "babel-runtime": "^6.11.6", "encoding": "^0.1.11", "is-stream": "^1.0.1" }, @@ -67,6 +69,7 @@ "src/*.js" ], "require": [ + "babel-polyfill", "babel-register" ], "sourceMap": false, diff --git a/src/body.js b/src/body.js index e7bbe1dac..ab76421dd 100644 --- a/src/body.js +++ b/src/body.js @@ -5,12 +5,17 @@ * Body interface provides common methods for Request and Response */ -var convert = require('encoding').convert; -var bodyStream = require('is-stream'); -var PassThrough = require('stream').PassThrough; -var FetchError = require('./fetch-error'); - -module.exports = Body; +import {convert} from 'encoding'; +import bodyStream from 'is-stream'; +import {PassThrough} from 'stream'; +import FetchError from './fetch-error.js'; + +const DISTURBED = Symbol('disturbed'); +const BYTES = Symbol('bytes'); +const RAW = Symbol('raw'); +const ABORT = Symbol('abort'); +const CONVERT = Symbol('convert'); +const DECODE = Symbol('decode'); /** * Body class @@ -19,211 +24,200 @@ module.exports = Body; * @param Object opts Response options * @return Void */ -function Body(body, opts) { - - opts = opts || {}; - - this.body = body; - this.bodyUsed = false; - this.size = opts.size || 0; - this.timeout = opts.timeout || 0; - this._raw = []; - this._abort = false; - -} - -/** - * Decode response as json - * - * @return Promise - */ -Body.prototype.json = function() { - - // for 204 No Content response, buffer will be empty, parsing it will throw error - if (this.status === 204) { - return Body.Promise.resolve({}); +export default class Body { + constructor(body, { + size = 0, + timeout = 0 + } = {}) { + this.body = body; + this[DISTURBED] = false; + this.size = size; + this[BYTES] = 0; + this.timeout = timeout; + this[RAW] = []; + this[ABORT] = false; } - return this._decode().then(function(buffer) { - return JSON.parse(buffer.toString()); - }); - -}; - -/** - * Decode response as text - * - * @return Promise - */ -Body.prototype.text = function() { - - return this._decode().then(function(buffer) { - return buffer.toString(); - }); - -}; - -/** - * Decode response as buffer (non-spec api) - * - * @return Promise - */ -Body.prototype.buffer = function() { - - return this._decode(); - -}; - -/** - * Decode buffers into utf-8 string - * - * @return Promise - */ -Body.prototype._decode = function() { + get bodyUsed() { + return this[DISTURBED]; + } - var self = this; + /** + * Decode response as json + * + * @return Promise + */ + json() { + // for 204 No Content response, buffer will be empty, parsing it will throw error + if (this.status === 204) { + return Body.Promise.resolve({}); + } - if (this.bodyUsed) { - return Body.Promise.reject(new Error('body used already for: ' + this.url)); + return this[DECODE]().then(buffer => JSON.parse(buffer.toString())); } - this.bodyUsed = true; - this._bytes = 0; - this._abort = false; - this._raw = []; - - return new Body.Promise(function(resolve, reject) { - var resTimeout; + /** + * Decode response as text + * + * @return Promise + */ + text() { + return this[DECODE]().then(buffer => buffer.toString()); + } - // body is string - if (typeof self.body === 'string') { - self._bytes = self.body.length; - self._raw = [new Buffer(self.body)]; - return resolve(self._convert()); - } + /** + * Decode response as buffer (non-spec api) + * + * @return Promise + */ + buffer() { + return this[DECODE](); + } - // body is buffer - if (self.body instanceof Buffer) { - self._bytes = self.body.length; - self._raw = [self.body]; - return resolve(self._convert()); + /** + * Decode buffers into utf-8 string + * + * @return Promise + */ + [DECODE]() { + if (this[DISTURBED]) { + return Body.Promise.reject(new Error(`body used already for: ${this.url}`)); } - // allow timeout on slow response body - if (self.timeout) { - resTimeout = setTimeout(function() { - self._abort = true; - reject(new FetchError('response timeout at ' + self.url + ' over limit: ' + self.timeout, 'body-timeout')); - }, self.timeout); - } + this[DISTURBED] = true; + this[BYTES] = 0; + this[ABORT] = false; + this[RAW] = []; - // handle stream error, such as incorrect content-encoding - self.body.on('error', function(err) { - reject(new FetchError('invalid response body at: ' + self.url + ' reason: ' + err.message, 'system', err)); - }); + return new Body.Promise((resolve, reject) => { + let resTimeout; - // body is stream - self.body.on('data', function(chunk) { - if (self._abort || chunk === null) { - return; + // body is string + if (typeof this.body === 'string') { + this[BYTES] = this.body.length; + this[RAW] = [new Buffer(this.body)]; + return resolve(this[CONVERT]()); } - if (self.size && self._bytes + chunk.length > self.size) { - self._abort = true; - reject(new FetchError('content size at ' + self.url + ' over limit: ' + self.size, 'max-size')); - return; + // body is buffer + if (this.body instanceof Buffer) { + this[BYTES] = this.body.length; + this[RAW] = [this.body]; + return resolve(this[CONVERT]()); } - self._bytes += chunk.length; - self._raw.push(chunk); - }); - - self.body.on('end', function() { - if (self._abort) { - return; + // allow timeout on slow response body + if (this.timeout) { + resTimeout = setTimeout(() => { + this[ABORT] = true; + reject(new FetchError('response timeout at ' + this.url + ' over limit: ' + this.timeout, 'body-timeout')); + }, this.timeout); } - clearTimeout(resTimeout); - resolve(self._convert()); + // handle stream error, such as incorrect content-encoding + this.body.on('error', err => { + reject(new FetchError('invalid response body at: ' + this.url + ' reason: ' + err.message, 'system', err)); + }); + + // body is stream + this.body.on('data', chunk => { + if (this[ABORT] || chunk === null) { + return; + } + + if (this.size && this[BYTES] + chunk.length > this.size) { + this[ABORT] = true; + reject(new FetchError(`content size at ${this.url} over limit: ${this.size}`, 'max-size')); + return; + } + + this[BYTES] += chunk.length; + this[RAW].push(chunk); + }); + + this.body.on('end', () => { + if (this[ABORT]) { + return; + } + + clearTimeout(resTimeout); + resolve(this[CONVERT]()); + }); }); - }); - -}; - -/** - * Detect buffer encoding and convert to target encoding - * ref: http://www.w3.org/TR/2011/WD-html5-20110113/parsing.html#determining-the-character-encoding - * - * @param String encoding Target encoding - * @return String - */ -Body.prototype._convert = function(encoding) { - - encoding = encoding || 'utf-8'; + } - var ct = this.headers.get('content-type'); - var charset = 'utf-8'; - var res, str; + /** + * Detect buffer encoding and convert to target encoding + * ref: http://www.w3.org/TR/2011/WD-html5-20110113/parsing.html#determining-the-character-encoding + * + * @param String encoding Target encoding + * @return String + */ + [CONVERT](encoding = 'utf-8') { + const ct = this.headers.get('content-type'); + let charset = 'utf-8'; + let res, str; + + // header + if (ct) { + // skip encoding detection altogether if not html/xml/plain text + if (!/text\/html|text\/plain|\+xml|\/xml/i.test(ct)) { + return Buffer.concat(this[RAW]); + } - // header - if (ct) { - // skip encoding detection altogether if not html/xml/plain text - if (!/text\/html|text\/plain|\+xml|\/xml/i.test(ct)) { - return Buffer.concat(this._raw); + res = /charset=([^;]*)/i.exec(ct); } - res = /charset=([^;]*)/i.exec(ct); - } - - // no charset in content type, peek at response body for at most 1024 bytes - if (!res && this._raw.length > 0) { - for (var i = 0; i < this._raw.length; i++) { - str += this._raw[i].toString() - if (str.length > 1024) { - break; + // no charset in content type, peek at response body for at most 1024 bytes + if (!res && this[RAW].length > 0) { + for (let i = 0; i < this[RAW].length; i++) { + str += this[RAW][i].toString() + if (str.length > 1024) { + break; + } } + str = str.substr(0, 1024); } - str = str.substr(0, 1024); - } - // html5 - if (!res && str) { - res = / { // build request object - var options = new Request(url, opts); + const options = new Request(url, opts); if (!options.protocol || !options.hostname) { throw new Error('only absolute urls are supported'); @@ -58,15 +46,10 @@ function Fetch(url, opts) { throw new Error('only http(s) protocols are supported'); } - var send; - if (options.protocol === 'https:') { - send = https.request; - } else { - send = http.request; - } + const send = (options.protocol === 'https:' ? https : http).request; // normalize headers - var headers = new Headers(options.headers); + const headers = new Headers(options.headers); if (options.compress) { headers.set('accept-encoding', 'gzip,deflate'); @@ -86,7 +69,7 @@ function Fetch(url, opts) { // detect form data input from form-data module, this hack avoid the need to pass multipart header manually if (!headers.has('content-type') && options.body && typeof options.body.getBoundary === 'function') { - headers.set('content-type', 'multipart/form-data; boundary=' + options.body.getBoundary()); + headers.set('content-type', `multipart/form-data; boundary=${options.body.getBoundary()}`); } // bring node-fetch closer to browser behavior by setting content-length automatically @@ -116,40 +99,40 @@ function Fetch(url, opts) { } // send request - var req = send(options); - var reqTimeout; + const req = send(options); + let reqTimeout; if (options.timeout) { - req.once('socket', function(socket) { - reqTimeout = setTimeout(function() { + req.once('socket', socket => { + reqTimeout = setTimeout(() => { req.abort(); - reject(new FetchError('network timeout at: ' + options.url, 'request-timeout')); + reject(new FetchError(`network timeout at: ${options.url}`, 'request-timeout')); }, options.timeout); }); } - req.on('error', function(err) { + req.on('error', err => { clearTimeout(reqTimeout); - reject(new FetchError('request to ' + options.url + ' failed, reason: ' + err.message, 'system', err)); + reject(new FetchError(`request to ${options.url} failed, reason: ${err.message}`, 'system', err)); }); - req.on('response', function(res) { + req.on('response', res => { clearTimeout(reqTimeout); // handle redirect - if (self.isRedirect(res.statusCode) && options.redirect !== 'manual') { + if (fetch.isRedirect(res.statusCode) && options.redirect !== 'manual') { if (options.redirect === 'error') { - reject(new FetchError('redirect mode is set to error: ' + options.url, 'no-redirect')); + reject(new FetchError(`redirect mode is set to error: ${options.url}`, 'no-redirect')); return; } if (options.counter >= options.follow) { - reject(new FetchError('maximum redirect reached at: ' + options.url, 'max-redirect')); + reject(new FetchError(`maximum redirect reached at: ${options.url}`, 'max-redirect')); return; } if (!res.headers.location) { - reject(new FetchError('redirect location header missing at: ' + options.url, 'invalid-redirect')); + reject(new FetchError(`redirect location header missing at: ${options.url}`, 'invalid-redirect')); return; } @@ -164,19 +147,19 @@ function Fetch(url, opts) { options.counter++; - resolve(Fetch(resolve_url(options.url, res.headers.location), options)); + resolve(fetch(resolve_url(options.url, res.headers.location), options)); return; } // normalize location header for manual redirect mode - var headers = new Headers(res.headers); + const headers = new Headers(res.headers); if (options.redirect === 'manual' && headers.has('location')) { headers.set('location', resolve_url(options.url, headers.get('location'))); } // prepare response - var body = res.pipe(new stream.PassThrough()); - var response_options = { + let body = res.pipe(new PassThrough()); + const response_options = { url: options.url , status: res.statusCode , statusText: res.statusMessage @@ -186,7 +169,7 @@ function Fetch(url, opts) { }; // response object - var output; + let output; // in following scenarios we ignore compression support // 1. compression support is disabled @@ -201,7 +184,7 @@ function Fetch(url, opts) { } // otherwise, check for gzip or deflate - var name = headers.get('content-encoding'); + let name = headers.get('content-encoding'); // for gzip if (name == 'gzip' || name == 'x-gzip') { @@ -214,8 +197,8 @@ function Fetch(url, opts) { } else if (name == 'deflate' || name == 'x-deflate') { // handle the infamous raw deflate response from old servers // a hack for old IIS and Apache servers - var raw = res.pipe(new stream.PassThrough()); - raw.once('data', function(chunk) { + const raw = res.pipe(new PassThrough()); + raw.once('data', chunk => { // see http://stackoverflow.com/questions/37519828 if ((chunk[0] & 0x0F) === 0x08) { body = body.pipe(zlib.createInflate()); @@ -254,18 +237,18 @@ function Fetch(url, opts) { }; +module.exports = fetch; + /** * Redirect code matching * * @param Number code Status code * @return Boolean */ -Fetch.prototype.isRedirect = function(code) { - return code === 301 || code === 302 || code === 303 || code === 307 || code === 308; -} +fetch.isRedirect = code => code === 301 || code === 302 || code === 303 || code === 307 || code === 308; // expose Promise -Fetch.Promise = global.Promise; -Fetch.Response = Response; -Fetch.Headers = Headers; -Fetch.Request = Request; +fetch.Promise = global.Promise; +fetch.Response = Response; +fetch.Headers = Headers; +fetch.Request = Request; diff --git a/src/request.js b/src/request.js index ec80635af..1508d67f9 100644 --- a/src/request.js +++ b/src/request.js @@ -5,11 +5,9 @@ * Request class contains server only options */ -var parse_url = require('url').parse; -import Headers from './headers'; -var Body = require('./body'); - -module.exports = Request; +import { parse as parse_url } from 'url'; +import Headers from './headers.js'; +import Body, { clone } from './body'; /** * Request class @@ -18,58 +16,62 @@ module.exports = Request; * @param Object init Custom options * @return Void */ -function Request(input, init) { - var url, url_parsed; - - // normalize input - if (!(input instanceof Request)) { - url = input; - url_parsed = parse_url(url); - input = {}; - } else { - url = input.url; - url_parsed = parse_url(url); - } +export default class Request extends Body { + constructor(input, init = {}) { + let url, url_parsed; - // normalize init - init = init || {}; + // normalize input + if (!(input instanceof Request)) { + url = input; + url_parsed = parse_url(url); + input = {}; + } else { + url = input.url; + url_parsed = parse_url(url); + } - // fetch spec options - this.method = init.method || input.method || 'GET'; - this.redirect = init.redirect || input.redirect || 'follow'; - this.headers = new Headers(init.headers || input.headers || {}); - this.url = url; + super(init.body || clone(input), { + timeout: init.timeout || input.timeout || 0, + size: init.size || input.size || 0 + }); - // server only options - this.follow = init.follow !== undefined ? - init.follow : input.follow !== undefined ? - input.follow : 20; - this.compress = init.compress !== undefined ? - init.compress : input.compress !== undefined ? - input.compress : true; - this.counter = init.counter || input.counter || 0; - this.agent = init.agent || input.agent; + // fetch spec options + this.method = init.method || input.method || 'GET'; + this.redirect = init.redirect || input.redirect || 'follow'; + this.headers = new Headers(init.headers || input.headers || {}); + this.url = url; - Body.call(this, init.body || this._clone(input), { - timeout: init.timeout || input.timeout || 0, - size: init.size || input.size || 0 - }); + // server only options + this.follow = init.follow !== undefined ? + init.follow : input.follow !== undefined ? + input.follow : 20; + this.compress = init.compress !== undefined ? + init.compress : input.compress !== undefined ? + input.compress : true; + this.counter = init.counter || input.counter || 0; + this.agent = init.agent || input.agent; - // server request options - this.protocol = url_parsed.protocol; - this.hostname = url_parsed.hostname; - this.port = url_parsed.port; - this.path = url_parsed.path; - this.auth = url_parsed.auth; -} + // server request options + this.protocol = url_parsed.protocol; + this.hostname = url_parsed.hostname; + this.port = url_parsed.port; + this.path = url_parsed.path; + this.auth = url_parsed.auth; + } -Request.prototype = Object.create(Body.prototype); + /** + * Clone this request + * + * @return Request + */ + clone() { + return new Request(this); + } -/** - * Clone this request - * - * @return Request - */ -Request.prototype.clone = function() { - return new Request(this); -}; + /** + * Tag used by `Object.prototype.toString()`. + */ + get [Symbol.toStringTag]() { + return 'Request'; + } +} diff --git a/src/response.js b/src/response.js index b079bd071..fb53a783c 100644 --- a/src/response.js +++ b/src/response.js @@ -5,11 +5,9 @@ * Response class provides content decoding */ -var http = require('http'); -import Headers from './headers'; -var Body = require('./body'); - -module.exports = Response; +import { STATUS_CODES } from 'http'; +import Headers from './headers.js'; +import Body, { clone } from './body'; /** * Response class @@ -18,33 +16,44 @@ module.exports = Response; * @param Object opts Response options * @return Void */ -function Response(body, opts) { - - opts = opts || {}; - - this.url = opts.url; - this.status = opts.status || 200; - this.statusText = opts.statusText || http.STATUS_CODES[this.status]; - this.headers = new Headers(opts.headers); - this.ok = this.status >= 200 && this.status < 300; - - Body.call(this, body, opts); - +export default class Response extends Body { + constructor(body, opts = {}) { + super(body, opts); + + this.url = opts.url; + this.status = opts.status || 200; + this.statusText = opts.statusText || STATUS_CODES[this.status]; + this.headers = new Headers(opts.headers); + } + + /** + * Convenience property representing if the request ended normally + */ + get ok() { + return this.status >= 200 && this.status < 300; + } + + /** + * Clone this response + * + * @return Response + */ + clone() { + + return new Response(clone(this), { + url: this.url + , status: this.status + , statusText: this.statusText + , headers: this.headers + , ok: this.ok + }); + + } + + /** + * Tag used by `Object.prototype.toString()`. + */ + get [Symbol.toStringTag]() { + return 'Response'; + } } - -Response.prototype = Object.create(Body.prototype); - -/** - * Clone this response - * - * @return Response - */ -Response.prototype.clone = function() { - return new Response(this._clone(this), { - url: this.url - , status: this.status - , statusText: this.statusText - , headers: this.headers - , ok: this.ok - }); -}; diff --git a/test/server.js b/test/server.js index 08e582d3b..5a75b2973 100644 --- a/test/server.js +++ b/test/server.js @@ -1,337 +1,331 @@ - -var http = require('http'); -var parse = require('url').parse; -var zlib = require('zlib'); -var stream = require('stream'); -var convert = require('encoding').convert; -var Multipart = require('parted').multipart; - -module.exports = TestServer; - -function TestServer() { - this.server = http.createServer(this.router); - this.port = 30001; - this.hostname = 'localhost'; - this.server.on('error', function(err) { - console.log(err.stack); - }); - this.server.on('connection', function(socket) { - socket.setTimeout(1500); - }); -} - -TestServer.prototype.start = function(cb) { - this.server.listen(this.port, this.hostname, cb); -} - -TestServer.prototype.stop = function(cb) { - this.server.close(cb); -} - -TestServer.prototype.router = function(req, res) { - - var p = parse(req.url).pathname; - - if (p === '/hello') { - res.statusCode = 200; - res.setHeader('Content-Type', 'text/plain'); - res.end('world'); +import 'babel-polyfill'; +import * as http from 'http'; +import { parse } from 'url'; +import * as zlib from 'zlib'; +import * as stream from 'stream'; +import { convert } from 'encoding'; +import { multipart as Multipart } from 'parted'; + +export default class TestServer { + constructor() { + this.server = http.createServer(this.router); + this.port = 30001; + this.hostname = 'localhost'; + this.server.on('error', function(err) { + console.log(err.stack); + }); + this.server.on('connection', function(socket) { + socket.setTimeout(1500); + }); } - if (p === '/plain') { - res.statusCode = 200; - res.setHeader('Content-Type', 'text/plain'); - res.end('text'); + start(cb) { + this.server.listen(this.port, this.hostname, cb); } - if (p === '/options') { - res.statusCode = 200; - res.setHeader('Allow', 'GET, HEAD, OPTIONS'); - res.end('hello world'); + stop(cb) { + this.server.close(cb); } - if (p === '/html') { - res.statusCode = 200; - res.setHeader('Content-Type', 'text/html'); - res.end(''); - } + router(req, res) { + let p = parse(req.url).pathname; - if (p === '/json') { - res.statusCode = 200; - res.setHeader('Content-Type', 'application/json'); - res.end(JSON.stringify({ - name: 'value' - })); - } + if (p === '/hello') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/plain'); + res.end('world'); + } - if (p === '/gzip') { - res.statusCode = 200; - res.setHeader('Content-Type', 'text/plain'); - res.setHeader('Content-Encoding', 'gzip'); - zlib.gzip('hello world', function(err, buffer) { - res.end(buffer); - }); - } + if (p === '/plain') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/plain'); + res.end('text'); + } - if (p === '/deflate') { - res.statusCode = 200; - res.setHeader('Content-Type', 'text/plain'); - res.setHeader('Content-Encoding', 'deflate'); - zlib.deflate('hello world', function(err, buffer) { - res.end(buffer); - }); - } + if (p === '/options') { + res.statusCode = 200; + res.setHeader('Allow', 'GET, HEAD, OPTIONS'); + res.end('hello world'); + } - if (p === '/deflate-raw') { - res.statusCode = 200; - res.setHeader('Content-Type', 'text/plain'); - res.setHeader('Content-Encoding', 'deflate'); - zlib.deflateRaw('hello world', function(err, buffer) { - res.end(buffer); - }); - } + if (p === '/html') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/html'); + res.end(''); + } - if (p === '/sdch') { - res.statusCode = 200; - res.setHeader('Content-Type', 'text/plain'); - res.setHeader('Content-Encoding', 'sdch'); - res.end('fake sdch string'); - } + if (p === '/json') { + res.statusCode = 200; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ + name: 'value' + })); + } - if (p === '/invalid-content-encoding') { - res.statusCode = 200; - res.setHeader('Content-Type', 'text/plain'); - res.setHeader('Content-Encoding', 'gzip'); - res.end('fake gzip string'); - } + if (p === '/gzip') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/plain'); + res.setHeader('Content-Encoding', 'gzip'); + zlib.gzip('hello world', function(err, buffer) { + res.end(buffer); + }); + } - if (p === '/timeout') { - setTimeout(function() { + if (p === '/deflate') { res.statusCode = 200; res.setHeader('Content-Type', 'text/plain'); - res.end('text'); - }, 1000); - } + res.setHeader('Content-Encoding', 'deflate'); + zlib.deflate('hello world', function(err, buffer) { + res.end(buffer); + }); + } - if (p === '/slow') { - res.statusCode = 200; - res.setHeader('Content-Type', 'text/plain'); - res.write('test'); - setTimeout(function() { - res.end('test'); - }, 1000); - } + if (p === '/deflate-raw') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/plain'); + res.setHeader('Content-Encoding', 'deflate'); + zlib.deflateRaw('hello world', function(err, buffer) { + res.end(buffer); + }); + } - if (p === '/cookie') { - res.statusCode = 200; - res.setHeader('Set-Cookie', ['a=1', 'b=1']); - res.end('cookie'); - } + if (p === '/sdch') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/plain'); + res.setHeader('Content-Encoding', 'sdch'); + res.end('fake sdch string'); + } + + if (p === '/invalid-content-encoding') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/plain'); + res.setHeader('Content-Encoding', 'gzip'); + res.end('fake gzip string'); + } - if (p === '/size/chunk') { - res.statusCode = 200; - res.setHeader('Content-Type', 'text/plain'); - setTimeout(function() { + if (p === '/timeout') { + setTimeout(function() { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/plain'); + res.end('text'); + }, 1000); + } + + if (p === '/slow') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/plain'); res.write('test'); - }, 50); - setTimeout(function() { - res.end('test'); - }, 100); - } + setTimeout(function() { + res.end('test'); + }, 1000); + } - if (p === '/size/long') { - res.statusCode = 200; - res.setHeader('Content-Type', 'text/plain'); - res.end('testtest'); - } + if (p === '/cookie') { + res.statusCode = 200; + res.setHeader('Set-Cookie', ['a=1', 'b=1']); + res.end('cookie'); + } - if (p === '/encoding/gbk') { - res.statusCode = 200; - res.setHeader('Content-Type', 'text/html'); - res.end(convert('
中文
', 'gbk')); - } + if (p === '/size/chunk') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/plain'); + setTimeout(function() { + res.write('test'); + }, 50); + setTimeout(function() { + res.end('test'); + }, 100); + } - if (p === '/encoding/gb2312') { - res.statusCode = 200; - res.setHeader('Content-Type', 'text/html'); - res.end(convert('
中文
', 'gb2312')); - } + if (p === '/size/long') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/plain'); + res.end('testtest'); + } - if (p === '/encoding/shift-jis') { - res.statusCode = 200; - res.setHeader('Content-Type', 'text/html; charset=Shift-JIS'); - res.end(convert('
日本語
', 'Shift_JIS')); - } + if (p === '/encoding/gbk') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/html'); + res.end(convert('
中文
', 'gbk')); + } - if (p === '/encoding/euc-jp') { - res.statusCode = 200; - res.setHeader('Content-Type', 'text/xml'); - res.end(convert('日本語', 'EUC-JP')); - } + if (p === '/encoding/gb2312') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/html'); + res.end(convert('
中文
', 'gb2312')); + } - if (p === '/encoding/utf8') { - res.statusCode = 200; - res.end('中文'); - } + if (p === '/encoding/shift-jis') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/html; charset=Shift-JIS'); + res.end(convert('
日本語
', 'Shift_JIS')); + } - if (p === '/encoding/order1') { - res.statusCode = 200; - res.setHeader('Content-Type', 'charset=gbk; text/plain'); - res.end(convert('中文', 'gbk')); - } + if (p === '/encoding/euc-jp') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/xml'); + res.end(convert('日本語', 'EUC-JP')); + } - if (p === '/encoding/order2') { - res.statusCode = 200; - res.setHeader('Content-Type', 'text/plain; charset=gbk; qs=1'); - res.end(convert('中文', 'gbk')); - } + if (p === '/encoding/utf8') { + res.statusCode = 200; + res.end('中文'); + } - if (p === '/encoding/chunked') { - res.statusCode = 200; - res.setHeader('Content-Type', 'text/html'); - res.setHeader('Transfer-Encoding', 'chunked'); - var padding = 'a'; - for (var i = 0; i < 10; i++) { - res.write(padding); + if (p === '/encoding/order1') { + res.statusCode = 200; + res.setHeader('Content-Type', 'charset=gbk; text/plain'); + res.end(convert('中文', 'gbk')); } - res.end(convert('
日本語
', 'Shift_JIS')); - } - if (p === '/encoding/invalid') { - res.statusCode = 200; - res.setHeader('Content-Type', 'text/html'); - res.setHeader('Transfer-Encoding', 'chunked'); - // because node v0.12 doesn't have str.repeat - var padding = new Array(120 + 1).join('a'); - for (var i = 0; i < 10; i++) { - res.write(padding); + if (p === '/encoding/order2') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/plain; charset=gbk; qs=1'); + res.end(convert('中文', 'gbk')); } - res.end(convert('中文', 'gbk')); - } - if (p === '/redirect/301') { - res.statusCode = 301; - res.setHeader('Location', '/inspect'); - res.end(); - } + if (p === '/encoding/chunked') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/html'); + res.setHeader('Transfer-Encoding', 'chunked'); + const padding = 'a'; + res.write(padding.repeat(10)); + res.end(convert('
日本語
', 'Shift_JIS')); + } - if (p === '/redirect/302') { - res.statusCode = 302; - res.setHeader('Location', '/inspect'); - res.end(); - } + if (p === '/encoding/invalid') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/html'); + res.setHeader('Transfer-Encoding', 'chunked'); + const padding = 'a'.repeat(120); + res.write(padding.repeat(10)); + res.end(convert('中文', 'gbk')); + } - if (p === '/redirect/303') { - res.statusCode = 303; - res.setHeader('Location', '/inspect'); - res.end(); - } + if (p === '/redirect/301') { + res.statusCode = 301; + res.setHeader('Location', '/inspect'); + res.end(); + } - if (p === '/redirect/307') { - res.statusCode = 307; - res.setHeader('Location', '/inspect'); - res.end(); - } + if (p === '/redirect/302') { + res.statusCode = 302; + res.setHeader('Location', '/inspect'); + res.end(); + } - if (p === '/redirect/308') { - res.statusCode = 308; - res.setHeader('Location', '/inspect'); - res.end(); - } + if (p === '/redirect/303') { + res.statusCode = 303; + res.setHeader('Location', '/inspect'); + res.end(); + } - if (p === '/redirect/chain') { - res.statusCode = 301; - res.setHeader('Location', '/redirect/301'); - res.end(); - } + if (p === '/redirect/307') { + res.statusCode = 307; + res.setHeader('Location', '/inspect'); + res.end(); + } - if (p === '/error/redirect') { - res.statusCode = 301; - //res.setHeader('Location', '/inspect'); - res.end(); - } + if (p === '/redirect/308') { + res.statusCode = 308; + res.setHeader('Location', '/inspect'); + res.end(); + } - if (p === '/error/400') { - res.statusCode = 400; - res.setHeader('Content-Type', 'text/plain'); - res.end('client error'); - } + if (p === '/redirect/chain') { + res.statusCode = 301; + res.setHeader('Location', '/redirect/301'); + res.end(); + } - if (p === '/error/404') { - res.statusCode = 404; - res.setHeader('Content-Encoding', 'gzip'); - res.end(); - } + if (p === '/error/redirect') { + res.statusCode = 301; + //res.setHeader('Location', '/inspect'); + res.end(); + } - if (p === '/error/500') { - res.statusCode = 500; - res.setHeader('Content-Type', 'text/plain'); - res.end('server error'); - } + if (p === '/error/400') { + res.statusCode = 400; + res.setHeader('Content-Type', 'text/plain'); + res.end('client error'); + } - if (p === '/error/reset') { - res.destroy(); - } + if (p === '/error/404') { + res.statusCode = 404; + res.setHeader('Content-Encoding', 'gzip'); + res.end(); + } - if (p === '/error/json') { - res.statusCode = 200; - res.setHeader('Content-Type', 'application/json'); - res.end('invalid json'); - } + if (p === '/error/500') { + res.statusCode = 500; + res.setHeader('Content-Type', 'text/plain'); + res.end('server error'); + } - if (p === '/no-content') { - res.statusCode = 204; - res.end(); - } + if (p === '/error/reset') { + res.destroy(); + } - if (p === '/no-content/gzip') { - res.statusCode = 204; - res.setHeader('Content-Encoding', 'gzip'); - res.end(); - } + if (p === '/error/json') { + res.statusCode = 200; + res.setHeader('Content-Type', 'application/json'); + res.end('invalid json'); + } - if (p === '/not-modified') { - res.statusCode = 304; - res.end(); - } + if (p === '/no-content') { + res.statusCode = 204; + res.end(); + } - if (p === '/not-modified/gzip') { - res.statusCode = 304; - res.setHeader('Content-Encoding', 'gzip'); - res.end(); - } + if (p === '/no-content/gzip') { + res.statusCode = 204; + res.setHeader('Content-Encoding', 'gzip'); + res.end(); + } - if (p === '/inspect') { - res.statusCode = 200; - res.setHeader('Content-Type', 'application/json'); - var body = ''; - req.on('data', function(c) { body += c }); - req.on('end', function() { - res.end(JSON.stringify({ - method: req.method, - url: req.url, - headers: req.headers, - body: body - })); - }); - } + if (p === '/not-modified') { + res.statusCode = 304; + res.end(); + } - if (p === '/multipart') { - res.statusCode = 200; - res.setHeader('Content-Type', 'application/json'); - var parser = new Multipart(req.headers['content-type']); - var body = ''; - parser.on('part', function(field, part) { - body += field + '=' + part; - }); - parser.on('end', function() { - res.end(JSON.stringify({ - method: req.method, - url: req.url, - headers: req.headers, - body: body - })); - }); - req.pipe(parser); + if (p === '/not-modified/gzip') { + res.statusCode = 304; + res.setHeader('Content-Encoding', 'gzip'); + res.end(); + } + + if (p === '/inspect') { + res.statusCode = 200; + res.setHeader('Content-Type', 'application/json'); + let body = ''; + req.on('data', function(c) { body += c }); + req.on('end', function() { + res.end(JSON.stringify({ + method: req.method, + url: req.url, + headers: req.headers, + body + })); + }); + } + + if (p === '/multipart') { + res.statusCode = 200; + res.setHeader('Content-Type', 'application/json'); + const parser = new Multipart(req.headers['content-type']); + let body = ''; + parser.on('part', function(field, part) { + body += field + '=' + part; + }); + parser.on('end', function() { + res.end(JSON.stringify({ + method: req.method, + url: req.url, + headers: req.headers, + body: body + })); + }); + req.pipe(parser); + } } } diff --git a/test/test.js b/test/test.js index b59340712..a1ea39a31 100644 --- a/test/test.js +++ b/test/test.js @@ -1,54 +1,55 @@ // test tools -var chai = require('chai'); -var cap = require('chai-as-promised'); +import chai from 'chai'; +import cap from 'chai-as-promised'; +import bluebird from 'bluebird'; +import then from 'promise'; +import {spawn} from 'child_process'; +import * as stream from 'stream'; +import resumer from 'resumer'; +import FormData from 'form-data'; +import * as http from 'http'; +import * as fs from 'fs'; + chai.use(cap); -var expect = chai.expect; -var bluebird = require('bluebird'); -var then = require('promise'); -var spawn = require('child_process').spawn; -var stream = require('stream'); -var resumer = require('resumer'); -var FormData = require('form-data'); -var http = require('http'); -var fs = require('fs'); - -var TestServer = require('./server'); +const expect = chai.expect; + +import TestServer from './server'; // test subjects -var fetch = require('../src/index.js'); +import fetch from '../src/index.js'; import Headers from '../src/headers.js'; -var Response = require('../src/response.js'); -var Request = require('../src/request.js'); -var Body = require('../src/body.js'); -var FetchError = require('../src/fetch-error.js'); +import Response from '../src/response.js'; +import Request from '../src/request.js'; +import Body from '../src/body.js'; +import FetchError from '../src/fetch-error.js'; // test with native promise on node 0.11, and bluebird for node 0.10 fetch.Promise = fetch.Promise || bluebird; -var url, opts, local, base; +let url, opts, local, base; -describe('node-fetch', function() { +describe('node-fetch', () => { - before(function(done) { + before(done => { local = new TestServer(); base = 'http://' + local.hostname + ':' + local.port; local.start(done); }); - after(function(done) { + after(done => { local.stop(done); }); it('should return a promise', function() { url = 'http://example.com/'; - var p = fetch(url); + const p = fetch(url); expect(p).to.be.an.instanceof(fetch.Promise); expect(p).to.have.property('then'); }); it('should allow custom promise', function() { url = 'http://example.com/'; - var old = fetch.Promise; + const old = fetch.Promise; fetch.Promise = then; expect(fetch(url)).to.be.an.instanceof(then); expect(fetch(url)).to.not.be.an.instanceof(bluebird); @@ -57,9 +58,9 @@ describe('node-fetch', function() { it('should throw error when no promise implementation are found', function() { url = 'http://example.com/'; - var old = fetch.Promise; + const old = fetch.Promise; fetch.Promise = undefined; - expect(function() { + expect(() => { fetch(url) }).to.throw(Error); fetch.Promise = old; @@ -94,8 +95,8 @@ describe('node-fetch', function() { }); it('should resolve into response', function() { - url = base + '/hello'; - return fetch(url).then(function(res) { + url = `${base}/hello`; + return fetch(url).then(res => { expect(res).to.be.an.instanceof(Response); expect(res.headers).to.be.an.instanceof(Headers); expect(res.body).to.be.an.instanceof(stream.Transform); @@ -109,10 +110,10 @@ describe('node-fetch', function() { }); it('should accept plain text response', function() { - url = base + '/plain'; - return fetch(url).then(function(res) { + url = `${base}/plain`; + return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); - return res.text().then(function(result) { + return res.text().then(result => { expect(res.bodyUsed).to.be.true; expect(result).to.be.a('string'); expect(result).to.equal('text'); @@ -121,10 +122,10 @@ describe('node-fetch', function() { }); it('should accept html response (like plain text)', function() { - url = base + '/html'; - return fetch(url).then(function(res) { + url = `${base}/html`; + return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/html'); - return res.text().then(function(result) { + return res.text().then(result => { expect(res.bodyUsed).to.be.true; expect(result).to.be.a('string'); expect(result).to.equal(''); @@ -133,10 +134,10 @@ describe('node-fetch', function() { }); it('should accept json response', function() { - url = base + '/json'; - return fetch(url).then(function(res) { + url = `${base}/json`; + return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('application/json'); - return res.json().then(function(result) { + return res.json().then(result => { expect(res.bodyUsed).to.be.true; expect(result).to.be.an('object'); expect(result).to.deep.equal({ name: 'value' }); @@ -145,102 +146,102 @@ describe('node-fetch', function() { }); it('should send request with custom headers', function() { - url = base + '/inspect'; + url = `${base}/inspect`; opts = { headers: { 'x-custom-header': 'abc' } }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { return res.json(); - }).then(function(res) { + }).then(res => { expect(res.headers['x-custom-header']).to.equal('abc'); }); }); it('should accept headers instance', function() { - url = base + '/inspect'; + url = `${base}/inspect`; opts = { headers: new Headers({ 'x-custom-header': 'abc' }) }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { return res.json(); - }).then(function(res) { + }).then(res => { expect(res.headers['x-custom-header']).to.equal('abc'); }); }); it('should accept custom host header', function() { - url = base + '/inspect'; + url = `${base}/inspect`; opts = { headers: { host: 'example.com' } }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { return res.json(); - }).then(function(res) { + }).then(res => { expect(res.headers['host']).to.equal('example.com'); }); }); it('should follow redirect code 301', function() { - url = base + '/redirect/301'; - return fetch(url).then(function(res) { - expect(res.url).to.equal(base + '/inspect'); + url = `${base}/redirect/301`; + return fetch(url).then(res => { + expect(res.url).to.equal(`${base}/inspect`); expect(res.status).to.equal(200); expect(res.ok).to.be.true; }); }); it('should follow redirect code 302', function() { - url = base + '/redirect/302'; - return fetch(url).then(function(res) { - expect(res.url).to.equal(base + '/inspect'); + url = `${base}/redirect/302`; + return fetch(url).then(res => { + expect(res.url).to.equal(`${base}/inspect`); expect(res.status).to.equal(200); }); }); it('should follow redirect code 303', function() { - url = base + '/redirect/303'; - return fetch(url).then(function(res) { - expect(res.url).to.equal(base + '/inspect'); + url = `${base}/redirect/303`; + return fetch(url).then(res => { + expect(res.url).to.equal(`${base}/inspect`); expect(res.status).to.equal(200); }); }); it('should follow redirect code 307', function() { - url = base + '/redirect/307'; - return fetch(url).then(function(res) { - expect(res.url).to.equal(base + '/inspect'); + url = `${base}/redirect/307`; + return fetch(url).then(res => { + expect(res.url).to.equal(`${base}/inspect`); expect(res.status).to.equal(200); }); }); it('should follow redirect code 308', function() { - url = base + '/redirect/308'; - return fetch(url).then(function(res) { - expect(res.url).to.equal(base + '/inspect'); + url = `${base}/redirect/308`; + return fetch(url).then(res => { + expect(res.url).to.equal(`${base}/inspect`); expect(res.status).to.equal(200); }); }); it('should follow redirect chain', function() { - url = base + '/redirect/chain'; - return fetch(url).then(function(res) { - expect(res.url).to.equal(base + '/inspect'); + url = `${base}/redirect/chain`; + return fetch(url).then(res => { + expect(res.url).to.equal(`${base}/inspect`); expect(res.status).to.equal(200); }); }); it('should follow POST request redirect code 301 with GET', function() { - url = base + '/redirect/301'; + url = `${base}/redirect/301`; opts = { method: 'POST' , body: 'a=1' }; - return fetch(url, opts).then(function(res) { - expect(res.url).to.equal(base + '/inspect'); + return fetch(url, opts).then(res => { + expect(res.url).to.equal(`${base}/inspect`); expect(res.status).to.equal(200); - return res.json().then(function(result) { + return res.json().then(result => { expect(result.method).to.equal('GET'); expect(result.body).to.equal(''); }); @@ -248,15 +249,15 @@ describe('node-fetch', function() { }); it('should follow POST request redirect code 302 with GET', function() { - url = base + '/redirect/302'; + url = `${base}/redirect/302`; opts = { method: 'POST' , body: 'a=1' }; - return fetch(url, opts).then(function(res) { - expect(res.url).to.equal(base + '/inspect'); + return fetch(url, opts).then(res => { + expect(res.url).to.equal(`${base}/inspect`); expect(res.status).to.equal(200); - return res.json().then(function(result) { + return res.json().then(result => { expect(result.method).to.equal('GET'); expect(result.body).to.equal(''); }); @@ -264,15 +265,15 @@ describe('node-fetch', function() { }); it('should follow redirect code 303 with GET', function() { - url = base + '/redirect/303'; + url = `${base}/redirect/303`; opts = { method: 'PUT' , body: 'a=1' }; - return fetch(url, opts).then(function(res) { - expect(res.url).to.equal(base + '/inspect'); + return fetch(url, opts).then(res => { + expect(res.url).to.equal(`${base}/inspect`); expect(res.status).to.equal(200); - return res.json().then(function(result) { + return res.json().then(result => { expect(result.method).to.equal('GET'); expect(result.body).to.equal(''); }); @@ -280,7 +281,7 @@ describe('node-fetch', function() { }); it('should obey maximum redirect, reject case', function() { - url = base + '/redirect/chain'; + url = `${base}/redirect/chain`; opts = { follow: 1 } @@ -290,18 +291,18 @@ describe('node-fetch', function() { }); it('should obey redirect chain, resolve case', function() { - url = base + '/redirect/chain'; + url = `${base}/redirect/chain`; opts = { follow: 2 } - return fetch(url, opts).then(function(res) { - expect(res.url).to.equal(base + '/inspect'); + return fetch(url, opts).then(res => { + expect(res.url).to.equal(`${base}/inspect`); expect(res.status).to.equal(200); }); }); it('should allow not following redirect', function() { - url = base + '/redirect/301'; + url = `${base}/redirect/301`; opts = { follow: 0 } @@ -311,19 +312,19 @@ describe('node-fetch', function() { }); it('should support redirect mode, manual flag', function() { - url = base + '/redirect/301'; + url = `${base}/redirect/301`; opts = { redirect: 'manual' }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { expect(res.url).to.equal(url); expect(res.status).to.equal(301); - expect(res.headers.get('location')).to.equal(base + '/inspect'); + expect(res.headers.get('location')).to.equal(`${base}/inspect`); }); }); it('should support redirect mode, error flag', function() { - url = base + '/redirect/301'; + url = `${base}/redirect/301`; opts = { redirect: 'error' }; @@ -333,11 +334,11 @@ describe('node-fetch', function() { }); it('should support redirect mode, manual flag when there is no redirect', function() { - url = base + '/hello'; + url = `${base}/hello`; opts = { redirect: 'manual' }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { expect(res.url).to.equal(url); expect(res.status).to.equal(200); expect(res.headers.get('location')).to.be.null; @@ -345,31 +346,31 @@ describe('node-fetch', function() { }); it('should follow redirect code 301 and keep existing headers', function() { - url = base + '/redirect/301'; + url = `${base}/redirect/301`; opts = { headers: new Headers({ 'x-custom-header': 'abc' }) }; - return fetch(url, opts).then(function(res) { - expect(res.url).to.equal(base + '/inspect'); + return fetch(url, opts).then(res => { + expect(res.url).to.equal(`${base}/inspect`); return res.json(); - }).then(function(res) { + }).then(res => { expect(res.headers['x-custom-header']).to.equal('abc'); }); }); it('should reject broken redirect', function() { - url = base + '/error/redirect'; + url = `${base}/error/redirect`; return expect(fetch(url)).to.eventually.be.rejected .and.be.an.instanceOf(FetchError) .and.have.property('type', 'invalid-redirect'); }); it('should not reject broken redirect under manual redirect', function() { - url = base + '/error/redirect'; + url = `${base}/error/redirect`; opts = { redirect: 'manual' }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { expect(res.url).to.equal(url); expect(res.status).to.equal(301); expect(res.headers.get('location')).to.be.null; @@ -377,13 +378,13 @@ describe('node-fetch', function() { }); it('should handle client-error response', function() { - url = base + '/error/400'; - return fetch(url).then(function(res) { + url = `${base}/error/400`; + return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); expect(res.status).to.equal(400); expect(res.statusText).to.equal('Bad Request'); expect(res.ok).to.be.false; - return res.text().then(function(result) { + return res.text().then(result => { expect(res.bodyUsed).to.be.true; expect(result).to.be.a('string'); expect(result).to.equal('client error'); @@ -392,13 +393,13 @@ describe('node-fetch', function() { }); it('should handle server-error response', function() { - url = base + '/error/500'; - return fetch(url).then(function(res) { + url = `${base}/error/500`; + return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); expect(res.status).to.equal(500); expect(res.statusText).to.equal('Internal Server Error'); expect(res.ok).to.be.false; - return res.text().then(function(result) { + return res.text().then(result => { expect(res.bodyUsed).to.be.true; expect(result).to.be.a('string'); expect(result).to.equal('server error'); @@ -407,7 +408,7 @@ describe('node-fetch', function() { }); it('should handle network-error response', function() { - url = base + '/error/reset'; + url = `${base}/error/reset`; return expect(fetch(url)).to.eventually.be.rejected .and.be.an.instanceOf(FetchError) .and.have.property('code', 'ECONNRESET'); @@ -421,20 +422,20 @@ describe('node-fetch', function() { }); it('should reject invalid json response', function() { - url = base + '/error/json'; - return fetch(url).then(function(res) { + url = `${base}/error/json`; + return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('application/json'); return expect(res.json()).to.eventually.be.rejectedWith(Error); }); }); it('should handle no content response', function() { - url = base + '/no-content'; - return fetch(url).then(function(res) { + url = `${base}/no-content`; + return fetch(url).then(res => { expect(res.status).to.equal(204); expect(res.statusText).to.equal('No Content'); expect(res.ok).to.be.true; - return res.text().then(function(result) { + return res.text().then(result => { expect(result).to.be.a('string'); expect(result).to.be.empty; }); @@ -442,9 +443,9 @@ describe('node-fetch', function() { }); it('should return empty object on no-content response', function() { - url = base + '/no-content'; - return fetch(url).then(function(res) { - return res.json().then(function(result) { + url = `${base}/no-content`; + return fetch(url).then(res => { + return res.json().then(result => { expect(result).to.be.an('object'); expect(result).to.be.empty; }); @@ -452,13 +453,13 @@ describe('node-fetch', function() { }); it('should handle no content response with gzip encoding', function() { - url = base + '/no-content/gzip'; - return fetch(url).then(function(res) { + url = `${base}/no-content/gzip`; + return fetch(url).then(res => { expect(res.status).to.equal(204); expect(res.statusText).to.equal('No Content'); expect(res.headers.get('content-encoding')).to.equal('gzip'); expect(res.ok).to.be.true; - return res.text().then(function(result) { + return res.text().then(result => { expect(result).to.be.a('string'); expect(result).to.be.empty; }); @@ -466,12 +467,12 @@ describe('node-fetch', function() { }); it('should handle not modified response', function() { - url = base + '/not-modified'; - return fetch(url).then(function(res) { + url = `${base}/not-modified`; + return fetch(url).then(res => { expect(res.status).to.equal(304); expect(res.statusText).to.equal('Not Modified'); expect(res.ok).to.be.false; - return res.text().then(function(result) { + return res.text().then(result => { expect(result).to.be.a('string'); expect(result).to.be.empty; }); @@ -479,13 +480,13 @@ describe('node-fetch', function() { }); it('should handle not modified response with gzip encoding', function() { - url = base + '/not-modified/gzip'; - return fetch(url).then(function(res) { + url = `${base}/not-modified/gzip`; + return fetch(url).then(res => { expect(res.status).to.equal(304); expect(res.statusText).to.equal('Not Modified'); expect(res.headers.get('content-encoding')).to.equal('gzip'); expect(res.ok).to.be.false; - return res.text().then(function(result) { + return res.text().then(result => { expect(result).to.be.a('string'); expect(result).to.be.empty; }); @@ -493,10 +494,10 @@ describe('node-fetch', function() { }); it('should decompress gzip response', function() { - url = base + '/gzip'; - return fetch(url).then(function(res) { + url = `${base}/gzip`; + return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); - return res.text().then(function(result) { + return res.text().then(result => { expect(result).to.be.a('string'); expect(result).to.equal('hello world'); }); @@ -504,10 +505,10 @@ describe('node-fetch', function() { }); it('should decompress deflate response', function() { - url = base + '/deflate'; - return fetch(url).then(function(res) { + url = `${base}/deflate`; + return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); - return res.text().then(function(result) { + return res.text().then(result => { expect(result).to.be.a('string'); expect(result).to.equal('hello world'); }); @@ -515,10 +516,10 @@ describe('node-fetch', function() { }); it('should decompress deflate raw response from old apache server', function() { - url = base + '/deflate-raw'; - return fetch(url).then(function(res) { + url = `${base}/deflate-raw`; + return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); - return res.text().then(function(result) { + return res.text().then(result => { expect(result).to.be.a('string'); expect(result).to.equal('hello world'); }); @@ -526,10 +527,10 @@ describe('node-fetch', function() { }); it('should skip decompression if unsupported', function() { - url = base + '/sdch'; - return fetch(url).then(function(res) { + url = `${base}/sdch`; + return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); - return res.text().then(function(result) { + return res.text().then(result => { expect(result).to.be.a('string'); expect(result).to.equal('fake sdch string'); }); @@ -537,8 +538,8 @@ describe('node-fetch', function() { }); it('should reject if response compression is invalid', function() { - url = base + '/invalid-content-encoding'; - return fetch(url).then(function(res) { + url = `${base}/invalid-content-encoding`; + return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); return expect(res.text()).to.eventually.be.rejected .and.be.an.instanceOf(FetchError) @@ -547,13 +548,13 @@ describe('node-fetch', function() { }); it('should allow disabling auto decompression', function() { - url = base + '/gzip'; + url = `${base}/gzip`; opts = { compress: false }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); - return res.text().then(function(result) { + return res.text().then(result => { expect(result).to.be.a('string'); expect(result).to.not.equal('hello world'); }); @@ -562,7 +563,7 @@ describe('node-fetch', function() { it('should allow custom timeout', function() { this.timeout(500); - url = base + '/timeout'; + url = `${base}/timeout`; opts = { timeout: 100 }; @@ -573,11 +574,11 @@ describe('node-fetch', function() { it('should allow custom timeout on response body', function() { this.timeout(500); - url = base + '/slow'; + url = `${base}/slow`; opts = { timeout: 100 }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { expect(res.ok).to.be.true; return expect(res.text()).to.eventually.be.rejected .and.be.an.instanceOf(FetchError) @@ -587,36 +588,36 @@ describe('node-fetch', function() { it('should clear internal timeout on fetch response', function (done) { this.timeout(1000); - spawn('node', ['-e', 'require("./")("' + base + '/hello", { timeout: 5000 })']) - .on('exit', function () { + spawn('node', ['-e', `require('./')('${base}/hello', { timeout: 5000 })`]) + .on('exit', () => { done(); }); }); it('should clear internal timeout on fetch redirect', function (done) { this.timeout(1000); - spawn('node', ['-e', 'require("./")("' + base + '/redirect/301", { timeout: 5000 })']) - .on('exit', function () { + spawn('node', ['-e', `require('./')('${base}/redirect/301', { timeout: 5000 })`]) + .on('exit', () => { done(); }); }); it('should clear internal timeout on fetch error', function (done) { this.timeout(1000); - spawn('node', ['-e', 'require("./")("' + base + '/error/reset", { timeout: 5000 })']) - .on('exit', function () { + spawn('node', ['-e', `require('./')('${base}/error/reset', { timeout: 5000 })`]) + .on('exit', () => { done(); }); }); it('should allow POST request', function() { - url = base + '/inspect'; + url = `${base}/inspect`; opts = { method: 'POST' }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { return res.json(); - }).then(function(res) { + }).then(res => { expect(res.method).to.equal('POST'); expect(res.headers['transfer-encoding']).to.be.undefined; expect(res.headers['content-length']).to.equal('0'); @@ -624,14 +625,14 @@ describe('node-fetch', function() { }); it('should allow POST request with string body', function() { - url = base + '/inspect'; + url = `${base}/inspect`; opts = { method: 'POST' , body: 'a=1' }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { return res.json(); - }).then(function(res) { + }).then(res => { expect(res.method).to.equal('POST'); expect(res.body).to.equal('a=1'); expect(res.headers['transfer-encoding']).to.be.undefined; @@ -640,14 +641,14 @@ describe('node-fetch', function() { }); it('should allow POST request with buffer body', function() { - url = base + '/inspect'; + url = `${base}/inspect`; opts = { method: 'POST' , body: new Buffer('a=1', 'utf-8') }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { return res.json(); - }).then(function(res) { + }).then(res => { expect(res.method).to.equal('POST'); expect(res.body).to.equal('a=1'); expect(res.headers['transfer-encoding']).to.equal('chunked'); @@ -656,17 +657,17 @@ describe('node-fetch', function() { }); it('should allow POST request with readable stream as body', function() { - var body = resumer().queue('a=1').end(); + let body = resumer().queue('a=1').end(); body = body.pipe(new stream.PassThrough()); - url = base + '/inspect'; + url = `${base}/inspect`; opts = { method: 'POST' - , body: body + , body }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { return res.json(); - }).then(function(res) { + }).then(res => { expect(res.method).to.equal('POST'); expect(res.body).to.equal('a=1'); expect(res.headers['transfer-encoding']).to.equal('chunked'); @@ -675,17 +676,17 @@ describe('node-fetch', function() { }); it('should allow POST request with form-data as body', function() { - var form = new FormData(); + const form = new FormData(); form.append('a','1'); - url = base + '/multipart'; + url = `${base}/multipart`; opts = { method: 'POST' , body: form }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { return res.json(); - }).then(function(res) { + }).then(res => { expect(res.method).to.equal('POST'); expect(res.headers['content-type']).to.contain('multipart/form-data'); expect(res.headers['content-length']).to.be.a('string'); @@ -694,18 +695,18 @@ describe('node-fetch', function() { }); it('should allow POST request with form-data using stream as body', function() { - var form = new FormData(); + const form = new FormData(); form.append('my_field', fs.createReadStream('test/dummy.txt')); - url = base + '/multipart'; + url = `${base}/multipart`; opts = { method: 'POST' , body: form }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { return res.json(); - }).then(function(res) { + }).then(res => { expect(res.method).to.equal('POST'); expect(res.headers['content-type']).to.contain('multipart/form-data'); expect(res.headers['content-length']).to.be.undefined; @@ -714,21 +715,21 @@ describe('node-fetch', function() { }); it('should allow POST request with form-data as body and custom headers', function() { - var form = new FormData(); + const form = new FormData(); form.append('a','1'); - var headers = form.getHeaders(); + const headers = form.getHeaders(); headers['b'] = '2'; - url = base + '/multipart'; + url = `${base}/multipart`; opts = { method: 'POST' , body: form - , headers: headers + , headers }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { return res.json(); - }).then(function(res) { + }).then(res => { expect(res.method).to.equal('POST'); expect(res.headers['content-type']).to.contain('multipart/form-data'); expect(res.headers['content-length']).to.be.a('string'); @@ -738,55 +739,55 @@ describe('node-fetch', function() { }); it('should allow POST request with object body', function() { - url = base + '/inspect'; + url = `${base}/inspect`; // note that fetch simply calls tostring on an object opts = { method: 'POST' , body: { a:1 } }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { return res.json(); - }).then(function(res) { + }).then(res => { expect(res.method).to.equal('POST'); expect(res.body).to.equal('[object Object]'); }); }); it('should allow PUT request', function() { - url = base + '/inspect'; + url = `${base}/inspect`; opts = { method: 'PUT' , body: 'a=1' }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { return res.json(); - }).then(function(res) { + }).then(res => { expect(res.method).to.equal('PUT'); expect(res.body).to.equal('a=1'); }); }); it('should allow DELETE request', function() { - url = base + '/inspect'; + url = `${base}/inspect`; opts = { method: 'DELETE' }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { return res.json(); - }).then(function(res) { + }).then(res => { expect(res.method).to.equal('DELETE'); }); }); it('should allow POST request with string body', function() { - url = base + '/inspect'; + url = `${base}/inspect`; opts = { method: 'POST' , body: 'a=1' }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { return res.json(); - }).then(function(res) { + }).then(res => { expect(res.method).to.equal('POST'); expect(res.body).to.equal('a=1'); expect(res.headers['transfer-encoding']).to.be.undefined; @@ -795,14 +796,14 @@ describe('node-fetch', function() { }); it('should allow DELETE request with string body', function() { - url = base + '/inspect'; + url = `${base}/inspect`; opts = { method: 'DELETE' , body: 'a=1' }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { return res.json(); - }).then(function(res) { + }).then(res => { expect(res.method).to.equal('DELETE'); expect(res.body).to.equal('a=1'); expect(res.headers['transfer-encoding']).to.be.undefined; @@ -811,55 +812,55 @@ describe('node-fetch', function() { }); it('should allow PATCH request', function() { - url = base + '/inspect'; + url = `${base}/inspect`; opts = { method: 'PATCH' , body: 'a=1' }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { return res.json(); - }).then(function(res) { + }).then(res => { expect(res.method).to.equal('PATCH'); expect(res.body).to.equal('a=1'); }); }); it('should allow HEAD request', function() { - url = base + '/hello'; + url = `${base}/hello`; opts = { method: 'HEAD' }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { expect(res.status).to.equal(200); expect(res.statusText).to.equal('OK'); expect(res.headers.get('content-type')).to.equal('text/plain'); expect(res.body).to.be.an.instanceof(stream.Transform); return res.text(); - }).then(function(text) { + }).then(text => { expect(text).to.equal(''); }); }); it('should allow HEAD request with content-encoding header', function() { - url = base + '/error/404'; + url = `${base}/error/404`; opts = { method: 'HEAD' }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { expect(res.status).to.equal(404); expect(res.headers.get('content-encoding')).to.equal('gzip'); return res.text(); - }).then(function(text) { + }).then(text => { expect(text).to.equal(''); }); }); it('should allow OPTIONS request', function() { - url = base + '/options'; + url = `${base}/options`; opts = { method: 'OPTIONS' }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { expect(res.status).to.equal(200); expect(res.statusText).to.equal('OK'); expect(res.headers.get('allow')).to.equal('GET, HEAD, OPTIONS'); @@ -868,10 +869,10 @@ describe('node-fetch', function() { }); it('should reject decoding body twice', function() { - url = base + '/plain'; - return fetch(url).then(function(res) { + url = `${base}/plain`; + return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); - return res.text().then(function(result) { + return res.text().then(result => { expect(res.bodyUsed).to.be.true; return expect(res.text()).to.eventually.be.rejectedWith(Error); }); @@ -879,11 +880,11 @@ describe('node-fetch', function() { }); it('should support maximum response size, multiple chunk', function() { - url = base + '/size/chunk'; + url = `${base}/size/chunk`; opts = { size: 5 }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { expect(res.status).to.equal(200); expect(res.headers.get('content-type')).to.equal('text/plain'); return expect(res.text()).to.eventually.be.rejected @@ -893,11 +894,11 @@ describe('node-fetch', function() { }); it('should support maximum response size, single chunk', function() { - url = base + '/size/long'; + url = `${base}/size/long`; opts = { size: 5 }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { expect(res.status).to.equal(200); expect(res.headers.get('content-type')).to.equal('text/plain'); return expect(res.text()).to.eventually.be.rejected @@ -907,142 +908,140 @@ describe('node-fetch', function() { }); it('should support encoding decode, xml dtd detect', function() { - url = base + '/encoding/euc-jp'; - return fetch(url).then(function(res) { + url = `${base}/encoding/euc-jp`; + return fetch(url).then(res => { expect(res.status).to.equal(200); - return res.text().then(function(result) { + return res.text().then(result => { expect(result).to.equal('日本語'); }); }); }); it('should support encoding decode, content-type detect', function() { - url = base + '/encoding/shift-jis'; - return fetch(url).then(function(res) { + url = `${base}/encoding/shift-jis`; + return fetch(url).then(res => { expect(res.status).to.equal(200); - return res.text().then(function(result) { + return res.text().then(result => { expect(result).to.equal('
日本語
'); }); }); }); it('should support encoding decode, html5 detect', function() { - url = base + '/encoding/gbk'; - return fetch(url).then(function(res) { + url = `${base}/encoding/gbk`; + return fetch(url).then(res => { expect(res.status).to.equal(200); - return res.text().then(function(result) { + return res.text().then(result => { expect(result).to.equal('
中文
'); }); }); }); it('should support encoding decode, html4 detect', function() { - url = base + '/encoding/gb2312'; - return fetch(url).then(function(res) { + url = `${base}/encoding/gb2312`; + return fetch(url).then(res => { expect(res.status).to.equal(200); - return res.text().then(function(result) { + return res.text().then(result => { expect(result).to.equal('
中文
'); }); }); }); it('should default to utf8 encoding', function() { - url = base + '/encoding/utf8'; - return fetch(url).then(function(res) { + url = `${base}/encoding/utf8`; + return fetch(url).then(res => { expect(res.status).to.equal(200); expect(res.headers.get('content-type')).to.be.null; - return res.text().then(function(result) { + return res.text().then(result => { expect(result).to.equal('中文'); }); }); }); it('should support uncommon content-type order, charset in front', function() { - url = base + '/encoding/order1'; - return fetch(url).then(function(res) { + url = `${base}/encoding/order1`; + return fetch(url).then(res => { expect(res.status).to.equal(200); - return res.text().then(function(result) { + return res.text().then(result => { expect(result).to.equal('中文'); }); }); }); it('should support uncommon content-type order, end with qs', function() { - url = base + '/encoding/order2'; - return fetch(url).then(function(res) { + url = `${base}/encoding/order2`; + return fetch(url).then(res => { expect(res.status).to.equal(200); - return res.text().then(function(result) { + return res.text().then(result => { expect(result).to.equal('中文'); }); }); }); it('should support chunked encoding, html4 detect', function() { - url = base + '/encoding/chunked'; - return fetch(url).then(function(res) { + url = `${base}/encoding/chunked`; + return fetch(url).then(res => { expect(res.status).to.equal(200); - // because node v0.12 doesn't have str.repeat - var padding = new Array(10 + 1).join('a'); - return res.text().then(function(result) { - expect(result).to.equal(padding + '
日本語
'); + const padding = 'a'.repeat(10); + return res.text().then(result => { + expect(result).to.equal(`${padding}
日本語
`); }); }); }); it('should only do encoding detection up to 1024 bytes', function() { - url = base + '/encoding/invalid'; - return fetch(url).then(function(res) { + url = `${base}/encoding/invalid`; + return fetch(url).then(res => { expect(res.status).to.equal(200); - // because node v0.12 doesn't have str.repeat - var padding = new Array(1200 + 1).join('a'); - return res.text().then(function(result) { - expect(result).to.not.equal(padding + '中文'); + const padding = 'a'.repeat(1200); + return res.text().then(result => { + expect(result).to.not.equal(`${padding}中文`); }); }); }); it('should allow piping response body as stream', function(done) { - url = base + '/hello'; - fetch(url).then(function(res) { + url = `${base}/hello`; + fetch(url).then(res => { expect(res.body).to.be.an.instanceof(stream.Transform); - res.body.on('data', function(chunk) { + res.body.on('data', chunk => { if (chunk === null) { return; } expect(chunk.toString()).to.equal('world'); }); - res.body.on('end', function() { + res.body.on('end', () => { done(); }); }); }); it('should allow cloning a response, and use both as stream', function(done) { - url = base + '/hello'; - return fetch(url).then(function(res) { - var counter = 0; - var r1 = res.clone(); + url = `${base}/hello`; + return fetch(url).then(res => { + let counter = 0; + const r1 = res.clone(); expect(res.body).to.be.an.instanceof(stream.Transform); expect(r1.body).to.be.an.instanceof(stream.Transform); - res.body.on('data', function(chunk) { + res.body.on('data', chunk => { if (chunk === null) { return; } expect(chunk.toString()).to.equal('world'); }); - res.body.on('end', function() { + res.body.on('end', () => { counter++; if (counter == 2) { done(); } }); - r1.body.on('data', function(chunk) { + r1.body.on('data', chunk => { if (chunk === null) { return; } expect(chunk.toString()).to.equal('world'); }); - r1.body.on('end', function() { + r1.body.on('end', () => { counter++; if (counter == 2) { done(); @@ -1052,10 +1051,10 @@ describe('node-fetch', function() { }); it('should allow cloning a json response and log it as text response', function() { - url = base + '/json'; - return fetch(url).then(function(res) { - var r1 = res.clone(); - return fetch.Promise.all([res.json(), r1.text()]).then(function(results) { + url = `${base}/json`; + return fetch(url).then(res => { + const r1 = res.clone(); + return fetch.Promise.all([res.json(), r1.text()]).then(results => { expect(results[0]).to.deep.equal({name: 'value'}); expect(results[1]).to.equal('{"name":"value"}'); }); @@ -1063,12 +1062,12 @@ describe('node-fetch', function() { }); it('should allow cloning a json response, and then log it as text response', function() { - url = base + '/json'; - return fetch(url).then(function(res) { - var r1 = res.clone(); - return res.json().then(function(result) { + url = `${base}/json`; + return fetch(url).then(res => { + const r1 = res.clone(); + return res.json().then(result => { expect(result).to.deep.equal({name: 'value'}); - return r1.text().then(function(result) { + return r1.text().then(result => { expect(result).to.equal('{"name":"value"}'); }); }); @@ -1076,12 +1075,12 @@ describe('node-fetch', function() { }); it('should allow cloning a json response, first log as text response, then return json object', function() { - url = base + '/json'; - return fetch(url).then(function(res) { - var r1 = res.clone(); - return r1.text().then(function(result) { + url = `${base}/json`; + return fetch(url).then(res => { + const r1 = res.clone(); + return r1.text().then(result => { expect(result).to.equal('{"name":"value"}'); - return res.json().then(function(result) { + return res.json().then(result => { expect(result).to.deep.equal({name: 'value'}); }); }); @@ -1089,19 +1088,19 @@ describe('node-fetch', function() { }); it('should not allow cloning a response after its been used', function() { - url = base + '/hello'; - return fetch(url).then(function(res) { - return res.text().then(function(result) { - expect(function() { - var r1 = res.clone(); + url = `${base}/hello`; + return fetch(url).then(res => + res.text().then(result => { + expect(() => { + res.clone(); }).to.throw(Error); - }); - }) + }) + ); }); it('should allow get all responses of a header', function() { - url = base + '/cookie'; - return fetch(url).then(function(res) { + url = `${base}/cookie`; + return fetch(url).then(res => { expect(res.headers.get('set-cookie')).to.equal('a=1'); expect(res.headers.get('Set-Cookie')).to.equal('a=1'); expect(res.headers.getAll('set-cookie')).to.deep.equal(['a=1', 'b=1']); @@ -1110,19 +1109,19 @@ describe('node-fetch', function() { }); it('should allow iterating through all headers', function() { - var headers = new Headers({ + const headers = new Headers({ a: 1 , b: [2, 3] , c: [4] }); expect(headers).to.have.property('forEach'); - var result = []; - headers.forEach(function(val, key) { + const result = []; + headers.forEach((val, key) => { result.push([key, val]); }); - var expected = [ + const expected = [ ["a", "1"] , ["b", "2"] , ["b", "3"] @@ -1132,7 +1131,7 @@ describe('node-fetch', function() { }); it('should allow iterating through all headers', function() { - var headers = new Headers({ + const headers = new Headers({ a: 1 , b: [2, 3] , c: [4] @@ -1142,7 +1141,7 @@ describe('node-fetch', function() { expect(headers).to.have.property('values'); expect(headers).to.have.property('entries'); - var result, expected; + let result, expected; result = []; for (let [key, val] of headers) { @@ -1191,8 +1190,8 @@ describe('node-fetch', function() { }); it('should allow deleting header', function() { - url = base + '/cookie'; - return fetch(url).then(function(res) { + url = `${base}/cookie`; + return fetch(url).then(res => { res.headers.delete('set-cookie'); expect(res.headers.get('set-cookie')).to.be.null; expect(res.headers.getAll('set-cookie')).to.be.empty; @@ -1200,25 +1199,25 @@ describe('node-fetch', function() { }); it('should send request with connection keep-alive if agent is provided', function() { - url = base + '/inspect'; + url = `${base}/inspect`; opts = { agent: new http.Agent({ keepAlive: true }) }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { return res.json(); - }).then(function(res) { + }).then(res => { expect(res.headers['connection']).to.equal('keep-alive'); }); }); it('should ignore unsupported attributes while reading headers', function() { - var FakeHeader = function() {}; + const FakeHeader = () => {}; // prototypes are ignored FakeHeader.prototype.z = 'fake'; - var res = new FakeHeader; + const res = new FakeHeader; // valid res.a = 'string'; res.b = ['1','2']; @@ -1236,8 +1235,8 @@ describe('node-fetch', function() { res.l = false; res.m = new Buffer('test'); - var h1 = new Headers(res); - var h1Raw = h1.raw(); + const h1 = new Headers(res); + const h1Raw = h1.raw(); expect(h1Raw['a']).to.include('string'); expect(h1Raw['b']).to.include('1'); @@ -1261,18 +1260,18 @@ describe('node-fetch', function() { }); it('should wrap headers', function() { - var h1 = new Headers({ + const h1 = new Headers({ a: '1' }); - var h1Raw = h1.raw(); + const h1Raw = h1.raw(); - var h2 = new Headers(h1); + const h2 = new Headers(h1); h2.set('b', '1'); - var h2Raw = h2.raw(); + const h2Raw = h2.raw(); - var h3 = new Headers(h2); + const h3 = new Headers(h2); h3.append('a', '2'); - var h3Raw = h3.raw(); + const h3Raw = h3.raw(); expect(h1Raw['a']).to.include('1'); expect(h1Raw['a']).to.not.include('2'); @@ -1287,9 +1286,9 @@ describe('node-fetch', function() { }); it('should support fetch with Request instance', function() { - url = base + '/hello'; - var req = new Request(url); - return fetch(req).then(function(res) { + url = `${base}/hello`; + const req = new Request(url); + return fetch(req).then(res => { expect(res.url).to.equal(url); expect(res.ok).to.be.true; expect(res.status).to.equal(200); @@ -1297,17 +1296,17 @@ describe('node-fetch', function() { }); it('should support wrapping Request instance', function() { - url = base + '/hello'; + url = `${base}/hello`; - var form = new FormData(); + const form = new FormData(); form.append('a', '1'); - var r1 = new Request(url, { + const r1 = new Request(url, { method: 'POST' , follow: 1 , body: form }); - var r2 = new Request(r1, { + const r2 = new Request(r1, { follow: 2 }); @@ -1322,8 +1321,8 @@ describe('node-fetch', function() { }); it('should support overwrite Request instance', function() { - url = base + '/inspect'; - var req = new Request(url, { + url = `${base}/inspect`; + const req = new Request(url, { method: 'POST' , headers: { a: '1' @@ -1334,25 +1333,25 @@ describe('node-fetch', function() { , headers: { a: '2' } - }).then(function(res) { + }).then(res => { return res.json(); - }).then(function(body) { + }).then(body => { expect(body.method).to.equal('GET'); expect(body.headers.a).to.equal('2'); }); }); it('should support empty options in Response constructor', function() { - var body = resumer().queue('a=1').end(); + let body = resumer().queue('a=1').end(); body = body.pipe(new stream.PassThrough()); - var res = new Response(body); - return res.text().then(function(result) { + const res = new Response(body); + return res.text().then(result => { expect(result).to.equal('a=1'); }); }); it('should support parsing headers in Response constructor', function() { - var res = new Response(null, { + const res = new Response(null, { headers: { a: '1' } @@ -1361,30 +1360,30 @@ describe('node-fetch', function() { }); it('should support text() method in Response constructor', function() { - var res = new Response('a=1'); - return res.text().then(function(result) { + const res = new Response('a=1'); + return res.text().then(result => { expect(result).to.equal('a=1'); }); }); it('should support json() method in Response constructor', function() { - var res = new Response('{"a":1}'); - return res.json().then(function(result) { + const res = new Response('{"a":1}'); + return res.json().then(result => { expect(result.a).to.equal(1); }); }); it('should support buffer() method in Response constructor', function() { - var res = new Response('a=1'); - return res.buffer().then(function(result) { + const res = new Response('a=1'); + return res.buffer().then(result => { expect(result.toString()).to.equal('a=1'); }); }); it('should support clone() method in Response constructor', function() { - var body = resumer().queue('a=1').end(); + let body = resumer().queue('a=1').end(); body = body.pipe(new stream.PassThrough()); - var res = new Response(body, { + const res = new Response(body, { headers: { a: '1' } @@ -1392,7 +1391,7 @@ describe('node-fetch', function() { , status: 346 , statusText: 'production' }); - var cl = res.clone(); + const cl = res.clone(); expect(cl.headers.get('a')).to.equal('1'); expect(cl.url).to.equal(base); expect(cl.status).to.equal(346); @@ -1400,42 +1399,42 @@ describe('node-fetch', function() { expect(cl.ok).to.be.false; // clone body shouldn't be the same body expect(cl.body).to.not.equal(body); - return cl.text().then(function(result) { + return cl.text().then(result => { expect(result).to.equal('a=1'); }); }); it('should support stream as body in Response constructor', function() { - var body = resumer().queue('a=1').end(); + let body = resumer().queue('a=1').end(); body = body.pipe(new stream.PassThrough()); - var res = new Response(body); - return res.text().then(function(result) { + const res = new Response(body); + return res.text().then(result => { expect(result).to.equal('a=1'); }); }); it('should support string as body in Response constructor', function() { - var res = new Response('a=1'); - return res.text().then(function(result) { + const res = new Response('a=1'); + return res.text().then(result => { expect(result).to.equal('a=1'); }); }); it('should support buffer as body in Response constructor', function() { - var res = new Response(new Buffer('a=1')); - return res.text().then(function(result) { + const res = new Response(new Buffer('a=1')); + return res.text().then(result => { expect(result).to.equal('a=1'); }); }); it('should default to 200 as status code', function() { - var res = new Response(null); + const res = new Response(null); expect(res.status).to.equal(200); }); it('should support parsing headers in Request constructor', function() { url = base; - var req = new Request(url, { + const req = new Request(url, { headers: { a: '1' } @@ -1446,50 +1445,50 @@ describe('node-fetch', function() { it('should support text() method in Request constructor', function() { url = base; - var req = new Request(url, { + const req = new Request(url, { body: 'a=1' }); expect(req.url).to.equal(url); - return req.text().then(function(result) { + return req.text().then(result => { expect(result).to.equal('a=1'); }); }); it('should support json() method in Request constructor', function() { url = base; - var req = new Request(url, { + const req = new Request(url, { body: '{"a":1}' }); expect(req.url).to.equal(url); - return req.json().then(function(result) { + return req.json().then(result => { expect(result.a).to.equal(1); }); }); it('should support buffer() method in Request constructor', function() { url = base; - var req = new Request(url, { + const req = new Request(url, { body: 'a=1' }); expect(req.url).to.equal(url); - return req.buffer().then(function(result) { + return req.buffer().then(result => { expect(result.toString()).to.equal('a=1'); }); }); it('should support arbitrary url in Request constructor', function() { url = 'anything'; - var req = new Request(url); + const req = new Request(url); expect(req.url).to.equal('anything'); }); it('should support clone() method in Request constructor', function() { url = base; - var body = resumer().queue('a=1').end(); + let body = resumer().queue('a=1').end(); body = body.pipe(new stream.PassThrough()); - var agent = new http.Agent(); - var req = new Request(url, { - body: body + const agent = new http.Agent(); + const req = new Request(url, { + body , method: 'POST' , redirect: 'manual' , headers: { @@ -1497,9 +1496,9 @@ describe('node-fetch', function() { } , follow: 3 , compress: false - , agent: agent + , agent }); - var cl = req.clone(); + const cl = req.clone(); expect(cl.url).to.equal(url); expect(cl.method).to.equal('POST'); expect(cl.redirect).to.equal('manual'); @@ -1511,24 +1510,24 @@ describe('node-fetch', function() { expect(cl.agent).to.equal(agent); // clone body shouldn't be the same body expect(cl.body).to.not.equal(body); - return fetch.Promise.all([cl.text(), req.text()]).then(function(results) { + return fetch.Promise.all([cl.text(), req.text()]).then(results => { expect(results[0]).to.equal('a=1'); expect(results[1]).to.equal('a=1'); }); }); it('should support text(), json() and buffer() method in Body constructor', function() { - var body = new Body('a=1'); + const body = new Body('a=1'); expect(body).to.have.property('text'); expect(body).to.have.property('json'); expect(body).to.have.property('buffer'); }); it('should create custom FetchError', function() { - var systemError = new Error('system'); + const systemError = new Error('system'); systemError.code = 'ESOMEERROR'; - var err = new FetchError('test message', 'test-error', systemError); + const err = new FetchError('test message', 'test-error', systemError); expect(err).to.be.an.instanceof(Error); expect(err).to.be.an.instanceof(FetchError); expect(err.name).to.equal('FetchError'); @@ -1544,7 +1543,7 @@ describe('node-fetch', function() { opts = { method: 'HEAD' }; - return fetch(url, opts).then(function(res) { + return fetch(url, opts).then(res => { expect(res.status).to.equal(200); expect(res.ok).to.be.true; }); From a0be6aa34aa4c98325011987e48074db0b933879 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sat, 8 Oct 2016 18:31:42 -0700 Subject: [PATCH 007/223] Simplify Request constructor Also make Request more standard compliant: > The `url` attribute's getter must return request's url, **serialized.** --- src/request.js | 21 +++--- test/test.js | 196 ++++++++++++++++++++++++------------------------- 2 files changed, 107 insertions(+), 110 deletions(-) diff --git a/src/request.js b/src/request.js index 1508d67f9..978e5ec6d 100644 --- a/src/request.js +++ b/src/request.js @@ -5,7 +5,7 @@ * Request class contains server only options */ -import { parse as parse_url } from 'url'; +import { format as format_url, parse as parse_url } from 'url'; import Headers from './headers.js'; import Body, { clone } from './body'; @@ -18,16 +18,14 @@ import Body, { clone } from './body'; */ export default class Request extends Body { constructor(input, init = {}) { - let url, url_parsed; + let parsedURL; // normalize input if (!(input instanceof Request)) { - url = input; - url_parsed = parse_url(url); + parsedURL = parse_url(input); input = {}; } else { - url = input.url; - url_parsed = parse_url(url); + parsedURL = parse_url(input.url); } super(init.body || clone(input), { @@ -39,7 +37,6 @@ export default class Request extends Body { this.method = init.method || input.method || 'GET'; this.redirect = init.redirect || input.redirect || 'follow'; this.headers = new Headers(init.headers || input.headers || {}); - this.url = url; // server only options this.follow = init.follow !== undefined ? @@ -52,11 +49,11 @@ export default class Request extends Body { this.agent = init.agent || input.agent; // server request options - this.protocol = url_parsed.protocol; - this.hostname = url_parsed.hostname; - this.port = url_parsed.port; - this.path = url_parsed.path; - this.auth = url_parsed.auth; + Object.assign(this, parsedURL); + } + + get url() { + return format_url(this); } /** diff --git a/test/test.js b/test/test.js index a1ea39a31..180953d82 100644 --- a/test/test.js +++ b/test/test.js @@ -32,7 +32,7 @@ describe('node-fetch', () => { before(done => { local = new TestServer(); - base = 'http://' + local.hostname + ':' + local.port; + base = `http://${local.hostname}:${local.port}/`; local.start(done); }); @@ -95,7 +95,7 @@ describe('node-fetch', () => { }); it('should resolve into response', function() { - url = `${base}/hello`; + url = `${base}hello`; return fetch(url).then(res => { expect(res).to.be.an.instanceof(Response); expect(res.headers).to.be.an.instanceof(Headers); @@ -110,7 +110,7 @@ describe('node-fetch', () => { }); it('should accept plain text response', function() { - url = `${base}/plain`; + url = `${base}plain`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); return res.text().then(result => { @@ -122,7 +122,7 @@ describe('node-fetch', () => { }); it('should accept html response (like plain text)', function() { - url = `${base}/html`; + url = `${base}html`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/html'); return res.text().then(result => { @@ -134,7 +134,7 @@ describe('node-fetch', () => { }); it('should accept json response', function() { - url = `${base}/json`; + url = `${base}json`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('application/json'); return res.json().then(result => { @@ -146,7 +146,7 @@ describe('node-fetch', () => { }); it('should send request with custom headers', function() { - url = `${base}/inspect`; + url = `${base}inspect`; opts = { headers: { 'x-custom-header': 'abc' } }; @@ -158,7 +158,7 @@ describe('node-fetch', () => { }); it('should accept headers instance', function() { - url = `${base}/inspect`; + url = `${base}inspect`; opts = { headers: new Headers({ 'x-custom-header': 'abc' }) }; @@ -170,7 +170,7 @@ describe('node-fetch', () => { }); it('should accept custom host header', function() { - url = `${base}/inspect`; + url = `${base}inspect`; opts = { headers: { host: 'example.com' @@ -184,62 +184,62 @@ describe('node-fetch', () => { }); it('should follow redirect code 301', function() { - url = `${base}/redirect/301`; + url = `${base}redirect/301`; return fetch(url).then(res => { - expect(res.url).to.equal(`${base}/inspect`); + expect(res.url).to.equal(`${base}inspect`); expect(res.status).to.equal(200); expect(res.ok).to.be.true; }); }); it('should follow redirect code 302', function() { - url = `${base}/redirect/302`; + url = `${base}redirect/302`; return fetch(url).then(res => { - expect(res.url).to.equal(`${base}/inspect`); + expect(res.url).to.equal(`${base}inspect`); expect(res.status).to.equal(200); }); }); it('should follow redirect code 303', function() { - url = `${base}/redirect/303`; + url = `${base}redirect/303`; return fetch(url).then(res => { - expect(res.url).to.equal(`${base}/inspect`); + expect(res.url).to.equal(`${base}inspect`); expect(res.status).to.equal(200); }); }); it('should follow redirect code 307', function() { - url = `${base}/redirect/307`; + url = `${base}redirect/307`; return fetch(url).then(res => { - expect(res.url).to.equal(`${base}/inspect`); + expect(res.url).to.equal(`${base}inspect`); expect(res.status).to.equal(200); }); }); it('should follow redirect code 308', function() { - url = `${base}/redirect/308`; + url = `${base}redirect/308`; return fetch(url).then(res => { - expect(res.url).to.equal(`${base}/inspect`); + expect(res.url).to.equal(`${base}inspect`); expect(res.status).to.equal(200); }); }); it('should follow redirect chain', function() { - url = `${base}/redirect/chain`; + url = `${base}redirect/chain`; return fetch(url).then(res => { - expect(res.url).to.equal(`${base}/inspect`); + expect(res.url).to.equal(`${base}inspect`); expect(res.status).to.equal(200); }); }); it('should follow POST request redirect code 301 with GET', function() { - url = `${base}/redirect/301`; + url = `${base}redirect/301`; opts = { method: 'POST' , body: 'a=1' }; return fetch(url, opts).then(res => { - expect(res.url).to.equal(`${base}/inspect`); + expect(res.url).to.equal(`${base}inspect`); expect(res.status).to.equal(200); return res.json().then(result => { expect(result.method).to.equal('GET'); @@ -249,13 +249,13 @@ describe('node-fetch', () => { }); it('should follow POST request redirect code 302 with GET', function() { - url = `${base}/redirect/302`; + url = `${base}redirect/302`; opts = { method: 'POST' , body: 'a=1' }; return fetch(url, opts).then(res => { - expect(res.url).to.equal(`${base}/inspect`); + expect(res.url).to.equal(`${base}inspect`); expect(res.status).to.equal(200); return res.json().then(result => { expect(result.method).to.equal('GET'); @@ -265,13 +265,13 @@ describe('node-fetch', () => { }); it('should follow redirect code 303 with GET', function() { - url = `${base}/redirect/303`; + url = `${base}redirect/303`; opts = { method: 'PUT' , body: 'a=1' }; return fetch(url, opts).then(res => { - expect(res.url).to.equal(`${base}/inspect`); + expect(res.url).to.equal(`${base}inspect`); expect(res.status).to.equal(200); return res.json().then(result => { expect(result.method).to.equal('GET'); @@ -281,7 +281,7 @@ describe('node-fetch', () => { }); it('should obey maximum redirect, reject case', function() { - url = `${base}/redirect/chain`; + url = `${base}redirect/chain`; opts = { follow: 1 } @@ -291,18 +291,18 @@ describe('node-fetch', () => { }); it('should obey redirect chain, resolve case', function() { - url = `${base}/redirect/chain`; + url = `${base}redirect/chain`; opts = { follow: 2 } return fetch(url, opts).then(res => { - expect(res.url).to.equal(`${base}/inspect`); + expect(res.url).to.equal(`${base}inspect`); expect(res.status).to.equal(200); }); }); it('should allow not following redirect', function() { - url = `${base}/redirect/301`; + url = `${base}redirect/301`; opts = { follow: 0 } @@ -312,19 +312,19 @@ describe('node-fetch', () => { }); it('should support redirect mode, manual flag', function() { - url = `${base}/redirect/301`; + url = `${base}redirect/301`; opts = { redirect: 'manual' }; return fetch(url, opts).then(res => { expect(res.url).to.equal(url); expect(res.status).to.equal(301); - expect(res.headers.get('location')).to.equal(`${base}/inspect`); + expect(res.headers.get('location')).to.equal(`${base}inspect`); }); }); it('should support redirect mode, error flag', function() { - url = `${base}/redirect/301`; + url = `${base}redirect/301`; opts = { redirect: 'error' }; @@ -334,7 +334,7 @@ describe('node-fetch', () => { }); it('should support redirect mode, manual flag when there is no redirect', function() { - url = `${base}/hello`; + url = `${base}hello`; opts = { redirect: 'manual' }; @@ -346,12 +346,12 @@ describe('node-fetch', () => { }); it('should follow redirect code 301 and keep existing headers', function() { - url = `${base}/redirect/301`; + url = `${base}redirect/301`; opts = { headers: new Headers({ 'x-custom-header': 'abc' }) }; return fetch(url, opts).then(res => { - expect(res.url).to.equal(`${base}/inspect`); + expect(res.url).to.equal(`${base}inspect`); return res.json(); }).then(res => { expect(res.headers['x-custom-header']).to.equal('abc'); @@ -359,14 +359,14 @@ describe('node-fetch', () => { }); it('should reject broken redirect', function() { - url = `${base}/error/redirect`; + url = `${base}error/redirect`; return expect(fetch(url)).to.eventually.be.rejected .and.be.an.instanceOf(FetchError) .and.have.property('type', 'invalid-redirect'); }); it('should not reject broken redirect under manual redirect', function() { - url = `${base}/error/redirect`; + url = `${base}error/redirect`; opts = { redirect: 'manual' }; @@ -378,7 +378,7 @@ describe('node-fetch', () => { }); it('should handle client-error response', function() { - url = `${base}/error/400`; + url = `${base}error/400`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); expect(res.status).to.equal(400); @@ -393,7 +393,7 @@ describe('node-fetch', () => { }); it('should handle server-error response', function() { - url = `${base}/error/500`; + url = `${base}error/500`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); expect(res.status).to.equal(500); @@ -408,7 +408,7 @@ describe('node-fetch', () => { }); it('should handle network-error response', function() { - url = `${base}/error/reset`; + url = `${base}error/reset`; return expect(fetch(url)).to.eventually.be.rejected .and.be.an.instanceOf(FetchError) .and.have.property('code', 'ECONNRESET'); @@ -422,7 +422,7 @@ describe('node-fetch', () => { }); it('should reject invalid json response', function() { - url = `${base}/error/json`; + url = `${base}error/json`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('application/json'); return expect(res.json()).to.eventually.be.rejectedWith(Error); @@ -430,7 +430,7 @@ describe('node-fetch', () => { }); it('should handle no content response', function() { - url = `${base}/no-content`; + url = `${base}no-content`; return fetch(url).then(res => { expect(res.status).to.equal(204); expect(res.statusText).to.equal('No Content'); @@ -443,7 +443,7 @@ describe('node-fetch', () => { }); it('should return empty object on no-content response', function() { - url = `${base}/no-content`; + url = `${base}no-content`; return fetch(url).then(res => { return res.json().then(result => { expect(result).to.be.an('object'); @@ -453,7 +453,7 @@ describe('node-fetch', () => { }); it('should handle no content response with gzip encoding', function() { - url = `${base}/no-content/gzip`; + url = `${base}no-content/gzip`; return fetch(url).then(res => { expect(res.status).to.equal(204); expect(res.statusText).to.equal('No Content'); @@ -467,7 +467,7 @@ describe('node-fetch', () => { }); it('should handle not modified response', function() { - url = `${base}/not-modified`; + url = `${base}not-modified`; return fetch(url).then(res => { expect(res.status).to.equal(304); expect(res.statusText).to.equal('Not Modified'); @@ -480,7 +480,7 @@ describe('node-fetch', () => { }); it('should handle not modified response with gzip encoding', function() { - url = `${base}/not-modified/gzip`; + url = `${base}not-modified/gzip`; return fetch(url).then(res => { expect(res.status).to.equal(304); expect(res.statusText).to.equal('Not Modified'); @@ -494,7 +494,7 @@ describe('node-fetch', () => { }); it('should decompress gzip response', function() { - url = `${base}/gzip`; + url = `${base}gzip`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); return res.text().then(result => { @@ -505,7 +505,7 @@ describe('node-fetch', () => { }); it('should decompress deflate response', function() { - url = `${base}/deflate`; + url = `${base}deflate`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); return res.text().then(result => { @@ -516,7 +516,7 @@ describe('node-fetch', () => { }); it('should decompress deflate raw response from old apache server', function() { - url = `${base}/deflate-raw`; + url = `${base}deflate-raw`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); return res.text().then(result => { @@ -527,7 +527,7 @@ describe('node-fetch', () => { }); it('should skip decompression if unsupported', function() { - url = `${base}/sdch`; + url = `${base}sdch`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); return res.text().then(result => { @@ -538,7 +538,7 @@ describe('node-fetch', () => { }); it('should reject if response compression is invalid', function() { - url = `${base}/invalid-content-encoding`; + url = `${base}invalid-content-encoding`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); return expect(res.text()).to.eventually.be.rejected @@ -548,7 +548,7 @@ describe('node-fetch', () => { }); it('should allow disabling auto decompression', function() { - url = `${base}/gzip`; + url = `${base}gzip`; opts = { compress: false }; @@ -563,7 +563,7 @@ describe('node-fetch', () => { it('should allow custom timeout', function() { this.timeout(500); - url = `${base}/timeout`; + url = `${base}timeout`; opts = { timeout: 100 }; @@ -574,7 +574,7 @@ describe('node-fetch', () => { it('should allow custom timeout on response body', function() { this.timeout(500); - url = `${base}/slow`; + url = `${base}slow`; opts = { timeout: 100 }; @@ -588,7 +588,7 @@ describe('node-fetch', () => { it('should clear internal timeout on fetch response', function (done) { this.timeout(1000); - spawn('node', ['-e', `require('./')('${base}/hello', { timeout: 5000 })`]) + spawn('node', ['-e', `require('./')('${base}hello', { timeout: 5000 })`]) .on('exit', () => { done(); }); @@ -596,7 +596,7 @@ describe('node-fetch', () => { it('should clear internal timeout on fetch redirect', function (done) { this.timeout(1000); - spawn('node', ['-e', `require('./')('${base}/redirect/301', { timeout: 5000 })`]) + spawn('node', ['-e', `require('./')('${base}redirect/301', { timeout: 5000 })`]) .on('exit', () => { done(); }); @@ -604,14 +604,14 @@ describe('node-fetch', () => { it('should clear internal timeout on fetch error', function (done) { this.timeout(1000); - spawn('node', ['-e', `require('./')('${base}/error/reset', { timeout: 5000 })`]) + spawn('node', ['-e', `require('./')('${base}error/reset', { timeout: 5000 })`]) .on('exit', () => { done(); }); }); it('should allow POST request', function() { - url = `${base}/inspect`; + url = `${base}inspect`; opts = { method: 'POST' }; @@ -625,7 +625,7 @@ describe('node-fetch', () => { }); it('should allow POST request with string body', function() { - url = `${base}/inspect`; + url = `${base}inspect`; opts = { method: 'POST' , body: 'a=1' @@ -641,7 +641,7 @@ describe('node-fetch', () => { }); it('should allow POST request with buffer body', function() { - url = `${base}/inspect`; + url = `${base}inspect`; opts = { method: 'POST' , body: new Buffer('a=1', 'utf-8') @@ -660,7 +660,7 @@ describe('node-fetch', () => { let body = resumer().queue('a=1').end(); body = body.pipe(new stream.PassThrough()); - url = `${base}/inspect`; + url = `${base}inspect`; opts = { method: 'POST' , body @@ -679,7 +679,7 @@ describe('node-fetch', () => { const form = new FormData(); form.append('a','1'); - url = `${base}/multipart`; + url = `${base}multipart`; opts = { method: 'POST' , body: form @@ -698,7 +698,7 @@ describe('node-fetch', () => { const form = new FormData(); form.append('my_field', fs.createReadStream('test/dummy.txt')); - url = `${base}/multipart`; + url = `${base}multipart`; opts = { method: 'POST' , body: form @@ -721,7 +721,7 @@ describe('node-fetch', () => { const headers = form.getHeaders(); headers['b'] = '2'; - url = `${base}/multipart`; + url = `${base}multipart`; opts = { method: 'POST' , body: form @@ -739,7 +739,7 @@ describe('node-fetch', () => { }); it('should allow POST request with object body', function() { - url = `${base}/inspect`; + url = `${base}inspect`; // note that fetch simply calls tostring on an object opts = { method: 'POST' @@ -754,7 +754,7 @@ describe('node-fetch', () => { }); it('should allow PUT request', function() { - url = `${base}/inspect`; + url = `${base}inspect`; opts = { method: 'PUT' , body: 'a=1' @@ -768,7 +768,7 @@ describe('node-fetch', () => { }); it('should allow DELETE request', function() { - url = `${base}/inspect`; + url = `${base}inspect`; opts = { method: 'DELETE' }; @@ -780,7 +780,7 @@ describe('node-fetch', () => { }); it('should allow POST request with string body', function() { - url = `${base}/inspect`; + url = `${base}inspect`; opts = { method: 'POST' , body: 'a=1' @@ -796,7 +796,7 @@ describe('node-fetch', () => { }); it('should allow DELETE request with string body', function() { - url = `${base}/inspect`; + url = `${base}inspect`; opts = { method: 'DELETE' , body: 'a=1' @@ -812,7 +812,7 @@ describe('node-fetch', () => { }); it('should allow PATCH request', function() { - url = `${base}/inspect`; + url = `${base}inspect`; opts = { method: 'PATCH' , body: 'a=1' @@ -826,7 +826,7 @@ describe('node-fetch', () => { }); it('should allow HEAD request', function() { - url = `${base}/hello`; + url = `${base}hello`; opts = { method: 'HEAD' }; @@ -842,7 +842,7 @@ describe('node-fetch', () => { }); it('should allow HEAD request with content-encoding header', function() { - url = `${base}/error/404`; + url = `${base}error/404`; opts = { method: 'HEAD' }; @@ -856,7 +856,7 @@ describe('node-fetch', () => { }); it('should allow OPTIONS request', function() { - url = `${base}/options`; + url = `${base}options`; opts = { method: 'OPTIONS' }; @@ -869,7 +869,7 @@ describe('node-fetch', () => { }); it('should reject decoding body twice', function() { - url = `${base}/plain`; + url = `${base}plain`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); return res.text().then(result => { @@ -880,7 +880,7 @@ describe('node-fetch', () => { }); it('should support maximum response size, multiple chunk', function() { - url = `${base}/size/chunk`; + url = `${base}size/chunk`; opts = { size: 5 }; @@ -894,7 +894,7 @@ describe('node-fetch', () => { }); it('should support maximum response size, single chunk', function() { - url = `${base}/size/long`; + url = `${base}size/long`; opts = { size: 5 }; @@ -908,7 +908,7 @@ describe('node-fetch', () => { }); it('should support encoding decode, xml dtd detect', function() { - url = `${base}/encoding/euc-jp`; + url = `${base}encoding/euc-jp`; return fetch(url).then(res => { expect(res.status).to.equal(200); return res.text().then(result => { @@ -918,7 +918,7 @@ describe('node-fetch', () => { }); it('should support encoding decode, content-type detect', function() { - url = `${base}/encoding/shift-jis`; + url = `${base}encoding/shift-jis`; return fetch(url).then(res => { expect(res.status).to.equal(200); return res.text().then(result => { @@ -928,7 +928,7 @@ describe('node-fetch', () => { }); it('should support encoding decode, html5 detect', function() { - url = `${base}/encoding/gbk`; + url = `${base}encoding/gbk`; return fetch(url).then(res => { expect(res.status).to.equal(200); return res.text().then(result => { @@ -938,7 +938,7 @@ describe('node-fetch', () => { }); it('should support encoding decode, html4 detect', function() { - url = `${base}/encoding/gb2312`; + url = `${base}encoding/gb2312`; return fetch(url).then(res => { expect(res.status).to.equal(200); return res.text().then(result => { @@ -948,7 +948,7 @@ describe('node-fetch', () => { }); it('should default to utf8 encoding', function() { - url = `${base}/encoding/utf8`; + url = `${base}encoding/utf8`; return fetch(url).then(res => { expect(res.status).to.equal(200); expect(res.headers.get('content-type')).to.be.null; @@ -959,7 +959,7 @@ describe('node-fetch', () => { }); it('should support uncommon content-type order, charset in front', function() { - url = `${base}/encoding/order1`; + url = `${base}encoding/order1`; return fetch(url).then(res => { expect(res.status).to.equal(200); return res.text().then(result => { @@ -969,7 +969,7 @@ describe('node-fetch', () => { }); it('should support uncommon content-type order, end with qs', function() { - url = `${base}/encoding/order2`; + url = `${base}encoding/order2`; return fetch(url).then(res => { expect(res.status).to.equal(200); return res.text().then(result => { @@ -979,7 +979,7 @@ describe('node-fetch', () => { }); it('should support chunked encoding, html4 detect', function() { - url = `${base}/encoding/chunked`; + url = `${base}encoding/chunked`; return fetch(url).then(res => { expect(res.status).to.equal(200); const padding = 'a'.repeat(10); @@ -990,7 +990,7 @@ describe('node-fetch', () => { }); it('should only do encoding detection up to 1024 bytes', function() { - url = `${base}/encoding/invalid`; + url = `${base}encoding/invalid`; return fetch(url).then(res => { expect(res.status).to.equal(200); const padding = 'a'.repeat(1200); @@ -1001,7 +1001,7 @@ describe('node-fetch', () => { }); it('should allow piping response body as stream', function(done) { - url = `${base}/hello`; + url = `${base}hello`; fetch(url).then(res => { expect(res.body).to.be.an.instanceof(stream.Transform); res.body.on('data', chunk => { @@ -1017,7 +1017,7 @@ describe('node-fetch', () => { }); it('should allow cloning a response, and use both as stream', function(done) { - url = `${base}/hello`; + url = `${base}hello`; return fetch(url).then(res => { let counter = 0; const r1 = res.clone(); @@ -1051,7 +1051,7 @@ describe('node-fetch', () => { }); it('should allow cloning a json response and log it as text response', function() { - url = `${base}/json`; + url = `${base}json`; return fetch(url).then(res => { const r1 = res.clone(); return fetch.Promise.all([res.json(), r1.text()]).then(results => { @@ -1062,7 +1062,7 @@ describe('node-fetch', () => { }); it('should allow cloning a json response, and then log it as text response', function() { - url = `${base}/json`; + url = `${base}json`; return fetch(url).then(res => { const r1 = res.clone(); return res.json().then(result => { @@ -1075,7 +1075,7 @@ describe('node-fetch', () => { }); it('should allow cloning a json response, first log as text response, then return json object', function() { - url = `${base}/json`; + url = `${base}json`; return fetch(url).then(res => { const r1 = res.clone(); return r1.text().then(result => { @@ -1088,7 +1088,7 @@ describe('node-fetch', () => { }); it('should not allow cloning a response after its been used', function() { - url = `${base}/hello`; + url = `${base}hello`; return fetch(url).then(res => res.text().then(result => { expect(() => { @@ -1099,7 +1099,7 @@ describe('node-fetch', () => { }); it('should allow get all responses of a header', function() { - url = `${base}/cookie`; + url = `${base}cookie`; return fetch(url).then(res => { expect(res.headers.get('set-cookie')).to.equal('a=1'); expect(res.headers.get('Set-Cookie')).to.equal('a=1'); @@ -1190,7 +1190,7 @@ describe('node-fetch', () => { }); it('should allow deleting header', function() { - url = `${base}/cookie`; + url = `${base}cookie`; return fetch(url).then(res => { res.headers.delete('set-cookie'); expect(res.headers.get('set-cookie')).to.be.null; @@ -1199,7 +1199,7 @@ describe('node-fetch', () => { }); it('should send request with connection keep-alive if agent is provided', function() { - url = `${base}/inspect`; + url = `${base}inspect`; opts = { agent: new http.Agent({ keepAlive: true @@ -1286,7 +1286,7 @@ describe('node-fetch', () => { }); it('should support fetch with Request instance', function() { - url = `${base}/hello`; + url = `${base}hello`; const req = new Request(url); return fetch(req).then(res => { expect(res.url).to.equal(url); @@ -1296,7 +1296,7 @@ describe('node-fetch', () => { }); it('should support wrapping Request instance', function() { - url = `${base}/hello`; + url = `${base}hello`; const form = new FormData(); form.append('a', '1'); @@ -1321,7 +1321,7 @@ describe('node-fetch', () => { }); it('should support overwrite Request instance', function() { - url = `${base}/inspect`; + url = `${base}inspect`; const req = new Request(url, { method: 'POST' , headers: { From 9d3cc52601d170aaedc86d9ad7a5c2b7f5bcbd97 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sat, 8 Oct 2016 18:51:34 -0700 Subject: [PATCH 008/223] Body: store fewer things in the class Incorporates some changes from #140, by Gabriel Wicke . --- src/body.js | 184 +++++++++++++++++++++++++--------------------------- 1 file changed, 88 insertions(+), 96 deletions(-) diff --git a/src/body.js b/src/body.js index ab76421dd..962232f63 100644 --- a/src/body.js +++ b/src/body.js @@ -11,11 +11,7 @@ import {PassThrough} from 'stream'; import FetchError from './fetch-error.js'; const DISTURBED = Symbol('disturbed'); -const BYTES = Symbol('bytes'); -const RAW = Symbol('raw'); -const ABORT = Symbol('abort'); -const CONVERT = Symbol('convert'); -const DECODE = Symbol('decode'); +const CONSUME_BODY = Symbol('consumeBody'); /** * Body class @@ -32,10 +28,7 @@ export default class Body { this.body = body; this[DISTURBED] = false; this.size = size; - this[BYTES] = 0; this.timeout = timeout; - this[RAW] = []; - this[ABORT] = false; } get bodyUsed() { @@ -53,7 +46,7 @@ export default class Body { return Body.Promise.resolve({}); } - return this[DECODE]().then(buffer => JSON.parse(buffer.toString())); + return this[CONSUME_BODY]().then(buffer => JSON.parse(buffer.toString())); } /** @@ -62,7 +55,7 @@ export default class Body { * @return Promise */ text() { - return this[DECODE]().then(buffer => buffer.toString()); + return this[CONSUME_BODY]().then(buffer => buffer.toString()); } /** @@ -71,7 +64,7 @@ export default class Body { * @return Promise */ buffer() { - return this[DECODE](); + return this[CONSUME_BODY](); } /** @@ -79,144 +72,143 @@ export default class Body { * * @return Promise */ - [DECODE]() { + [CONSUME_BODY]() { if (this[DISTURBED]) { return Body.Promise.reject(new Error(`body used already for: ${this.url}`)); } this[DISTURBED] = true; - this[BYTES] = 0; - this[ABORT] = false; - this[RAW] = []; - return new Body.Promise((resolve, reject) => { - let resTimeout; + // body is string + if (typeof this.body === 'string') { + return Body.Promise.resolve(convertBody([new Buffer(this.body)], this.headers)); + } - // body is string - if (typeof this.body === 'string') { - this[BYTES] = this.body.length; - this[RAW] = [new Buffer(this.body)]; - return resolve(this[CONVERT]()); - } + // body is buffer + if (Buffer.isBuffer(this.body)) { + return Body.Promise.resolve(convertBody([this.body], this.headers)); + } - // body is buffer - if (this.body instanceof Buffer) { - this[BYTES] = this.body.length; - this[RAW] = [this.body]; - return resolve(this[CONVERT]()); - } + // body is stream + // get ready to actually consume the body + let accum = []; + let accumBytes = 0; + let abort = false; + + return new Body.Promise((resolve, reject) => { + let resTimeout; // allow timeout on slow response body if (this.timeout) { resTimeout = setTimeout(() => { - this[ABORT] = true; - reject(new FetchError('response timeout at ' + this.url + ' over limit: ' + this.timeout, 'body-timeout')); + abort = true; + reject(new FetchError(`Response timeout while trying to fetch ${this.url} (over ${this.timeout}ms)`, 'body-timeout')); }, this.timeout); } // handle stream error, such as incorrect content-encoding this.body.on('error', err => { - reject(new FetchError('invalid response body at: ' + this.url + ' reason: ' + err.message, 'system', err)); + reject(new FetchError(`Invalid response body while trying to fetch ${this.url}: ${err.message}`, 'system', err)); }); - // body is stream this.body.on('data', chunk => { - if (this[ABORT] || chunk === null) { + if (abort || chunk === null) { return; } - if (this.size && this[BYTES] + chunk.length > this.size) { - this[ABORT] = true; + if (this.size && accumBytes + chunk.length > this.size) { + abort = true; reject(new FetchError(`content size at ${this.url} over limit: ${this.size}`, 'max-size')); return; } - this[BYTES] += chunk.length; - this[RAW].push(chunk); + accumBytes += chunk.length; + accum.push(chunk); }); this.body.on('end', () => { - if (this[ABORT]) { + if (abort) { return; } clearTimeout(resTimeout); - resolve(this[CONVERT]()); + resolve(convertBody(accum, this.headers)); }); }); } - /** - * Detect buffer encoding and convert to target encoding - * ref: http://www.w3.org/TR/2011/WD-html5-20110113/parsing.html#determining-the-character-encoding - * - * @param String encoding Target encoding - * @return String - */ - [CONVERT](encoding = 'utf-8') { - const ct = this.headers.get('content-type'); - let charset = 'utf-8'; - let res, str; - - // header - if (ct) { - // skip encoding detection altogether if not html/xml/plain text - if (!/text\/html|text\/plain|\+xml|\/xml/i.test(ct)) { - return Buffer.concat(this[RAW]); - } +} - res = /charset=([^;]*)/i.exec(ct); +/** + * Detect buffer encoding and convert to target encoding + * ref: http://www.w3.org/TR/2011/WD-html5-20110113/parsing.html#determining-the-character-encoding + * + * @param Array arrayOfBuffers Array of buffers + * @param String encoding Target encoding + * @return String + */ +function convertBody(arrayOfBuffers, headers) { + const ct = headers.get('content-type'); + let charset = 'utf-8'; + let res, str; + + // header + if (ct) { + // skip encoding detection altogether if not html/xml/plain text + if (!/text\/html|text\/plain|\+xml|\/xml/i.test(ct)) { + return Buffer.concat(arrayOfBuffers); } - // no charset in content type, peek at response body for at most 1024 bytes - if (!res && this[RAW].length > 0) { - for (let i = 0; i < this[RAW].length; i++) { - str += this[RAW][i].toString() - if (str.length > 1024) { - break; - } + res = /charset=([^;]*)/i.exec(ct); + } + + // no charset in content type, peek at response body for at most 1024 bytes + if (!res && arrayOfBuffers.length > 0) { + for (let i = 0; i < arrayOfBuffers.length; i++) { + str += arrayOfBuffers[i].toString() + if (str.length > 1024) { + break; } - str = str.substr(0, 1024); } + str = str.substr(0, 1024); + } - // html5 - if (!res && str) { - res = / Date: Sat, 8 Oct 2016 19:40:56 -0700 Subject: [PATCH 009/223] Improve Body spec compliance when body is null --- src/body.js | 5 +++++ src/response.js | 2 +- test/test.js | 8 ++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/body.js b/src/body.js index 962232f63..349657eb8 100644 --- a/src/body.js +++ b/src/body.js @@ -79,6 +79,11 @@ export default class Body { this[DISTURBED] = true; + // body is null + if (!this.body) { + return Body.Promise.resolve(new Buffer(0)); + } + // body is string if (typeof this.body === 'string') { return Body.Promise.resolve(convertBody([new Buffer(this.body)], this.headers)); diff --git a/src/response.js b/src/response.js index fb53a783c..bc3175b06 100644 --- a/src/response.js +++ b/src/response.js @@ -17,7 +17,7 @@ import Body, { clone } from './body'; * @return Void */ export default class Response extends Body { - constructor(body, opts = {}) { + constructor(body = null, opts = {}) { super(body, opts); this.url = opts.url; diff --git a/test/test.js b/test/test.js index 180953d82..34880c2c7 100644 --- a/test/test.js +++ b/test/test.js @@ -1427,6 +1427,14 @@ describe('node-fetch', () => { }); }); + it('should default to null as body', function() { + const res = new Response(); + expect(res.body).to.equal(null); + return res.text().then(result => { + expect(result).to.equal(''); + }); + }); + it('should default to 200 as status code', function() { const res = new Response(null); expect(res.status).to.equal(200); From c3a121a36030a6c23732cb4f9897248b0f318172 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sat, 8 Oct 2016 19:41:45 -0700 Subject: [PATCH 010/223] Add support for Body#arrayBuffer --- package.json | 1 + src/body.js | 10 ++++++++++ test/test.js | 16 +++++++++++++++- 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 155270426..af51757f1 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ }, "dependencies": { "babel-runtime": "^6.11.6", + "buffer-to-arraybuffer": "0.0.4", "encoding": "^0.1.11", "is-stream": "^1.0.1" }, diff --git a/src/body.js b/src/body.js index 349657eb8..bd30eb637 100644 --- a/src/body.js +++ b/src/body.js @@ -7,6 +7,7 @@ import {convert} from 'encoding'; import bodyStream from 'is-stream'; +import toArrayBuffer from 'buffer-to-arraybuffer'; import {PassThrough} from 'stream'; import FetchError from './fetch-error.js'; @@ -35,6 +36,15 @@ export default class Body { return this[DISTURBED]; } + /** + * Decode response as ArrayBuffer + * + * @return Promise + */ + arrayBuffer() { + return this[CONSUME_BODY]().then(buf => toArrayBuffer(buf)); + } + /** * Decode response as json * diff --git a/test/test.js b/test/test.js index 34880c2c7..679a414b9 100644 --- a/test/test.js +++ b/test/test.js @@ -1451,6 +1451,19 @@ describe('node-fetch', () => { expect(req.headers.get('a')).to.equal('1'); }); + it('should support arrayBuffer() method in Request constructor', function() { + url = base; + var req = new Request(url, { + body: 'a=1' + }); + expect(req.url).to.equal(url); + return req.arrayBuffer().then(function(result) { + expect(result).to.be.an.instanceOf(ArrayBuffer); + const str = String.fromCharCode.apply(null, new Uint8Array(result)); + expect(str).to.equal('a=1'); + }); + }); + it('should support text() method in Request constructor', function() { url = base; const req = new Request(url, { @@ -1524,8 +1537,9 @@ describe('node-fetch', () => { }); }); - it('should support text(), json() and buffer() method in Body constructor', function() { + it('should support arrayBuffer(), text(), json() and buffer() method in Body constructor', function() { const body = new Body('a=1'); + expect(body).to.have.property('arrayBuffer'); expect(body).to.have.property('text'); expect(body).to.have.property('json'); expect(body).to.have.property('buffer'); From 0f65af3fd86a9015360bafec2cb7a928e0bf19c0 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Mon, 10 Oct 2016 12:05:02 -0700 Subject: [PATCH 011/223] Split Headers iterable test into four --- package.json | 1 + test/test.js | 99 +++++++++++++++++++++++++++------------------------- 2 files changed, 53 insertions(+), 47 deletions(-) diff --git a/package.json b/package.json index af51757f1..18109f8e1 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "bluebird": "^3.3.4", "chai": "^3.5.0", "chai-as-promised": "^5.2.0", + "chai-iterator": "^1.1.1", "coveralls": "^2.11.2", "cross-env": "2.0.1", "form-data": ">=1.0.0", diff --git a/test/test.js b/test/test.js index 679a414b9..753290c67 100644 --- a/test/test.js +++ b/test/test.js @@ -1,7 +1,8 @@ // test tools import chai from 'chai'; -import cap from 'chai-as-promised'; +import chaiPromised from 'chai-as-promised'; +import chaiIterator from 'chai-iterator'; import bluebird from 'bluebird'; import then from 'promise'; import {spawn} from 'child_process'; @@ -11,7 +12,8 @@ import FormData from 'form-data'; import * as http from 'http'; import * as fs from 'fs'; -chai.use(cap); +chai.use(chaiPromised); +chai.use(chaiIterator); const expect = chai.expect; import TestServer from './server'; @@ -1108,7 +1110,7 @@ describe('node-fetch', () => { }); }); - it('should allow iterating through all headers', function() { + it('should allow iterating through all headers with forEach', function() { const headers = new Headers({ a: 1 , b: [2, 3] @@ -1130,63 +1132,66 @@ describe('node-fetch', () => { expect(result).to.deep.equal(expected); }); - it('should allow iterating through all headers', function() { + it('should allow iterating through all headers with for-of loop', function() { const headers = new Headers({ - a: 1 - , b: [2, 3] - , c: [4] + a: '1' + , b: '2' + , c: '4' }); - expect(headers).to.have.property(Symbol.iterator); - expect(headers).to.have.property('keys'); - expect(headers).to.have.property('values'); - expect(headers).to.have.property('entries'); + headers.append('b', '3'); + expect(headers).to.be.iterable; - let result, expected; - - result = []; - for (let [key, val] of headers) { - result.push([key, val]); + const result = []; + for (let pair of headers) { + result.push(pair); } - - expected = [ + expect(result).to.deep.equal([ ["a", "1"] , ["b", "2"] , ["b", "3"] , ["c", "4"] - ]; - expect(result).to.deep.equal(expected); + ]); + }); - result = []; - for (let [key, val] of headers.entries()) { - result.push([key, val]); - } - expect(result).to.deep.equal(expected); + it('should allow iterating through all headers with entries()', function() { + const headers = new Headers({ + a: '1' + , b: '2' + , c: '4' + }); + headers.append('b', '3'); - result = []; - for (let key of headers.keys()) { - result.push(key); - } + expect(headers.entries()).to.be.iterable + .and.to.deep.iterate.over([ + ["a", "1"] + , ["b", "2"] + , ["b", "3"] + , ["c", "4"] + ]); + }); - expected = [ - "a" - , "b" - , "b" - , "c" - ]; - expect(result).to.deep.equal(expected); + it('should allow iterating through all headers with keys()', function() { + const headers = new Headers({ + a: '1' + , b: '2' + , c: '4' + }); + headers.append('b', '3'); - result = []; - for (let key of headers.values()) { - result.push(key); - } + expect(headers.keys()).to.be.iterable + .and.to.iterate.over(['a', 'b', 'b', 'c']); + }); - expected = [ - "1" - , "2" - , "3" - , "4" - ]; - expect(result).to.deep.equal(expected); + it('should allow iterating through all headers with values()', function() { + const headers = new Headers({ + a: '1' + , b: '2' + , c: '4' + }); + headers.append('b', '3'); + + expect(headers.values()).to.be.iterable + .and.to.iterate.over(['1', '2', '3', '4']); }); it('should allow deleting header', function() { From 4d81cb4877cb047c46abd2fdbd3086d9ea16b8f9 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Mon, 10 Oct 2016 12:41:13 -0700 Subject: [PATCH 012/223] Test @@toStringTag getter of all classes --- test/test.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/test.js b/test/test.js index 753290c67..444ca1f80 100644 --- a/test/test.js +++ b/test/test.js @@ -171,6 +171,11 @@ describe('node-fetch', () => { }); }); + it('should support proper toString output for Headers instance', function() { + const headers = new Headers(); + expect(headers.toString()).to.equal('[object Headers]'); + }); + it('should accept custom host header', function() { url = `${base}inspect`; opts = { @@ -1300,6 +1305,11 @@ describe('node-fetch', () => { }); }); + it('should support proper toString output for Request instance', function() { + const req = new Request(base); + expect(req.toString()).to.equal('[object Request]'); + }); + it('should support wrapping Request instance', function() { url = `${base}hello`; @@ -1355,6 +1365,11 @@ describe('node-fetch', () => { }); }); + it('should support proper toString output for Response instance', function() { + const res = new Response(); + expect(res.toString()).to.equal('[object Response]'); + }); + it('should support parsing headers in Response constructor', function() { const res = new Response(null, { headers: { From c2c6550e549e34ea0ce65a9fac9b7d1c80a61498 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Wed, 12 Oct 2016 18:16:46 -0700 Subject: [PATCH 013/223] Use loose mode when compiling Babel --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 18109f8e1..c408a6e83 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ }, "babel": { "presets": [ - "es2015" + ["es2015", {"loose": true}] ], "plugins": [ "transform-runtime" From 67326e3873ba1ee3dff79871f256fdf77dff0ab2 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Wed, 12 Oct 2016 20:51:19 -0700 Subject: [PATCH 014/223] Condense class toString tests --- test/test.js | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/test/test.js b/test/test.js index 444ca1f80..d78116097 100644 --- a/test/test.js +++ b/test/test.js @@ -74,6 +74,12 @@ describe('node-fetch', () => { expect(fetch.Request).to.equal(Request); }); + it('should support proper toString output for Headers, Response and Request objects', function() { + expect(new Headers().toString()).to.equal('[object Headers]'); + expect(new Response().toString()).to.equal('[object Response]'); + expect(new Request(base).toString()).to.equal('[object Request]'); + }); + it('should reject with error if url is protocol relative', function() { url = '//example.com/'; return expect(fetch(url)).to.eventually.be.rejectedWith(Error); @@ -171,11 +177,6 @@ describe('node-fetch', () => { }); }); - it('should support proper toString output for Headers instance', function() { - const headers = new Headers(); - expect(headers.toString()).to.equal('[object Headers]'); - }); - it('should accept custom host header', function() { url = `${base}inspect`; opts = { @@ -1305,11 +1306,6 @@ describe('node-fetch', () => { }); }); - it('should support proper toString output for Request instance', function() { - const req = new Request(base); - expect(req.toString()).to.equal('[object Request]'); - }); - it('should support wrapping Request instance', function() { url = `${base}hello`; @@ -1365,11 +1361,6 @@ describe('node-fetch', () => { }); }); - it('should support proper toString output for Response instance', function() { - const res = new Response(); - expect(res.toString()).to.equal('[object Response]'); - }); - it('should support parsing headers in Response constructor', function() { const res = new Response(null, { headers: { From ea111626e9ae3ea112bf7bc8fec4b28a905ddf37 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Thu, 13 Oct 2016 00:32:52 -0700 Subject: [PATCH 015/223] Switch to Codecov Fixes #186. --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index c408a6e83..bdc513f90 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "prepublish": "npm run build", "test": "mocha --compilers js:babel-polyfill --compilers js:babel-register test/test.js", "report": "cross-env BABEL_ENV=test nyc --reporter lcov --reporter text mocha -R spec test/test.js", - "coverage": "cross-env BABEL_ENV=test nyc --reporter lcovonly mocha -R spec test/test.js && cat ./coverage/lcov.info | coveralls" + "coverage": "cross-env BABEL_ENV=test nyc --reporter json --reporter text mocha -R spec test/test.js && codecov -f coverage/coverage-final.json" }, "repository": { "type": "git", @@ -36,7 +36,7 @@ "chai": "^3.5.0", "chai-as-promised": "^5.2.0", "chai-iterator": "^1.1.1", - "coveralls": "^2.11.2", + "codecov": "^1.0.1", "cross-env": "2.0.1", "form-data": ">=1.0.0", "mocha": "^2.1.0", From b9b0341db7375f07114fb93283676d94647c0932 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Thu, 13 Oct 2016 00:40:36 -0700 Subject: [PATCH 016/223] Add Codecov badges and settings --- .codecov.yml | 5 +++++ README.md | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 .codecov.yml diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 000000000..dcdc85aaf --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,5 @@ +parsers: + javascript: + enable_partials: yes +codecov: + branch: v2 diff --git a/README.md b/README.md index 96d69f68f..499324fae 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ node-fetch [![npm version][npm-image]][npm-url] [![build status][travis-image]][travis-url] -[![coverage status][coveralls-image]][coveralls-url] +[![coverage status][codecov-image]][codecov-url] A light-weight module that brings `window.fetch` to Node.js @@ -206,5 +206,5 @@ Thanks to [github/fetch](https://github.com/github/fetch) for providing a solid [npm-url]: https://www.npmjs.com/package/node-fetch [travis-image]: https://img.shields.io/travis/bitinn/node-fetch.svg?style=flat-square [travis-url]: https://travis-ci.org/bitinn/node-fetch -[coveralls-image]: https://img.shields.io/coveralls/bitinn/node-fetch.svg?style=flat-square -[coveralls-url]: https://coveralls.io/r/bitinn/node-fetch +[coveralls-image]: https://img.shields.io/codecov/c/github/bitinn/node-fetch.svg?style=flat-square +[coveralls-url]: https://codecov.io/gh/bitinn/node-fetch From ba8c392965d718e16fffb3f7a2777b64e352f2d1 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Thu, 13 Oct 2016 00:42:26 -0700 Subject: [PATCH 017/223] Cache node_modules in Travis [ci skip] --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a1358b0d9..b6c20826d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,4 +9,7 @@ env: before_script: - 'if [ "$FORMDATA_VERSION" ]; then npm install form-data@^$FORMDATA_VERSION; fi' before_install: npm install -g npm -script: npm run coverage \ No newline at end of file +script: npm run coverage +cache: + directories: + - node_modules From a914cca57713914d3d76357883d2bd0f957e473e Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Thu, 13 Oct 2016 00:44:31 -0700 Subject: [PATCH 018/223] Remove all information related to Coveralls --- .gitignore | 3 --- README.md | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 99c8c2d57..f212ad965 100644 --- a/.gitignore +++ b/.gitignore @@ -31,8 +31,5 @@ node_modules # OS files .DS_Store -# Coveralls token files -.coveralls.yml - # Babel-compiled files lib diff --git a/README.md b/README.md index 499324fae..0bfb38755 100644 --- a/README.md +++ b/README.md @@ -206,5 +206,5 @@ Thanks to [github/fetch](https://github.com/github/fetch) for providing a solid [npm-url]: https://www.npmjs.com/package/node-fetch [travis-image]: https://img.shields.io/travis/bitinn/node-fetch.svg?style=flat-square [travis-url]: https://travis-ci.org/bitinn/node-fetch -[coveralls-image]: https://img.shields.io/codecov/c/github/bitinn/node-fetch.svg?style=flat-square -[coveralls-url]: https://codecov.io/gh/bitinn/node-fetch +[codecov-image]: https://img.shields.io/codecov/c/github/bitinn/node-fetch.svg?style=flat-square +[codecov-url]: https://codecov.io/gh/bitinn/node-fetch From 82c1e781847789d14ea67ad989c167c6eaa52b9d Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Mon, 10 Oct 2016 13:49:12 -0700 Subject: [PATCH 019/223] Allow constructing Headers with an Array --- src/headers.js | 13 ++++++++++++- test/test.js | 15 +++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/headers.js b/src/headers.js index ad2e9e4aa..88f25f667 100644 --- a/src/headers.js +++ b/src/headers.js @@ -17,7 +17,18 @@ export default class Headers { constructor(headers) { this[MAP] = {}; - // Headers + if (Array.isArray(headers)) { + // array of tuples + for (let el of headers) { + if (!Array.isArray(el) || el.length !== 2) { + throw new TypeError('Header pairs must contain exactly two items'); + } + this.append(el[0], el[1]); + } + return; + } + + // Headers if (headers instanceof Headers) { headers = headers.raw(); } diff --git a/test/test.js b/test/test.js index d78116097..bbf71cfda 100644 --- a/test/test.js +++ b/test/test.js @@ -1296,6 +1296,21 @@ describe('node-fetch', () => { expect(h3Raw['b']).to.include('1'); }); + it('should accept headers as an array of tuples', function() { + const headers = new Headers([ + ['a', '1'], + ['b', '2'], + ['a', '3'] + ]); + expect(headers.getAll('a')).to.deep.equal(['1', '3']); + expect(headers.getAll('b')).to.deep.equal(['2']); + }); + + it('should throw a TypeError if non-tuple exists in a headers initializer', function() { + expect(() => new Headers([ ['b', '2', 'huh?'] ])).to.throw(TypeError); + expect(() => new Headers([ 'b2' ])).to.throw(TypeError); + }); + it('should support fetch with Request instance', function() { url = `${base}hello`; const req = new Request(url); From ba226399d4b201230397ddb25b3f42725e7484b9 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Mon, 10 Oct 2016 14:12:57 -0700 Subject: [PATCH 020/223] Construct Headers object in a spec-compliant fashion --- src/index.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/index.js b/src/index.js index 07963a53f..f27b5a3bf 100644 --- a/src/index.js +++ b/src/index.js @@ -152,7 +152,16 @@ function fetch(url, opts) { } // normalize location header for manual redirect mode - const headers = new Headers(res.headers); + const headers = new Headers(); + for (const name of Object.keys(res.headers)) { + if (Array.isArray(res.headers[name])) { + for (const val of res.headers[name]) { + headers.append(name, val); + } + } else { + headers.append(name, res.headers[name]); + } + } if (options.redirect === 'manual' && headers.has('location')) { headers.set('location', resolve_url(options.url, headers.get('location'))); } From fba873d3feea4d1422885ac1c8a1d2d9e83882a3 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Mon, 10 Oct 2016 15:04:16 -0700 Subject: [PATCH 021/223] Make sure to coerce header values to string --- src/headers.js | 44 +++++++++++++++++++++----------------------- test/test.js | 40 +++++++++++++++++++--------------------- 2 files changed, 40 insertions(+), 44 deletions(-) diff --git a/src/headers.js b/src/headers.js index 88f25f667..279077266 100644 --- a/src/headers.js +++ b/src/headers.js @@ -17,7 +17,15 @@ export default class Headers { constructor(headers) { this[MAP] = {}; - if (Array.isArray(headers)) { + // Headers + if (headers instanceof Headers) { + let init = headers.raw(); + for (let name of Object.keys(init)) { + for (let value of init[name]) { + this.append(name, value); + } + } + } else if (Array.isArray(headers)) { // array of tuples for (let el of headers) { if (!Array.isArray(el) || el.length !== 2) { @@ -25,28 +33,12 @@ export default class Headers { } this.append(el[0], el[1]); } - return; - } - - // Headers - if (headers instanceof Headers) { - headers = headers.raw(); - } - - // plain object - for (const prop in headers) { - if (!headers.hasOwnProperty(prop)) { - continue; - } - - if (typeof headers[prop] === 'string') { - this.set(prop, headers[prop]); - } else if (typeof headers[prop] === 'number' && !isNaN(headers[prop])) { - this.set(prop, headers[prop].toString()); - } else if (headers[prop] instanceof Array) { - headers[prop].forEach(item => { - this.append(prop, item.toString()); - }); + } else if (typeof headers === 'object') { + // plain object + for (const prop of Object.keys(headers)) { + // We don't worry about converting prop to ByteString here as append() + // will handle it. + this.append(prop, headers[prop]); } } } @@ -99,6 +91,9 @@ export default class Headers { * @return Void */ set(name, value) { + name += ''; + value += ''; + this[MAP][name.toLowerCase()] = [value]; } @@ -110,6 +105,9 @@ export default class Headers { * @return Void */ append(name, value) { + name += ''; + value += ''; + if (!this.has(name)) { this.set(name, value); return; diff --git a/test/test.js b/test/test.js index bbf71cfda..3995857f7 100644 --- a/test/test.js +++ b/test/test.js @@ -1131,8 +1131,7 @@ describe('node-fetch', () => { const expected = [ ["a", "1"] - , ["b", "2"] - , ["b", "3"] + , ["b", "2,3"] , ["c", "4"] ]; expect(result).to.deep.equal(expected); @@ -1224,20 +1223,18 @@ describe('node-fetch', () => { }); it('should ignore unsupported attributes while reading headers', function() { - const FakeHeader = () => {}; - // prototypes are ignored + const FakeHeader = function () {}; + // prototypes are currently ignored + // This might change in the future: #181 FakeHeader.prototype.z = 'fake'; const res = new FakeHeader; - // valid res.a = 'string'; res.b = ['1','2']; res.c = ''; res.d = []; - // common mistakes, normalized res.e = 1; res.f = [1, 2]; - // invalid, ignored res.g = { a:1 }; res.h = undefined; res.i = null; @@ -1247,25 +1244,26 @@ describe('node-fetch', () => { res.m = new Buffer('test'); const h1 = new Headers(res); + h1.set('n', [1, 2]); + h1.append('n', ['3', 4]) + const h1Raw = h1.raw(); expect(h1Raw['a']).to.include('string'); - expect(h1Raw['b']).to.include('1'); - expect(h1Raw['b']).to.include('2'); + expect(h1Raw['b']).to.include('1,2'); expect(h1Raw['c']).to.include(''); - expect(h1Raw['d']).to.be.undefined; - + expect(h1Raw['d']).to.include(''); expect(h1Raw['e']).to.include('1'); - expect(h1Raw['f']).to.include('1'); - expect(h1Raw['f']).to.include('2'); - - expect(h1Raw['g']).to.be.undefined; - expect(h1Raw['h']).to.be.undefined; - expect(h1Raw['i']).to.be.undefined; - expect(h1Raw['j']).to.be.undefined; - expect(h1Raw['k']).to.be.undefined; - expect(h1Raw['l']).to.be.undefined; - expect(h1Raw['m']).to.be.undefined; + expect(h1Raw['f']).to.include('1,2'); + expect(h1Raw['g']).to.include('[object Object]'); + expect(h1Raw['h']).to.include('undefined'); + expect(h1Raw['i']).to.include('null'); + expect(h1Raw['j']).to.include('NaN'); + expect(h1Raw['k']).to.include('true'); + expect(h1Raw['l']).to.include('false'); + expect(h1Raw['m']).to.include('test'); + expect(h1Raw['n']).to.include('1,2'); + expect(h1Raw['n']).to.include('3,4'); expect(h1Raw['z']).to.be.undefined; }); From 2cafdcb5e449f02980ea54b74342ab41d77a1f76 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Mon, 10 Oct 2016 15:32:56 -0700 Subject: [PATCH 022/223] Validate headers --- src/common.js | 111 +++++++++++++++++++++++++++++++++++++++++++++++++ src/headers.js | 37 +++++++++++------ test/test.js | 16 +++++++ 3 files changed, 151 insertions(+), 13 deletions(-) create mode 100644 src/common.js diff --git a/src/common.js b/src/common.js new file mode 100644 index 000000000..f4908a3cb --- /dev/null +++ b/src/common.js @@ -0,0 +1,111 @@ +/** + * A set of utilities borrowed from Node.js' _http_common.js + */ + +/** + * Verifies that the given val is a valid HTTP token + * per the rules defined in RFC 7230 + * See https://tools.ietf.org/html/rfc7230#section-3.2.6 + * + * Allowed characters in an HTTP token: + * ^_`a-z 94-122 + * A-Z 65-90 + * - 45 + * 0-9 48-57 + * ! 33 + * #$%&' 35-39 + * *+ 42-43 + * . 46 + * | 124 + * ~ 126 + * + * This implementation of checkIsHttpToken() loops over the string instead of + * using a regular expression since the former is up to 180% faster with v8 4.9 + * depending on the string length (the shorter the string, the larger the + * performance difference) + * + * Additionally, checkIsHttpToken() is currently designed to be inlinable by v8, + * so take care when making changes to the implementation so that the source + * code size does not exceed v8's default max_inlined_source_size setting. + **/ +/* istanbul ignore next */ +function isValidTokenChar(ch) { + if (ch >= 94 && ch <= 122) + return true; + if (ch >= 65 && ch <= 90) + return true; + if (ch === 45) + return true; + if (ch >= 48 && ch <= 57) + return true; + if (ch === 34 || ch === 40 || ch === 41 || ch === 44) + return false; + if (ch >= 33 && ch <= 46) + return true; + if (ch === 124 || ch === 126) + return true; + return false; +} +/* istanbul ignore next */ +function checkIsHttpToken(val) { + if (typeof val !== 'string' || val.length === 0) + return false; + if (!isValidTokenChar(val.charCodeAt(0))) + return false; + const len = val.length; + if (len > 1) { + if (!isValidTokenChar(val.charCodeAt(1))) + return false; + if (len > 2) { + if (!isValidTokenChar(val.charCodeAt(2))) + return false; + if (len > 3) { + if (!isValidTokenChar(val.charCodeAt(3))) + return false; + for (var i = 4; i < len; i++) { + if (!isValidTokenChar(val.charCodeAt(i))) + return false; + } + } + } + } + return true; +} +exports._checkIsHttpToken = checkIsHttpToken; + +/** + * True if val contains an invalid field-vchar + * field-value = *( field-content / obs-fold ) + * field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ] + * field-vchar = VCHAR / obs-text + * + * checkInvalidHeaderChar() is currently designed to be inlinable by v8, + * so take care when making changes to the implementation so that the source + * code size does not exceed v8's default max_inlined_source_size setting. + **/ +/* istanbul ignore next */ +function checkInvalidHeaderChar(val) { + val += ''; + if (val.length < 1) + return false; + var c = val.charCodeAt(0); + if ((c <= 31 && c !== 9) || c > 255 || c === 127) + return true; + if (val.length < 2) + return false; + c = val.charCodeAt(1); + if ((c <= 31 && c !== 9) || c > 255 || c === 127) + return true; + if (val.length < 3) + return false; + c = val.charCodeAt(2); + if ((c <= 31 && c !== 9) || c > 255 || c === 127) + return true; + for (var i = 3; i < val.length; ++i) { + c = val.charCodeAt(i); + if ((c <= 31 && c !== 9) || c > 255 || c === 127) + return true; + } + return false; +} +exports._checkInvalidHeaderChar = checkInvalidHeaderChar; diff --git a/src/headers.js b/src/headers.js index 279077266..a7eff5574 100644 --- a/src/headers.js +++ b/src/headers.js @@ -5,8 +5,25 @@ * Headers class offers convenient helpers */ -export const MAP = Symbol('map'); +import { _checkIsHttpToken, _checkInvalidHeaderChar } from './common.js'; + +function sanitizeName(name) { + name += ''; + if (!_checkIsHttpToken(name)) { + throw new TypeError(`${name} is not a legal HTTP header name`); + } + return name.toLowerCase(); +} + +function sanitizeValue(value) { + value += ''; + if (_checkInvalidHeaderChar(value)) { + throw new TypeError(`${value} is not a legal HTTP header value`); + } + return value; +} +export const MAP = Symbol('map'); export default class Headers { /** * Headers class @@ -50,7 +67,7 @@ export default class Headers { * @return Mixed */ get(name) { - const list = this[MAP][name.toLowerCase()]; + const list = this[MAP][sanitizeName(name)]; return list ? list[0] : null; } @@ -65,7 +82,7 @@ export default class Headers { return []; } - return this[MAP][name.toLowerCase()]; + return this[MAP][sanitizeName(name)]; } /** @@ -91,10 +108,7 @@ export default class Headers { * @return Void */ set(name, value) { - name += ''; - value += ''; - - this[MAP][name.toLowerCase()] = [value]; + this[MAP][sanitizeName(name)] = [sanitizeValue(value)]; } /** @@ -105,15 +119,12 @@ export default class Headers { * @return Void */ append(name, value) { - name += ''; - value += ''; - if (!this.has(name)) { this.set(name, value); return; } - this[MAP][name.toLowerCase()].push(value); + this[MAP][sanitizeName(name)].push(sanitizeValue(value)); } /** @@ -123,7 +134,7 @@ export default class Headers { * @return Boolean */ has(name) { - return this[MAP].hasOwnProperty(name.toLowerCase()); + return this[MAP].hasOwnProperty(sanitizeName(name)); } /** @@ -133,7 +144,7 @@ export default class Headers { * @return Void */ delete(name) { - delete this[MAP][name.toLowerCase()]; + delete this[MAP][sanitizeName(name)]; }; /** diff --git a/test/test.js b/test/test.js index 3995857f7..1cad47635 100644 --- a/test/test.js +++ b/test/test.js @@ -1208,6 +1208,22 @@ describe('node-fetch', () => { }); }); + it('should reject illegal header', function() { + const headers = new Headers(); + expect(() => new Headers({ 'He y': 'ok' })).to.throw(TypeError); + expect(() => new Headers({ 'Hé-y': 'ok' })).to.throw(TypeError); + expect(() => new Headers({ 'He-y': 'ăk' })).to.throw(TypeError); + expect(() => headers.append('Hé-y', 'ok')) .to.throw(TypeError); + expect(() => headers.delete('Hé-y')) .to.throw(TypeError); + expect(() => headers.get('Hé-y')) .to.throw(TypeError); + expect(() => headers.getAll('Hé-y')) .to.throw(TypeError); + expect(() => headers.has('Hé-y')) .to.throw(TypeError); + expect(() => headers.set('Hé-y', 'ok')) .to.throw(TypeError); + + // 'o k' is valid value but invalid name + new Headers({ 'He-y': 'o k' }); + }); + it('should send request with connection keep-alive if agent is provided', function() { url = `${base}inspect`; opts = { From 2a7ef63bc410fea58776bce1f9a11729f435c351 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Mon, 10 Oct 2016 18:31:53 -0700 Subject: [PATCH 023/223] Add FOLLOW_SPEC mode --- src/headers.js | 46 +++++++++++++----- src/index.js | 10 ++++ test/test.js | 125 ++++++++++++++++++++++++++++++++----------------- 3 files changed, 127 insertions(+), 54 deletions(-) diff --git a/src/headers.js b/src/headers.js index a7eff5574..9ad129e8f 100644 --- a/src/headers.js +++ b/src/headers.js @@ -24,6 +24,7 @@ function sanitizeValue(value) { } export const MAP = Symbol('map'); +const FOLLOW_SPEC = Symbol('followSpec'); export default class Headers { /** * Headers class @@ -33,6 +34,7 @@ export default class Headers { */ constructor(headers) { this[MAP] = {}; + this[FOLLOW_SPEC] = Headers.FOLLOW_SPEC; // Headers if (headers instanceof Headers) { @@ -68,7 +70,11 @@ export default class Headers { */ get(name) { const list = this[MAP][sanitizeName(name)]; - return list ? list[0] : null; + if (!list) { + return null; + } + + return this[FOLLOW_SPEC] ? list.join(',') : list[0]; } /** @@ -162,8 +168,12 @@ export default class Headers { * @return Iterator */ keys() { - const keys = []; - this.forEach((_, name) => keys.push(name)); + let keys = []; + if (this[FOLLOW_SPEC]) { + keys = Object.keys(this[MAP]).sort(); + } else { + this.forEach((_, name) => keys.push(name)); + }; return new Iterator(keys); } @@ -172,10 +182,16 @@ export default class Headers { * * @return Iterator */ - values() { - const values = []; - this.forEach(value => values.push(value)); - return new Iterator(values); + *values() { + if (this[FOLLOW_SPEC]) { + for (const name of this.keys()) { + yield this.get(name); + } + } else { + const values = []; + this.forEach(value => values.push(value)); + yield* new Iterator(values); + } } /** @@ -183,10 +199,16 @@ export default class Headers { * * @return Iterator */ - entries() { - const entries = []; - this.forEach((value, name) => entries.push([name, value])); - return new Iterator(entries); + *entries() { + if (this[FOLLOW_SPEC]) { + for (const name of this.keys()) { + yield [name, this.get(name)]; + } + } else { + const entries = []; + this.forEach((value, name) => entries.push([name, value])); + yield* new Iterator(entries); + } } /** @@ -208,6 +230,8 @@ export default class Headers { } } +Headers.FOLLOW_SPEC = false; + const ITEMS = Symbol('items'); class Iterator { constructor(items) { diff --git a/src/index.js b/src/index.js index f27b5a3bf..6a889fddf 100644 --- a/src/index.js +++ b/src/index.js @@ -32,6 +32,7 @@ function fetch(url, opts) { } Body.Promise = fetch.Promise; + Headers.FOLLOW_SPEC = fetch.FOLLOW_SPEC; // wrap http.request into fetch return new fetch.Promise((resolve, reject) => { @@ -258,6 +259,15 @@ fetch.isRedirect = code => code === 301 || code === 302 || code === 303 || code // expose Promise fetch.Promise = global.Promise; +/** + * Option to make newly constructed Headers objects conformant to the + * **latest** version of the Fetch Standard. Note, that most other + * implementations of fetch() have not yet been updated to the latest + * version, so enabling this option almost certainly breaks any isomorphic + * attempt. Also, changing this variable will only affect new Headers + * objects; existing objects are not affected. + */ +fetch.FOLLOW_SPEC = false; fetch.Response = Response; fetch.Headers = Headers; fetch.Request = Request; diff --git a/test/test.js b/test/test.js index 1cad47635..5f61eb862 100644 --- a/test/test.js +++ b/test/test.js @@ -28,18 +28,26 @@ import FetchError from '../src/fetch-error.js'; // test with native promise on node 0.11, and bluebird for node 0.10 fetch.Promise = fetch.Promise || bluebird; -let url, opts, local, base; +const local = new TestServer(); +const base = `http://${local.hostname}:${local.port}/`; +let url, opts; -describe('node-fetch', () => { +before(done => { + local.start(done); +}); - before(done => { - local = new TestServer(); - base = `http://${local.hostname}:${local.port}/`; - local.start(done); - }); +after(done => { + local.stop(done); +}); - after(done => { - local.stop(done); +(runner => { + runner(false); + runner(true); +})(defaultFollowSpec => { + +describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { + before(() => { + fetch.FOLLOW_SPEC = Headers.FOLLOW_SPEC = defaultFollowSpec; }); it('should return a promise', function() { @@ -1109,8 +1117,9 @@ describe('node-fetch', () => { it('should allow get all responses of a header', function() { url = `${base}cookie`; return fetch(url).then(res => { - expect(res.headers.get('set-cookie')).to.equal('a=1'); - expect(res.headers.get('Set-Cookie')).to.equal('a=1'); + const expected = fetch.FOLLOW_SPEC ? 'a=1,b=1' : 'a=1'; + expect(res.headers.get('set-cookie')).to.equal(expected); + expect(res.headers.get('Set-Cookie')).to.equal(expected); expect(res.headers.getAll('set-cookie')).to.deep.equal(['a=1', 'b=1']); expect(res.headers.getAll('Set-Cookie')).to.deep.equal(['a=1', 'b=1']); }); @@ -1138,11 +1147,11 @@ describe('node-fetch', () => { }); it('should allow iterating through all headers with for-of loop', function() { - const headers = new Headers({ - a: '1' - , b: '2' - , c: '4' - }); + const headers = new Headers([ + ['b', '2'], + ['c', '4'], + ['a', '1'] + ]); headers.append('b', '3'); expect(headers).to.be.iterable; @@ -1150,53 +1159,81 @@ describe('node-fetch', () => { for (let pair of headers) { result.push(pair); } - expect(result).to.deep.equal([ - ["a", "1"] - , ["b", "2"] - , ["b", "3"] - , ["c", "4"] + expect(result).to.deep.equal(Headers.FOLLOW_SPEC ? [ + ['a', '1'], + ['b', '2,3'], + ['c', '4'] + ] : [ + ['b', '2'], + ['b', '3'], + ['c', '4'], + ['a', '1'], ]); }); it('should allow iterating through all headers with entries()', function() { - const headers = new Headers({ - a: '1' - , b: '2' - , c: '4' - }); + const headers = new Headers([ + ['b', '2'], + ['c', '4'], + ['a', '1'] + ]); headers.append('b', '3'); expect(headers.entries()).to.be.iterable - .and.to.deep.iterate.over([ - ["a", "1"] - , ["b", "2"] - , ["b", "3"] - , ["c", "4"] + .and.to.deep.iterate.over(Headers.FOLLOW_SPEC ? [ + ['a', '1'], + ['b', '2,3'], + ['c', '4'] + ] : [ + ['b', '2'], + ['b', '3'], + ['c', '4'], + ['a', '1'], ]); }); it('should allow iterating through all headers with keys()', function() { - const headers = new Headers({ - a: '1' - , b: '2' - , c: '4' - }); + const headers = new Headers([ + ['b', '2'], + ['c', '4'], + ['a', '1'] + ]); headers.append('b', '3'); expect(headers.keys()).to.be.iterable - .and.to.iterate.over(['a', 'b', 'b', 'c']); + .and.to.iterate.over(Headers.FOLLOW_SPEC ? ['a', 'b', 'c'] : ['b', 'b', 'c', 'a']); }); it('should allow iterating through all headers with values()', function() { - const headers = new Headers({ - a: '1' - , b: '2' - , c: '4' - }); + const headers = new Headers([ + ['b', '2'], + ['c', '4'], + ['a', '1'] + ]); headers.append('b', '3'); expect(headers.values()).to.be.iterable - .and.to.iterate.over(['1', '2', '3', '4']); + .and.to.iterate.over(Headers.FOLLOW_SPEC ? ['1', '2,3', '4'] : ['2', '3', '4', '1']); + }); + + it('should only apply FOLLOW_SPEC when it is requested', function () { + Headers.FOLLOW_SPEC = true; + + const src = [ + ['b', '2'], + ['b', '3'] + ]; + + let headers = new Headers(src); + expect(headers.get('b')).to.equal('2,3'); + + Headers.FOLLOW_SPEC = false; + expect(headers.get('b')).to.equal('2,3'); + + headers = new Headers(src); + expect(headers.get('b')).to.equal('2'); + + Headers.FOLLOW_SPEC = defaultFollowSpec; }); it('should allow deleting header', function() { @@ -1612,3 +1649,5 @@ describe('node-fetch', () => { }); }); + +}); From f829b71ddb1ec433e463585fdcb69e232bd6fffb Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Wed, 12 Oct 2016 17:48:19 -0700 Subject: [PATCH 024/223] Use babel-runtime's Iterator implementation --- src/headers.js | 33 ++++----------------------------- 1 file changed, 4 insertions(+), 29 deletions(-) diff --git a/src/headers.js b/src/headers.js index 9ad129e8f..ece6b0b1d 100644 --- a/src/headers.js +++ b/src/headers.js @@ -5,6 +5,7 @@ * Headers class offers convenient helpers */ +import getIterator from 'babel-runtime/core-js/get-iterator'; import { _checkIsHttpToken, _checkInvalidHeaderChar } from './common.js'; function sanitizeName(name) { @@ -174,7 +175,7 @@ export default class Headers { } else { this.forEach((_, name) => keys.push(name)); }; - return new Iterator(keys); + return getIterator(keys); } /** @@ -190,7 +191,7 @@ export default class Headers { } else { const values = []; this.forEach(value => values.push(value)); - yield* new Iterator(values); + yield* getIterator(values); } } @@ -207,7 +208,7 @@ export default class Headers { } else { const entries = []; this.forEach((value, name) => entries.push([name, value])); - yield* new Iterator(entries); + yield* getIterator(entries); } } @@ -231,29 +232,3 @@ export default class Headers { } Headers.FOLLOW_SPEC = false; - -const ITEMS = Symbol('items'); -class Iterator { - constructor(items) { - this[ITEMS] = items; - } - - next() { - if (!this[ITEMS].length) { - return { - value: undefined, - done: true - }; - } - - return { - value: this[ITEMS].shift(), - done: false - }; - - } - - [Symbol.iterator]() { - return this; - } -} From 8f3e886c7fe377e95f2338cecb939a1de3bf3488 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Thu, 13 Oct 2016 00:32:52 -0700 Subject: [PATCH 025/223] Switch to Codecov Fixes #186. --- .gitignore | 3 --- README.md | 6 +++--- codecov.yml | 3 +++ package.json | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) create mode 100644 codecov.yml diff --git a/.gitignore b/.gitignore index a2234e079..b4905739c 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,3 @@ node_modules # OS files .DS_Store - -# Coveralls token files -.coveralls.yml diff --git a/README.md b/README.md index 5a4f380f3..ecb5b816d 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ node-fetch [![npm version][npm-image]][npm-url] [![build status][travis-image]][travis-url] -[![coverage status][coveralls-image]][coveralls-url] +[![coverage status][codecov-image]][codecov-url] A light-weight module that brings `window.fetch` to Node.js @@ -220,5 +220,5 @@ Thanks to [github/fetch](https://github.com/github/fetch) for providing a solid [npm-url]: https://www.npmjs.com/package/node-fetch [travis-image]: https://img.shields.io/travis/bitinn/node-fetch.svg?style=flat-square [travis-url]: https://travis-ci.org/bitinn/node-fetch -[coveralls-image]: https://img.shields.io/coveralls/bitinn/node-fetch.svg?style=flat-square -[coveralls-url]: https://coveralls.io/r/bitinn/node-fetch +[codecov-image]: https://img.shields.io/codecov/c/github/bitinn/node-fetch.svg?style=flat-square +[codecov-url]: https://codecov.io/gh/bitinn/node-fetch diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..b4e9d3fcd --- /dev/null +++ b/codecov.yml @@ -0,0 +1,3 @@ +parsers: + javascript: + enable_partials: yes diff --git a/package.json b/package.json index 262708091..4ff2149aa 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "scripts": { "test": "mocha test/test.js", "report": "istanbul cover _mocha -- -R spec test/test.js", - "coverage": "istanbul cover _mocha --report lcovonly -- -R spec test/test.js && cat ./coverage/lcov.info | coveralls" + "coverage": "istanbul cover _mocha --report lcovonly -- -R spec test/test.js && codecov -f coverage/coverage.json" }, "repository": { "type": "git", @@ -27,7 +27,7 @@ "bluebird": "^3.3.4", "chai": "^3.5.0", "chai-as-promised": "^5.2.0", - "coveralls": "^2.11.2", + "codecov": "^1.0.1", "form-data": ">=1.0.0", "istanbul": "^0.4.2", "mocha": "^2.1.0", From d708acfb9ca8e58f71c575da6934cd6d84352e7a Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Thu, 13 Oct 2016 00:42:26 -0700 Subject: [PATCH 026/223] Cache node_modules in Travis [ci skip] --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a1358b0d9..b6c20826d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,4 +9,7 @@ env: before_script: - 'if [ "$FORMDATA_VERSION" ]; then npm install form-data@^$FORMDATA_VERSION; fi' before_install: npm install -g npm -script: npm run coverage \ No newline at end of file +script: npm run coverage +cache: + directories: + - node_modules From d3b4161d7c26210aa115047844d2504ca58a87a8 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Wed, 12 Oct 2016 21:47:36 -0700 Subject: [PATCH 027/223] Add a new res.textConverted() and always use UTF-8 for res.text() Also uses iconv-lite directly instead of using the "encoding" package. Fixes #184. --- src/body.js | 38 ++++++++++++++++++++------------------ test/test.js | 28 +++++++++++++++++++--------- 2 files changed, 39 insertions(+), 27 deletions(-) diff --git a/src/body.js b/src/body.js index bd30eb637..48a7c8934 100644 --- a/src/body.js +++ b/src/body.js @@ -77,6 +77,16 @@ export default class Body { return this[CONSUME_BODY](); } + /** + * Decode response as text, while automatically detecting the encoding and + * trying to decode to UTF-8 (non-spec api) + * + * @return Promise + */ + textConverted() { + return this[CONSUME_BODY]().then(buffer => convertBody(buffer, this.headers)); + } + /** * Decode buffers into utf-8 string * @@ -96,12 +106,12 @@ export default class Body { // body is string if (typeof this.body === 'string') { - return Body.Promise.resolve(convertBody([new Buffer(this.body)], this.headers)); + return Body.Promise.resolve(new Buffer(this.body)); } // body is buffer if (Buffer.isBuffer(this.body)) { - return Body.Promise.resolve(convertBody([this.body], this.headers)); + return Body.Promise.resolve(this.body); } // body is stream @@ -147,7 +157,7 @@ export default class Body { } clearTimeout(resTimeout); - resolve(convertBody(accum, this.headers)); + resolve(Buffer.concat(accum)); }); }); } @@ -158,11 +168,11 @@ export default class Body { * Detect buffer encoding and convert to target encoding * ref: http://www.w3.org/TR/2011/WD-html5-20110113/parsing.html#determining-the-character-encoding * - * @param Array arrayOfBuffers Array of buffers + * @param Buffer buffer Incoming buffer * @param String encoding Target encoding * @return String */ -function convertBody(arrayOfBuffers, headers) { +function convertBody(buffer, headers) { const ct = headers.get('content-type'); let charset = 'utf-8'; let res, str; @@ -171,22 +181,14 @@ function convertBody(arrayOfBuffers, headers) { if (ct) { // skip encoding detection altogether if not html/xml/plain text if (!/text\/html|text\/plain|\+xml|\/xml/i.test(ct)) { - return Buffer.concat(arrayOfBuffers); + return buffer; } res = /charset=([^;]*)/i.exec(ct); } // no charset in content type, peek at response body for at most 1024 bytes - if (!res && arrayOfBuffers.length > 0) { - for (let i = 0; i < arrayOfBuffers.length; i++) { - str += arrayOfBuffers[i].toString() - if (str.length > 1024) { - break; - } - } - str = str.substr(0, 1024); - } + str = buffer.slice(0, 1024).toString(); // html5 if (!res && str) { @@ -220,10 +222,10 @@ function convertBody(arrayOfBuffers, headers) { // turn raw buffers into a single utf-8 buffer return convert( - Buffer.concat(arrayOfBuffers) - , 'utf-8' + buffer + , 'UTF-8' , charset - ); + ).toString(); } /** diff --git a/test/test.js b/test/test.js index 5f61eb862..51d10eb02 100644 --- a/test/test.js +++ b/test/test.js @@ -923,11 +923,21 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { }); }); - it('should support encoding decode, xml dtd detect', function() { + it('should only use UTF-8 decoding with text()', function() { url = `${base}encoding/euc-jp`; return fetch(url).then(res => { expect(res.status).to.equal(200); return res.text().then(result => { + expect(result).to.equal('\ufffd\ufffd\ufffd\u0738\ufffd'); + }); + }); + }); + + it('should support encoding decode, xml dtd detect', function() { + url = `${base}encoding/euc-jp`; + return fetch(url).then(res => { + expect(res.status).to.equal(200); + return res.textConverted().then(result => { expect(result).to.equal('日本語'); }); }); @@ -937,7 +947,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { url = `${base}encoding/shift-jis`; return fetch(url).then(res => { expect(res.status).to.equal(200); - return res.text().then(result => { + return res.textConverted().then(result => { expect(result).to.equal('
日本語
'); }); }); @@ -947,7 +957,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { url = `${base}encoding/gbk`; return fetch(url).then(res => { expect(res.status).to.equal(200); - return res.text().then(result => { + return res.textConverted().then(result => { expect(result).to.equal('
中文
'); }); }); @@ -957,7 +967,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { url = `${base}encoding/gb2312`; return fetch(url).then(res => { expect(res.status).to.equal(200); - return res.text().then(result => { + return res.textConverted().then(result => { expect(result).to.equal('
中文
'); }); }); @@ -968,7 +978,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { return fetch(url).then(res => { expect(res.status).to.equal(200); expect(res.headers.get('content-type')).to.be.null; - return res.text().then(result => { + return res.textConverted().then(result => { expect(result).to.equal('中文'); }); }); @@ -978,7 +988,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { url = `${base}encoding/order1`; return fetch(url).then(res => { expect(res.status).to.equal(200); - return res.text().then(result => { + return res.textConverted().then(result => { expect(result).to.equal('中文'); }); }); @@ -988,7 +998,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { url = `${base}encoding/order2`; return fetch(url).then(res => { expect(res.status).to.equal(200); - return res.text().then(result => { + return res.textConverted().then(result => { expect(result).to.equal('中文'); }); }); @@ -999,7 +1009,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { return fetch(url).then(res => { expect(res.status).to.equal(200); const padding = 'a'.repeat(10); - return res.text().then(result => { + return res.textConverted().then(result => { expect(result).to.equal(`${padding}
日本語
`); }); }); @@ -1010,7 +1020,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { return fetch(url).then(res => { expect(res.status).to.equal(200); const padding = 'a'.repeat(1200); - return res.text().then(result => { + return res.textConverted().then(result => { expect(result).to.not.equal(`${padding}中文`); }); }); From 1b951701ece70345976925cc0b0dfffdd27b19b8 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Thu, 13 Oct 2016 00:26:55 -0700 Subject: [PATCH 028/223] Remove logic to skip if not HTML in textConverted As this function is now separated from the general purpose text(), we should be more specific in our purpose. --- src/body.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/body.js b/src/body.js index 48a7c8934..dee16de09 100644 --- a/src/body.js +++ b/src/body.js @@ -179,11 +179,6 @@ function convertBody(buffer, headers) { // header if (ct) { - // skip encoding detection altogether if not html/xml/plain text - if (!/text\/html|text\/plain|\+xml|\/xml/i.test(ct)) { - return buffer; - } - res = /charset=([^;]*)/i.exec(ct); } From 53e1055845459d77dedf29451bc2b1b784ad96f1 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sat, 15 Oct 2016 10:02:52 -0700 Subject: [PATCH 029/223] Use Object.create(null) for Headers' internal map Suggested by @jimmywarting. --- src/headers.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/headers.js b/src/headers.js index ece6b0b1d..ce320b562 100644 --- a/src/headers.js +++ b/src/headers.js @@ -34,7 +34,7 @@ export default class Headers { * @return Void */ constructor(headers) { - this[MAP] = {}; + this[MAP] = Object.create(null); this[FOLLOW_SPEC] = Headers.FOLLOW_SPEC; // Headers @@ -100,11 +100,11 @@ export default class Headers { * @return Void */ forEach(callback, thisArg) { - Object.getOwnPropertyNames(this[MAP]).forEach(name => { + for (let name in this[MAP]) { this[MAP][name].forEach(value => { callback.call(thisArg, value, name, this); }); - }); + } } /** @@ -141,7 +141,7 @@ export default class Headers { * @return Boolean */ has(name) { - return this[MAP].hasOwnProperty(sanitizeName(name)); + return !!this[MAP][sanitizeName(name)]; } /** From b092a8ed12f07ec2838d3b05572a3d3920c1b350 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Wed, 12 Oct 2016 19:25:00 -0700 Subject: [PATCH 030/223] Add test for constructing Request with parsed URL object --- test/test.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/test.js b/test/test.js index 51d10eb02..0871a07df 100644 --- a/test/test.js +++ b/test/test.js @@ -9,6 +9,7 @@ import {spawn} from 'child_process'; import * as stream from 'stream'; import resumer from 'resumer'; import FormData from 'form-data'; +import {parse as parseURL} from 'url'; import * as http from 'http'; import * as fs from 'fs'; @@ -1382,6 +1383,17 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { }); }); + it('should support fetch with Node.js URL object', function() { + url = `${base}hello`; + const urlObj = parseURL(url); + const req = new Request(urlObj); + return fetch(req).then(res => { + expect(res.url).to.equal(url); + expect(res.ok).to.be.true; + expect(res.status).to.equal(200); + }); + }); + it('should support wrapping Request instance', function() { url = `${base}hello`; From 76cb57cace246fa6d5764d761247fdbe8e08519a Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Wed, 12 Oct 2016 19:56:47 -0700 Subject: [PATCH 031/223] Support WHATWG URL objects Fixes #175. --- package.json | 10 ++++++++-- src/request.js | 10 +++++++++- test/test.js | 17 +++++++++++++++++ 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index bdc513f90..1b0b39ac9 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,8 @@ "nyc": "^8.3.0", "parted": "^0.1.1", "promise": "^7.1.1", - "resumer": "0.0.0" + "resumer": "0.0.0", + "whatwg-url": "^3.0.0" }, "dependencies": { "babel-runtime": "^6.11.6", @@ -53,7 +54,12 @@ }, "babel": { "presets": [ - ["es2015", {"loose": true}] + [ + "es2015", + { + "loose": true + } + ] ], "plugins": [ "transform-runtime" diff --git a/src/request.js b/src/request.js index 978e5ec6d..0ba13e7cd 100644 --- a/src/request.js +++ b/src/request.js @@ -22,7 +22,15 @@ export default class Request extends Body { // normalize input if (!(input instanceof Request)) { - parsedURL = parse_url(input); + if (input && input.href) { + // in order to support Node.js' Url objects; though WHATWG's URL objects + // will fall into this branch also (since their `toString()` will return + // `href` property anyway) + parsedURL = parse_url(input.href); + } else { + // coerce input to a string before attempting to parse + parsedURL = parse_url(input + ''); + } input = {}; } else { parsedURL = parse_url(input.url); diff --git a/test/test.js b/test/test.js index 0871a07df..0fa797bc9 100644 --- a/test/test.js +++ b/test/test.js @@ -29,6 +29,12 @@ import FetchError from '../src/fetch-error.js'; // test with native promise on node 0.11, and bluebird for node 0.10 fetch.Promise = fetch.Promise || bluebird; +let URL; +// whatwg-url doesn't support old Node.js, so make it optional +try { + URL = require('whatwg-url').URL; +} catch (err) {} + const local = new TestServer(); const base = `http://${local.hostname}:${local.port}/`; let url, opts; @@ -1394,6 +1400,17 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { }); }); + (URL ? it : it.skip)('should support fetch with WHATWG URL object', function() { + url = `${base}hello`; + const urlObj = new URL(url); + const req = new Request(urlObj); + return fetch(req).then(res => { + expect(res.url).to.equal(url); + expect(res.ok).to.be.true; + expect(res.status).to.equal(200); + }); + }); + it('should support wrapping Request instance', function() { url = `${base}hello`; From 4a9a3246f633c3ddb358d3e79a38b39bb2a61bb1 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Fri, 4 Nov 2016 21:50:58 -0700 Subject: [PATCH 032/223] Remove dependency on Node.js' util module Closes #194. --- src/fetch-error.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/fetch-error.js b/src/fetch-error.js index cc67925eb..a41791955 100644 --- a/src/fetch-error.js +++ b/src/fetch-error.js @@ -14,11 +14,11 @@ * @return FetchError */ export default function FetchError(message, type, systemError) { + Error.call(this, message); // hide custom error implementation details from end-users Error.captureStackTrace(this, this.constructor); - this.name = this.constructor.name; this.message = message; this.type = type; @@ -29,4 +29,5 @@ export default function FetchError(message, type, systemError) { } -require('util').inherits(FetchError, Error); +FetchError.prototype = Object.create(Error.prototype); +FetchError.prototype.name = 'FetchError'; From 72d34af6e257503d79e77cdd3b520999f2d0fd94 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sat, 5 Nov 2016 09:39:01 -0700 Subject: [PATCH 033/223] Start test server when executed directly --- test/server.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/server.js b/test/server.js index 5a75b2973..403f9ed61 100644 --- a/test/server.js +++ b/test/server.js @@ -329,3 +329,10 @@ export default class TestServer { } } } + +if (require.main === module) { + const server = new TestServer; + server.start(() => { + console.log(`Server started listening at port ${server.port}`); + }); +} From 5fe80dba06fbf384cc712a299c698b884a857cfc Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sat, 5 Nov 2016 10:26:30 -0700 Subject: [PATCH 034/223] Remove dependency on babel-polyfill This way the tests can better emulate the real Node.js environment. --- package.json | 4 +--- test/server.js | 8 +++----- test/test.js | 24 ++++++++++++++++-------- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index 1b0b39ac9..2e27124ee 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "babel -d lib src", "prepublish": "npm run build", - "test": "mocha --compilers js:babel-polyfill --compilers js:babel-register test/test.js", + "test": "mocha --compilers js:babel-register test/test.js", "report": "cross-env BABEL_ENV=test nyc --reporter lcov --reporter text mocha -R spec test/test.js", "coverage": "cross-env BABEL_ENV=test nyc --reporter json --reporter text mocha -R spec test/test.js && codecov -f coverage/coverage-final.json" }, @@ -29,7 +29,6 @@ "babel-cli": "^6.16.0", "babel-plugin-istanbul": "^2.0.1", "babel-plugin-transform-runtime": "^6.15.0", - "babel-polyfill": "^6.16.0", "babel-preset-es2015": "^6.16.0", "babel-register": "^6.16.3", "bluebird": "^3.3.4", @@ -77,7 +76,6 @@ "src/*.js" ], "require": [ - "babel-polyfill", "babel-register" ], "sourceMap": false, diff --git a/test/server.js b/test/server.js index 403f9ed61..804603a04 100644 --- a/test/server.js +++ b/test/server.js @@ -1,4 +1,4 @@ -import 'babel-polyfill'; +import repeat from 'babel-runtime/core-js/string/repeat'; import * as http from 'http'; import { parse } from 'url'; import * as zlib from 'zlib'; @@ -188,8 +188,7 @@ export default class TestServer { res.statusCode = 200; res.setHeader('Content-Type', 'text/html'); res.setHeader('Transfer-Encoding', 'chunked'); - const padding = 'a'; - res.write(padding.repeat(10)); + res.write(repeat('a', 10)); res.end(convert('
日本語
', 'Shift_JIS')); } @@ -197,8 +196,7 @@ export default class TestServer { res.statusCode = 200; res.setHeader('Content-Type', 'text/html'); res.setHeader('Transfer-Encoding', 'chunked'); - const padding = 'a'.repeat(120); - res.write(padding.repeat(10)); + res.write(repeat('a', 1200)); res.end(convert('中文', 'gbk')); } diff --git a/test/test.js b/test/test.js index 0fa797bc9..5dd5d5b20 100644 --- a/test/test.js +++ b/test/test.js @@ -1,5 +1,6 @@ // test tools +import repeat from 'babel-runtime/core-js/string/repeat'; import chai from 'chai'; import chaiPromised from 'chai-as-promised'; import chaiIterator from 'chai-iterator'; @@ -35,6 +36,11 @@ try { URL = require('whatwg-url').URL; } catch (err) {} +const supportToString = ({ + [Symbol.toStringTag]: 'z' +}).toString() === '[object z]'; +const supportIterator = !!(global.Symbol && global.Symbol.iterator); + const local = new TestServer(); const base = `http://${local.hostname}:${local.port}/`; let url, opts; @@ -89,7 +95,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { expect(fetch.Request).to.equal(Request); }); - it('should support proper toString output for Headers, Response and Request objects', function() { + (supportToString ? it : it.skip)('should support proper toString output for Headers, Response and Request objects', function() { expect(new Headers().toString()).to.equal('[object Headers]'); expect(new Response().toString()).to.equal('[object Response]'); expect(new Request(base).toString()).to.equal('[object Request]'); @@ -1015,7 +1021,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { url = `${base}encoding/chunked`; return fetch(url).then(res => { expect(res.status).to.equal(200); - const padding = 'a'.repeat(10); + const padding = repeat('a', 10); return res.textConverted().then(result => { expect(result).to.equal(`${padding}
日本語
`); }); @@ -1026,7 +1032,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { url = `${base}encoding/invalid`; return fetch(url).then(res => { expect(res.status).to.equal(200); - const padding = 'a'.repeat(1200); + const padding = repeat('a', 1200); return res.textConverted().then(result => { expect(result).to.not.equal(`${padding}中文`); }); @@ -1170,7 +1176,9 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { ['a', '1'] ]); headers.append('b', '3'); - expect(headers).to.be.iterable; + if (supportIterator) { + expect(headers).to.be.iterable; + } const result = []; for (let pair of headers) { @@ -1181,14 +1189,14 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { ['b', '2,3'], ['c', '4'] ] : [ - ['b', '2'], + ['b', '2'], ['b', '3'], ['c', '4'], ['a', '1'], ]); }); - it('should allow iterating through all headers with entries()', function() { + (supportIterator ? it : it.skip)('should allow iterating through all headers with entries()', function() { const headers = new Headers([ ['b', '2'], ['c', '4'], @@ -1209,7 +1217,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { ]); }); - it('should allow iterating through all headers with keys()', function() { + (supportIterator ? it : it.skip)('should allow iterating through all headers with keys()', function() { const headers = new Headers([ ['b', '2'], ['c', '4'], @@ -1221,7 +1229,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { .and.to.iterate.over(Headers.FOLLOW_SPEC ? ['a', 'b', 'c'] : ['b', 'b', 'c', 'a']); }); - it('should allow iterating through all headers with values()', function() { + (supportIterator ? it : it.skip)('should allow iterating through all headers with values()', function() { const headers = new Headers([ ['b', '2'], ['c', '4'], From 31bc2835dd45927af2697073ca4b294ec7244742 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Wed, 23 Nov 2016 11:17:42 -0800 Subject: [PATCH 035/223] Start using Rollup Smaller distributed tarball. --- .babelrc | 25 +++++++++++++++++++++++++ .nycrc | 10 ++++++++++ package.json | 47 ++++++++++++----------------------------------- rollup.config.js | 17 +++++++++++++++++ src/common.js | 6 ++---- src/headers.js | 6 +++--- 6 files changed, 69 insertions(+), 42 deletions(-) create mode 100644 .babelrc create mode 100644 .nycrc create mode 100644 rollup.config.js diff --git a/.babelrc b/.babelrc new file mode 100644 index 000000000..50318ff6d --- /dev/null +++ b/.babelrc @@ -0,0 +1,25 @@ +{ + "plugins": [ + "transform-runtime" + ], + "env": { + "test": { + "presets": [ + [ "es2015", { "loose": true } ] + ] + }, + "coverage": { + "presets": [ + [ "es2015", { "loose": true } ] + ], + "plugins": [ + "istanbul" + ] + }, + "rollup": { + "presets": [ + [ "es2015", { "loose": true, "modules": false } ] + ] + } + } +} diff --git a/.nycrc b/.nycrc new file mode 100644 index 000000000..ad1e79031 --- /dev/null +++ b/.nycrc @@ -0,0 +1,10 @@ +{ + "include": [ + "src/*.js" + ], + "require": [ + "babel-register" + ], + "sourceMap": false, + "instrument": false +} diff --git a/package.json b/package.json index 2e27124ee..2b514acfd 100644 --- a/package.json +++ b/package.json @@ -3,12 +3,17 @@ "version": "1.6.3", "description": "A light-weight module that brings window.fetch to node.js and io.js", "main": "lib/index.js", + "jsnext:main": "lib/index.es.js", + "files": [ + "lib/index.js", + "lib/index.es.js" + ], "scripts": { - "build": "babel -d lib src", + "build": "rollup -c", "prepublish": "npm run build", - "test": "mocha --compilers js:babel-register test/test.js", - "report": "cross-env BABEL_ENV=test nyc --reporter lcov --reporter text mocha -R spec test/test.js", - "coverage": "cross-env BABEL_ENV=test nyc --reporter json --reporter text mocha -R spec test/test.js && codecov -f coverage/coverage-final.json" + "test": "cross-env BABEL_ENV=test mocha --compilers js:babel-register test/test.js", + "report": "cross-env BABEL_ENV=coverage nyc --reporter lcov --reporter text mocha -R spec test/test.js", + "coverage": "cross-env BABEL_ENV=coverage nyc --reporter json --reporter text mocha -R spec test/test.js && codecov -f coverage/coverage-final.json" }, "repository": { "type": "git", @@ -26,7 +31,6 @@ }, "homepage": "https://github.com/bitinn/node-fetch", "devDependencies": { - "babel-cli": "^6.16.0", "babel-plugin-istanbul": "^2.0.1", "babel-plugin-transform-runtime": "^6.15.0", "babel-preset-es2015": "^6.16.0", @@ -43,6 +47,9 @@ "parted": "^0.1.1", "promise": "^7.1.1", "resumer": "0.0.0", + "rollup": "^0.36.4", + "rollup-plugin-babel": "^2.6.1", + "rollup-plugin-node-resolve": "^2.0.0", "whatwg-url": "^3.0.0" }, "dependencies": { @@ -50,35 +57,5 @@ "buffer-to-arraybuffer": "0.0.4", "encoding": "^0.1.11", "is-stream": "^1.0.1" - }, - "babel": { - "presets": [ - [ - "es2015", - { - "loose": true - } - ] - ], - "plugins": [ - "transform-runtime" - ], - "env": { - "test": { - "plugins": [ - "istanbul" - ] - } - } - }, - "nyc": { - "include": [ - "src/*.js" - ], - "require": [ - "babel-register" - ], - "sourceMap": false, - "instrument": false } } diff --git a/rollup.config.js b/rollup.config.js new file mode 100644 index 000000000..4a57423de --- /dev/null +++ b/rollup.config.js @@ -0,0 +1,17 @@ +import babel from 'rollup-plugin-babel'; +import resolve from 'rollup-plugin-node-resolve'; + +process.env.BABEL_ENV = 'rollup'; + +export default { + entry: 'src/index.js', + plugins: [ + babel({ + runtimeHelpers: true + }) + ], + targets: [ + { dest: 'lib/index.js', format: 'cjs' }, + { dest: 'lib/index.es.js', format: 'es' } + ] +}; diff --git a/src/common.js b/src/common.js index f4908a3cb..a53e051d9 100644 --- a/src/common.js +++ b/src/common.js @@ -47,7 +47,7 @@ function isValidTokenChar(ch) { return false; } /* istanbul ignore next */ -function checkIsHttpToken(val) { +export function checkIsHttpToken(val) { if (typeof val !== 'string' || val.length === 0) return false; if (!isValidTokenChar(val.charCodeAt(0))) @@ -71,7 +71,6 @@ function checkIsHttpToken(val) { } return true; } -exports._checkIsHttpToken = checkIsHttpToken; /** * True if val contains an invalid field-vchar @@ -84,7 +83,7 @@ exports._checkIsHttpToken = checkIsHttpToken; * code size does not exceed v8's default max_inlined_source_size setting. **/ /* istanbul ignore next */ -function checkInvalidHeaderChar(val) { +export function checkInvalidHeaderChar(val) { val += ''; if (val.length < 1) return false; @@ -108,4 +107,3 @@ function checkInvalidHeaderChar(val) { } return false; } -exports._checkInvalidHeaderChar = checkInvalidHeaderChar; diff --git a/src/headers.js b/src/headers.js index ce320b562..07f1ff342 100644 --- a/src/headers.js +++ b/src/headers.js @@ -6,11 +6,11 @@ */ import getIterator from 'babel-runtime/core-js/get-iterator'; -import { _checkIsHttpToken, _checkInvalidHeaderChar } from './common.js'; +import { checkIsHttpToken, checkInvalidHeaderChar } from './common.js'; function sanitizeName(name) { name += ''; - if (!_checkIsHttpToken(name)) { + if (!checkIsHttpToken(name)) { throw new TypeError(`${name} is not a legal HTTP header name`); } return name.toLowerCase(); @@ -18,7 +18,7 @@ function sanitizeName(name) { function sanitizeValue(value) { value += ''; - if (_checkInvalidHeaderChar(value)) { + if (checkInvalidHeaderChar(value)) { throw new TypeError(`${value} is not a legal HTTP header value`); } return value; From a355664e642cade7d92db56a1fcc28639a11229a Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Wed, 23 Nov 2016 11:30:01 -0800 Subject: [PATCH 036/223] Update packages --- .nycrc | 3 --- package.json | 12 ++++++------ test/test.js | 51 +++++++++++++++++++++++---------------------------- 3 files changed, 29 insertions(+), 37 deletions(-) diff --git a/.nycrc b/.nycrc index ad1e79031..d8d9c1432 100644 --- a/.nycrc +++ b/.nycrc @@ -1,7 +1,4 @@ { - "include": [ - "src/*.js" - ], "require": [ "babel-register" ], diff --git a/package.json b/package.json index 2b514acfd..3c693ece5 100644 --- a/package.json +++ b/package.json @@ -31,26 +31,26 @@ }, "homepage": "https://github.com/bitinn/node-fetch", "devDependencies": { - "babel-plugin-istanbul": "^2.0.1", + "babel-plugin-istanbul": "^3.0.0", "babel-plugin-transform-runtime": "^6.15.0", "babel-preset-es2015": "^6.16.0", "babel-register": "^6.16.3", "bluebird": "^3.3.4", "chai": "^3.5.0", - "chai-as-promised": "^5.2.0", + "chai-as-promised": "^6.0.0", "chai-iterator": "^1.1.1", "codecov": "^1.0.1", - "cross-env": "2.0.1", + "cross-env": "^3.1.3", "form-data": ">=1.0.0", - "mocha": "^2.1.0", - "nyc": "^8.3.0", + "mocha": "^3.1.2", + "nyc": "^10.0.0", "parted": "^0.1.1", "promise": "^7.1.1", "resumer": "0.0.0", "rollup": "^0.36.4", "rollup-plugin-babel": "^2.6.1", "rollup-plugin-node-resolve": "^2.0.0", - "whatwg-url": "^3.0.0" + "whatwg-url": "^4.0.0" }, "dependencies": { "babel-runtime": "^6.11.6", diff --git a/test/test.js b/test/test.js index 5dd5d5b20..6ad17dff8 100644 --- a/test/test.js +++ b/test/test.js @@ -1039,53 +1039,36 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { }); }); - it('should allow piping response body as stream', function(done) { + it('should allow piping response body as stream', function() { url = `${base}hello`; - fetch(url).then(res => { + return fetch(url).then(res => { expect(res.body).to.be.an.instanceof(stream.Transform); - res.body.on('data', chunk => { + return streamToPromise(res.body, chunk => { if (chunk === null) { return; } expect(chunk.toString()).to.equal('world'); }); - res.body.on('end', () => { - done(); - }); }); }); - it('should allow cloning a response, and use both as stream', function(done) { + it('should allow cloning a response, and use both as stream', function() { url = `${base}hello`; return fetch(url).then(res => { - let counter = 0; const r1 = res.clone(); expect(res.body).to.be.an.instanceof(stream.Transform); expect(r1.body).to.be.an.instanceof(stream.Transform); - res.body.on('data', chunk => { - if (chunk === null) { - return; - } - expect(chunk.toString()).to.equal('world'); - }); - res.body.on('end', () => { - counter++; - if (counter == 2) { - done(); - } - }); - r1.body.on('data', chunk => { + const dataHandler = chunk => { if (chunk === null) { return; } expect(chunk.toString()).to.equal('world'); - }); - r1.body.on('end', () => { - counter++; - if (counter == 2) { - done(); - } - }); + }; + + return Promise.all([ + streamToPromise(res.body, dataHandler), + streamToPromise(r1.body, dataHandler) + ]); }); }); @@ -1698,3 +1681,15 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { }); }); + +function streamToPromise(stream, dataHandler) { + return new Promise((resolve, reject) => { + stream.on('data', (...args) => { + Promise.resolve() + .then(() => dataHandler(...args)) + .catch(reject); + }); + stream.on('end', resolve); + stream.on('error', reject); + }); +} From 908bcaac4cacf5b027e3a82e883cabe09f22a0ed Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Wed, 23 Nov 2016 11:32:43 -0800 Subject: [PATCH 037/223] Work around istanbul's bug See istanbuljs/istanbul-lib-instrument#30. --- src/common.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/common.js b/src/common.js index a53e051d9..b0f9c999a 100644 --- a/src/common.js +++ b/src/common.js @@ -47,7 +47,7 @@ function isValidTokenChar(ch) { return false; } /* istanbul ignore next */ -export function checkIsHttpToken(val) { +function checkIsHttpToken(val) { if (typeof val !== 'string' || val.length === 0) return false; if (!isValidTokenChar(val.charCodeAt(0))) @@ -71,6 +71,7 @@ export function checkIsHttpToken(val) { } return true; } +export { checkIsHttpToken }; /** * True if val contains an invalid field-vchar @@ -83,7 +84,7 @@ export function checkIsHttpToken(val) { * code size does not exceed v8's default max_inlined_source_size setting. **/ /* istanbul ignore next */ -export function checkInvalidHeaderChar(val) { +function checkInvalidHeaderChar(val) { val += ''; if (val.length < 1) return false; @@ -107,3 +108,4 @@ export function checkInvalidHeaderChar(val) { } return false; } +export { checkInvalidHeaderChar }; From 049585be113af88c2315db186717929903954205 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Wed, 23 Nov 2016 11:43:51 -0800 Subject: [PATCH 038/223] Revert to cross-env 2.0.1 The Object.assign used in that module is not polyfilled. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 3c693ece5..d1ff6693b 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "chai-as-promised": "^6.0.0", "chai-iterator": "^1.1.1", "codecov": "^1.0.1", - "cross-env": "^3.1.3", + "cross-env": "2.0.1", "form-data": ">=1.0.0", "mocha": "^3.1.2", "nyc": "^10.0.0", From 25d139a4d0a4ea1d703abbacec8fb31589a9086e Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Wed, 23 Nov 2016 11:50:05 -0800 Subject: [PATCH 039/223] Officially drop support for 0.10 Current LTS releases are added. --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b6c20826d..23a2e4adc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,8 @@ language: node_js node_js: - - "0.10" - "0.12" + - "4" + - "6" - "node" env: - FORMDATA_VERSION=1.0.0 From 4ae42ea5eec9260c9372b30f47e052dee9569914 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Wed, 23 Nov 2016 14:36:08 -0800 Subject: [PATCH 040/223] Fix class names for prototypes Per Web IDL specification. --- src/headers.js | 21 ++++++++++++++------- src/request.js | 21 ++++++++++++++------- src/response.js | 21 ++++++++++++++------- 3 files changed, 42 insertions(+), 21 deletions(-) diff --git a/src/headers.js b/src/headers.js index 07f1ff342..703ed67a7 100644 --- a/src/headers.js +++ b/src/headers.js @@ -61,6 +61,13 @@ export default class Headers { this.append(prop, headers[prop]); } } + + Object.defineProperty(this, Symbol.toStringTag, { + value: 'Headers', + writable: false, + enumerable: false, + configurable: true + }); } /** @@ -222,13 +229,13 @@ export default class Headers { [Symbol.iterator]() { return this.entries(); } - - /** - * Tag used by `Object.prototype.toString()`. - */ - get [Symbol.toStringTag]() { - return 'Headers'; - } } +Object.defineProperty(Headers.prototype, Symbol.toStringTag, { + value: 'HeadersPrototype', + writable: false, + enumerable: false, + configurable: true +}); + Headers.FOLLOW_SPEC = false; diff --git a/src/request.js b/src/request.js index 0ba13e7cd..136570ac3 100644 --- a/src/request.js +++ b/src/request.js @@ -58,6 +58,13 @@ export default class Request extends Body { // server request options Object.assign(this, parsedURL); + + Object.defineProperty(this, Symbol.toStringTag, { + value: 'Request', + writable: false, + enumerable: false, + configurable: true + }); } get url() { @@ -72,11 +79,11 @@ export default class Request extends Body { clone() { return new Request(this); } - - /** - * Tag used by `Object.prototype.toString()`. - */ - get [Symbol.toStringTag]() { - return 'Request'; - } } + +Object.defineProperty(Request.prototype, Symbol.toStringTag, { + value: 'RequestPrototype', + writable: false, + enumerable: false, + configurable: true +}); diff --git a/src/response.js b/src/response.js index bc3175b06..85b1820ef 100644 --- a/src/response.js +++ b/src/response.js @@ -24,6 +24,13 @@ export default class Response extends Body { this.status = opts.status || 200; this.statusText = opts.statusText || STATUS_CODES[this.status]; this.headers = new Headers(opts.headers); + + Object.defineProperty(this, Symbol.toStringTag, { + value: 'Response', + writable: false, + enumerable: false, + configurable: true + }); } /** @@ -49,11 +56,11 @@ export default class Response extends Body { }); } - - /** - * Tag used by `Object.prototype.toString()`. - */ - get [Symbol.toStringTag]() { - return 'Response'; - } } + +Object.defineProperty(Response.prototype, Symbol.toStringTag, { + value: 'ResponsePrototype', + writable: false, + enumerable: false, + configurable: true +}); From 25ff99677dbf61ece16a7b8f39776cd2f30c6ff5 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Wed, 23 Nov 2016 15:06:30 -0800 Subject: [PATCH 041/223] Improve Headers' iteration support Class strings, spec-compliant forEach, etc. --- src/headers.js | 136 ++++++++++++++++++++++++++++++++++--------------- test/test.js | 18 ++++--- 2 files changed, 107 insertions(+), 47 deletions(-) diff --git a/src/headers.js b/src/headers.js index 703ed67a7..ce4c32a7e 100644 --- a/src/headers.js +++ b/src/headers.js @@ -5,7 +5,6 @@ * Headers class offers convenient helpers */ -import getIterator from 'babel-runtime/core-js/get-iterator'; import { checkIsHttpToken, checkInvalidHeaderChar } from './common.js'; function sanitizeName(name) { @@ -106,11 +105,14 @@ export default class Headers { * @param Boolean thisArg `this` context for callback function * @return Void */ - forEach(callback, thisArg) { - for (let name in this[MAP]) { - this[MAP][name].forEach(value => { - callback.call(thisArg, value, name, this); - }); + forEach(callback, thisArg = undefined) { + let pairs = getHeaderPairs(this); + let i = 0; + while (i < pairs.length) { + const [name, value] = pairs[i]; + callback.call(thisArg, value, name, this); + pairs = getHeaderPairs(this); + i++; } } @@ -176,13 +178,7 @@ export default class Headers { * @return Iterator */ keys() { - let keys = []; - if (this[FOLLOW_SPEC]) { - keys = Object.keys(this[MAP]).sort(); - } else { - this.forEach((_, name) => keys.push(name)); - }; - return getIterator(keys); + return createHeadersIterator(this, 'key'); } /** @@ -190,33 +186,8 @@ export default class Headers { * * @return Iterator */ - *values() { - if (this[FOLLOW_SPEC]) { - for (const name of this.keys()) { - yield this.get(name); - } - } else { - const values = []; - this.forEach(value => values.push(value)); - yield* getIterator(values); - } - } - - /** - * Get an iterator on entries. - * - * @return Iterator - */ - *entries() { - if (this[FOLLOW_SPEC]) { - for (const name of this.keys()) { - yield [name, this.get(name)]; - } - } else { - const entries = []; - this.forEach((value, name) => entries.push([name, value])); - yield* getIterator(entries); - } + values() { + return createHeadersIterator(this, 'value'); } /** @@ -227,9 +198,10 @@ export default class Headers { * @return Iterator */ [Symbol.iterator]() { - return this.entries(); + return createHeadersIterator(this, 'key+value'); } } +Headers.prototype.entries = Headers.prototype[Symbol.iterator]; Object.defineProperty(Headers.prototype, Symbol.toStringTag, { value: 'HeadersPrototype', @@ -238,4 +210,86 @@ Object.defineProperty(Headers.prototype, Symbol.toStringTag, { configurable: true }); +function getHeaderPairs(headers, kind) { + if (headers[FOLLOW_SPEC]) { + const keys = Object.keys(headers[MAP]).sort(); + return keys.map( + kind === 'key' ? + k => [k] : + k => [k, headers.get(k)] + ); + } + + const values = []; + + for (let name in headers[MAP]) { + for (let value of headers[MAP][name]) { + values.push([name, value]); + } + } + + return values; +} + +const INTERNAL = Symbol('internal'); + +function createHeadersIterator(target, kind) { + const iterator = Object.create(HeadersIteratorPrototype); + iterator[INTERNAL] = { + target, + kind, + index: 0 + }; + return iterator; +} + +const HeadersIteratorPrototype = Object.setPrototypeOf({ + next() { + if (!this || + Object.getPrototypeOf(this) !== HeadersIteratorPrototype) { + throw new TypeError('Value of `this` is not a HeadersIterator'); + } + + const { + target, + kind, + index + } = this[INTERNAL]; + const values = getHeaderPairs(target, kind); + const len = values.length; + if (index >= len) { + return { + value: undefined, + done: true + }; + } + + const pair = values[index]; + this[INTERNAL].index = index + 1; + + let result; + if (kind === 'key') { + result = pair[0]; + } else if (kind === 'value') { + result = pair[1]; + } else { + result = pair; + } + + return { + value: result, + done: false + }; + } +}, Object.getPrototypeOf( + Object.getPrototypeOf([][Symbol.iterator]()) +)); + +Object.defineProperty(HeadersIteratorPrototype, Symbol.toStringTag, { + value: 'HeadersIterator', + writable: false, + enumerable: false, + configurable: true +}); + Headers.FOLLOW_SPEC = false; diff --git a/test/test.js b/test/test.js index 6ad17dff8..114869cfa 100644 --- a/test/test.js +++ b/test/test.js @@ -1132,11 +1132,12 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { }); it('should allow iterating through all headers with forEach', function() { - const headers = new Headers({ - a: 1 - , b: [2, 3] - , c: [4] - }); + const headers = new Headers([ + ['b', '2'], + ['c', '4'], + ['b', '3'], + ['a', '1'] + ]); expect(headers).to.have.property('forEach'); const result = []; @@ -1144,10 +1145,15 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { result.push([key, val]); }); - const expected = [ + const expected = Headers.FOLLOW_SPEC ? [ ["a", "1"] , ["b", "2,3"] , ["c", "4"] + ] : [ + ["b", "2"] + , ["b", "3"] + , ["c", "4"] + , ["a", "1"] ]; expect(result).to.deep.equal(expected); }); From 7f0e50260ea0f9a7662a33ea1bd209f6cb4c5407 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Wed, 23 Nov 2016 15:24:51 -0800 Subject: [PATCH 042/223] Add a polyfill for Node.js v0.12's broken %IteratorPrototype% --- src/headers.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/headers.js b/src/headers.js index ce4c32a7e..1edf19066 100644 --- a/src/headers.js +++ b/src/headers.js @@ -285,6 +285,13 @@ const HeadersIteratorPrototype = Object.setPrototypeOf({ Object.getPrototypeOf([][Symbol.iterator]()) )); +// On Node.js v0.12 the %IteratorPrototype% object is broken +if (typeof HeadersIteratorPrototype[Symbol.iterator] !== 'function') { + HeadersIteratorPrototype[Symbol.iterator] = function () { + return this; + }; +} + Object.defineProperty(HeadersIteratorPrototype, Symbol.toStringTag, { value: 'HeadersIterator', writable: false, From d3071fa46a27abc744b803db890d9323636eabec Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sat, 26 Nov 2016 09:07:12 -0800 Subject: [PATCH 043/223] Revert "Return empty .json() object on 204. Fix #165. (#166)" (#201) This reverts commit 95b58936b8673f7fdc21050c3bfeb460e9b00bf1. Fixes #165. --- src/body.js | 5 ----- test/test.js | 10 ---------- 2 files changed, 15 deletions(-) diff --git a/src/body.js b/src/body.js index dee16de09..cf69ec3b3 100644 --- a/src/body.js +++ b/src/body.js @@ -51,11 +51,6 @@ export default class Body { * @return Promise */ json() { - // for 204 No Content response, buffer will be empty, parsing it will throw error - if (this.status === 204) { - return Body.Promise.resolve({}); - } - return this[CONSUME_BODY]().then(buffer => JSON.parse(buffer.toString())); } diff --git a/test/test.js b/test/test.js index 114869cfa..27d74d2ab 100644 --- a/test/test.js +++ b/test/test.js @@ -471,16 +471,6 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { }); }); - it('should return empty object on no-content response', function() { - url = `${base}no-content`; - return fetch(url).then(res => { - return res.json().then(result => { - expect(result).to.be.an('object'); - expect(result).to.be.empty; - }); - }); - }); - it('should handle no content response with gzip encoding', function() { url = `${base}no-content/gzip`; return fetch(url).then(res => { From 0285828fb8ca41a04ff7eb613d49fbaafd74f781 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Wed, 23 Nov 2016 12:42:24 -0800 Subject: [PATCH 044/223] Split Content-Type extraction to Request and Body It is done in this way in the spec. --- src/body.js | 20 ++++++++++++++++++++ src/index.js | 5 ----- src/request.js | 9 ++++++++- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/body.js b/src/body.js index cf69ec3b3..a198c6cec 100644 --- a/src/body.js +++ b/src/body.js @@ -249,5 +249,25 @@ export function clone(instance) { return body; } +/** + * Performs the operation "extract a `Content-Type` value from |object|" as + * specified in the specification: + * https://fetch.spec.whatwg.org/#concept-bodyinit-extract + * + * This function assumes that instance.body is present and non-null. + * + * @param Mixed instance Response or Request instance + */ +export function extractContentType(instance) { + // detect form data input from form-data module + if (typeof instance.body.getBoundary === 'function') { + return `multipart/form-data;boundary=${instance.body.getBoundary()}`; + } + + if (typeof instance.body === 'string') { + return 'text/plain;charset=UTF-8'; + } +} + // expose Promise Body.Promise = global.Promise; diff --git a/src/index.js b/src/index.js index 6a889fddf..fc4799334 100644 --- a/src/index.js +++ b/src/index.js @@ -68,11 +68,6 @@ function fetch(url, opts) { headers.set('accept', '*/*'); } - // detect form data input from form-data module, this hack avoid the need to pass multipart header manually - if (!headers.has('content-type') && options.body && typeof options.body.getBoundary === 'function') { - headers.set('content-type', `multipart/form-data; boundary=${options.body.getBoundary()}`); - } - // bring node-fetch closer to browser behavior by setting content-length automatically if (!headers.has('content-length') && /post|put|patch|delete/i.test(options.method)) { if (typeof options.body === 'string') { diff --git a/src/request.js b/src/request.js index 136570ac3..0223a4599 100644 --- a/src/request.js +++ b/src/request.js @@ -7,7 +7,7 @@ import { format as format_url, parse as parse_url } from 'url'; import Headers from './headers.js'; -import Body, { clone } from './body'; +import Body, { clone, extractContentType } from './body'; /** * Request class @@ -46,6 +46,13 @@ export default class Request extends Body { this.redirect = init.redirect || input.redirect || 'follow'; this.headers = new Headers(init.headers || input.headers || {}); + if (init.body) { + const contentType = extractContentType(this); + if (contentType && !this.headers.has('Content-Type')) { + this.headers.append('Content-Type', contentType); + } + } + // server only options this.follow = init.follow !== undefined ? init.follow : input.follow !== undefined ? From 70f61e0c7dfb2d7fcfdc741a7d02403e9ddcf86e Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Wed, 23 Nov 2016 13:39:35 -0800 Subject: [PATCH 045/223] Split http.request options generation --- src/body.js | 17 ++++++++ src/index.js | 106 ++++++++++++++++--------------------------------- src/request.js | 46 +++++++++++++++++++-- 3 files changed, 93 insertions(+), 76 deletions(-) diff --git a/src/body.js b/src/body.js index a198c6cec..9f3e1f1eb 100644 --- a/src/body.js +++ b/src/body.js @@ -269,5 +269,22 @@ export function extractContentType(instance) { } } +export function getTotalBytes(instance) { + const {body} = instance; + + if (typeof body === 'string') { + return Buffer.byteLength(body); + } else if (body && typeof body.getLengthSync === 'function') { + // detect form data input from form-data module + if (body._lengthRetrievers && body._lengthRetrievers.length == 0 || // 1.x + body.hasKnownLength && body.hasKnownLength()) { // 2.x + return body.getLengthSync(); + } + } else if (body === undefined || body === null) { + // this is only necessary for older nodejs releases (before iojs merge) + return 0; + } +} + // expose Promise Body.Promise = global.Promise; diff --git a/src/index.js b/src/index.js index fc4799334..cbb966acf 100644 --- a/src/index.js +++ b/src/index.js @@ -14,7 +14,7 @@ import {PassThrough} from 'stream'; import Body from './body'; import Response from './response'; import Headers from './headers'; -import Request from './request'; +import Request, { getNodeRequestOptions } from './request'; import FetchError from './fetch-error'; /** @@ -37,7 +37,9 @@ function fetch(url, opts) { // wrap http.request into fetch return new fetch.Promise((resolve, reject) => { // build request object - const options = new Request(url, opts); + const request = new Request(url, opts); + + const options = getNodeRequestOptions(request); if (!options.protocol || !options.hostname) { throw new Error('only absolute urls are supported'); @@ -49,46 +51,6 @@ function fetch(url, opts) { const send = (options.protocol === 'https:' ? https : http).request; - // normalize headers - const headers = new Headers(options.headers); - - if (options.compress) { - headers.set('accept-encoding', 'gzip,deflate'); - } - - if (!headers.has('user-agent')) { - headers.set('user-agent', 'node-fetch/1.0 (+https://github.com/bitinn/node-fetch)'); - } - - if (!headers.has('connection') && !options.agent) { - headers.set('connection', 'close'); - } - - if (!headers.has('accept')) { - headers.set('accept', '*/*'); - } - - // bring node-fetch closer to browser behavior by setting content-length automatically - if (!headers.has('content-length') && /post|put|patch|delete/i.test(options.method)) { - if (typeof options.body === 'string') { - headers.set('content-length', Buffer.byteLength(options.body)); - // detect form data input from form-data module, this hack avoid the need to add content-length header manually - } else if (options.body && typeof options.body.getLengthSync === 'function') { - // for form-data 1.x - if (options.body._lengthRetrievers && options.body._lengthRetrievers.length == 0) { - headers.set('content-length', options.body.getLengthSync().toString()); - // for form-data 2.x - } else if (options.body.hasKnownLength && options.body.hasKnownLength()) { - headers.set('content-length', options.body.getLengthSync().toString()); - } - // this is only necessary for older nodejs releases (before iojs merge) - } else if (options.body === undefined || options.body === null) { - headers.set('content-length', '0'); - } - } - - options.headers = headers.raw(); - // http.request only support string as host header, this hack make custom host header possible if (options.headers.host) { options.headers.host = options.headers.host[0]; @@ -98,52 +60,52 @@ function fetch(url, opts) { const req = send(options); let reqTimeout; - if (options.timeout) { + if (request.timeout) { req.once('socket', socket => { reqTimeout = setTimeout(() => { req.abort(); - reject(new FetchError(`network timeout at: ${options.url}`, 'request-timeout')); - }, options.timeout); + reject(new FetchError(`network timeout at: ${request.url}`, 'request-timeout')); + }, request.timeout); }); } req.on('error', err => { clearTimeout(reqTimeout); - reject(new FetchError(`request to ${options.url} failed, reason: ${err.message}`, 'system', err)); + reject(new FetchError(`request to ${request.url} failed, reason: ${err.message}`, 'system', err)); }); req.on('response', res => { clearTimeout(reqTimeout); // handle redirect - if (fetch.isRedirect(res.statusCode) && options.redirect !== 'manual') { - if (options.redirect === 'error') { - reject(new FetchError(`redirect mode is set to error: ${options.url}`, 'no-redirect')); + if (fetch.isRedirect(res.statusCode) && request.redirect !== 'manual') { + if (request.redirect === 'error') { + reject(new FetchError(`redirect mode is set to error: ${request.url}`, 'no-redirect')); return; } - if (options.counter >= options.follow) { - reject(new FetchError(`maximum redirect reached at: ${options.url}`, 'max-redirect')); + if (request.counter >= request.follow) { + reject(new FetchError(`maximum redirect reached at: ${request.url}`, 'max-redirect')); return; } if (!res.headers.location) { - reject(new FetchError(`redirect location header missing at: ${options.url}`, 'invalid-redirect')); + reject(new FetchError(`redirect location header missing at: ${request.url}`, 'invalid-redirect')); return; } // per fetch spec, for POST request with 301/302 response, or any request with 303 response, use GET when following redirect if (res.statusCode === 303 - || ((res.statusCode === 301 || res.statusCode === 302) && options.method === 'POST')) + || ((res.statusCode === 301 || res.statusCode === 302) && request.method === 'POST')) { - options.method = 'GET'; - delete options.body; - delete options.headers['content-length']; + request.method = 'GET'; + request.body = null; + request.headers.delete('content-length'); } - options.counter++; + request.counter++; - resolve(fetch(resolve_url(options.url, res.headers.location), options)); + resolve(fetch(resolve_url(request.url, res.headers.location), request)); return; } @@ -158,19 +120,19 @@ function fetch(url, opts) { headers.append(name, res.headers[name]); } } - if (options.redirect === 'manual' && headers.has('location')) { - headers.set('location', resolve_url(options.url, headers.get('location'))); + if (request.redirect === 'manual' && headers.has('location')) { + headers.set('location', resolve_url(request.url, headers.get('location'))); } // prepare response let body = res.pipe(new PassThrough()); const response_options = { - url: options.url + url: request.url , status: res.statusCode , statusText: res.statusMessage , headers: headers - , size: options.size - , timeout: options.timeout + , size: request.size + , timeout: request.timeout }; // response object @@ -182,7 +144,7 @@ function fetch(url, opts) { // 3. no content-encoding header // 4. no content response (204) // 5. content not modified response (304) - if (!options.compress || options.method === 'HEAD' || !headers.has('content-encoding') || res.statusCode === 204 || res.statusCode === 304) { + if (!request.compress || request.method === 'HEAD' || !headers.has('content-encoding') || res.statusCode === 204 || res.statusCode === 304) { output = new Response(body, response_options); resolve(output); return; @@ -224,16 +186,16 @@ function fetch(url, opts) { // accept string, buffer or readable stream as body // per spec we will call tostring on non-stream objects - if (typeof options.body === 'string') { - req.write(options.body); + if (typeof request.body === 'string') { + req.write(request.body); req.end(); - } else if (options.body instanceof Buffer) { - req.write(options.body); + } else if (request.body instanceof Buffer) { + req.write(request.body); req.end() - } else if (typeof options.body === 'object' && options.body.pipe) { - options.body.pipe(req); - } else if (typeof options.body === 'object') { - req.write(options.body.toString()); + } else if (typeof request.body === 'object' && request.body.pipe) { + request.body.pipe(req); + } else if (typeof request.body === 'object') { + req.write(request.body.toString()); req.end(); } else { req.end(); diff --git a/src/request.js b/src/request.js index 0223a4599..774dca7a7 100644 --- a/src/request.js +++ b/src/request.js @@ -7,7 +7,9 @@ import { format as format_url, parse as parse_url } from 'url'; import Headers from './headers.js'; -import Body, { clone, extractContentType } from './body'; +import Body, { clone, extractContentType, getTotalBytes } from './body'; + +const PARSED_URL = Symbol('url'); /** * Request class @@ -63,8 +65,7 @@ export default class Request extends Body { this.counter = init.counter || input.counter || 0; this.agent = init.agent || input.agent; - // server request options - Object.assign(this, parsedURL); + this[PARSED_URL] = parsedURL; Object.defineProperty(this, Symbol.toStringTag, { value: 'Request', @@ -75,7 +76,7 @@ export default class Request extends Body { } get url() { - return format_url(this); + return format_url(this[PARSED_URL]); } /** @@ -94,3 +95,40 @@ Object.defineProperty(Request.prototype, Symbol.toStringTag, { enumerable: false, configurable: true }); + +function normalizeHeaders(request) { + const headers = new Headers(request.headers); + + if (request.compress) { + headers.set('accept-encoding', 'gzip,deflate'); + } + + if (!headers.has('user-agent')) { + headers.set('user-agent', 'node-fetch/1.0 (+https://github.com/bitinn/node-fetch)'); + } + + if (!headers.has('connection') && !request.agent) { + headers.set('connection', 'close'); + } + + if (!headers.has('accept')) { + headers.set('accept', '*/*'); + } + + if (!headers.has('content-length') && /post|put|patch|delete/i.test(request.method)) { + const totalBytes = getTotalBytes(request); + if (typeof totalBytes === 'number') { + headers.set('content-length', totalBytes); + } + } + + return headers; +} + +export function getNodeRequestOptions(request) { + return Object.assign({}, request[PARSED_URL], { + method: request.method, + headers: normalizeHeaders(request).raw(), + agent: request.agent + }); +} From 3d676235a8045b406ce6e8af9d2bb600b627ed5b Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 4 Dec 2016 13:13:51 -0800 Subject: [PATCH 046/223] Throw when a GET/HEAD Request is created with body As mandated by the spec --- src/request.js | 11 +++++++++-- test/test.js | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/request.js b/src/request.js index 774dca7a7..d36edcd96 100644 --- a/src/request.js +++ b/src/request.js @@ -38,17 +38,24 @@ export default class Request extends Body { parsedURL = parse_url(input.url); } + let method = init.method || input.method || 'GET'; + + if ((init.body != null || input instanceof Request && input.body != null) && + (method === 'GET' || method === 'HEAD')) { + throw new TypeError('Request with GET/HEAD method cannot have body'); + } + super(init.body || clone(input), { timeout: init.timeout || input.timeout || 0, size: init.size || input.size || 0 }); // fetch spec options - this.method = init.method || input.method || 'GET'; + this.method = method; this.redirect = init.redirect || input.redirect || 'follow'; this.headers = new Headers(init.headers || input.headers || {}); - if (init.body) { + if (init.body != null) { const contentType = extractContentType(this); if (contentType && !this.headers.has('Content-Type')) { this.headers.append('Content-Type', contentType); diff --git a/test/test.js b/test/test.js index 27d74d2ab..1bfbf0a68 100644 --- a/test/test.js +++ b/test/test.js @@ -1444,6 +1444,17 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { }); }); + it('should throw error with GET/HEAD requests with body', function() { + expect(() => new Request('.', { body: '' })) + .to.throw(TypeError); + expect(() => new Request('.', { body: 'a' })) + .to.throw(TypeError); + expect(() => new Request('.', { body: '', method: 'HEAD' })) + .to.throw(TypeError); + expect(() => new Request('.', { body: 'a', method: 'HEAD' })) + .to.throw(TypeError); + }); + it('should support empty options in Response constructor', function() { let body = resumer().queue('a=1').end(); body = body.pipe(new stream.PassThrough()); @@ -1557,6 +1568,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { it('should support arrayBuffer() method in Request constructor', function() { url = base; var req = new Request(url, { + method: 'POST', body: 'a=1' }); expect(req.url).to.equal(url); @@ -1570,6 +1582,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { it('should support text() method in Request constructor', function() { url = base; const req = new Request(url, { + method: 'POST', body: 'a=1' }); expect(req.url).to.equal(url); @@ -1581,6 +1594,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { it('should support json() method in Request constructor', function() { url = base; const req = new Request(url, { + method: 'POST', body: '{"a":1}' }); expect(req.url).to.equal(url); @@ -1592,6 +1606,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { it('should support buffer() method in Request constructor', function() { url = base; const req = new Request(url, { + method: 'POST', body: 'a=1' }); expect(req.url).to.equal(url); From cc4ace1778a9235de38c3d5666668b79f25d69a9 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 4 Dec 2016 13:16:03 -0800 Subject: [PATCH 047/223] Make body default to null in Request Fixes #208. --- src/index.js | 6 +++--- src/request.js | 10 ++++++++-- test/test.js | 11 ++++++++--- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/index.js b/src/index.js index cbb966acf..9fb24403c 100644 --- a/src/index.js +++ b/src/index.js @@ -184,7 +184,7 @@ function fetch(url, opts) { return; }); - // accept string, buffer or readable stream as body + // accept string, buffer, readable stream or null as body // per spec we will call tostring on non-stream objects if (typeof request.body === 'string') { req.write(request.body); @@ -192,9 +192,9 @@ function fetch(url, opts) { } else if (request.body instanceof Buffer) { req.write(request.body); req.end() - } else if (typeof request.body === 'object' && request.body.pipe) { + } else if (request.body && typeof request.body === 'object' && request.body.pipe) { request.body.pipe(req); - } else if (typeof request.body === 'object') { + } else if (request.body && typeof request.body === 'object') { req.write(request.body.toString()); req.end(); } else { diff --git a/src/request.js b/src/request.js index d36edcd96..0241c702d 100644 --- a/src/request.js +++ b/src/request.js @@ -40,12 +40,18 @@ export default class Request extends Body { let method = init.method || input.method || 'GET'; - if ((init.body != null || input instanceof Request && input.body != null) && + if ((init.body != null || input instanceof Request && input.body !== null) && (method === 'GET' || method === 'HEAD')) { throw new TypeError('Request with GET/HEAD method cannot have body'); } - super(init.body || clone(input), { + let inputBody = init.body != null ? + init.body : + input instanceof Request && input.body !== null ? + clone(input) : + null; + + super(inputBody, { timeout: init.timeout || input.timeout || 0, size: init.size || input.size || 0 }); diff --git a/test/test.js b/test/test.js index 1bfbf0a68..892f1b4e4 100644 --- a/test/test.js +++ b/test/test.js @@ -1544,9 +1544,14 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { it('should default to null as body', function() { const res = new Response(); expect(res.body).to.equal(null); - return res.text().then(result => { - expect(result).to.equal(''); - }); + const req = new Request('.'); + expect(req.body).to.equal(null); + + const cb = result => expect(result).to.equal(''); + return Promise.all([ + res.text().then(cb), + req.text().then(cb) + ]); }); it('should default to 200 as status code', function() { From e7a13a5314e8bd3000e5fed8e42eebcab2d15512 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sat, 15 Oct 2016 14:21:33 -0700 Subject: [PATCH 048/223] Add support for blobs --- src/blob.js | 104 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/body.js | 22 +++++++++++ test/test.js | 59 ++++++++++++++++++++++++++++- 3 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 src/blob.js diff --git a/src/blob.js b/src/blob.js new file mode 100644 index 000000000..89e6ef2de --- /dev/null +++ b/src/blob.js @@ -0,0 +1,104 @@ +// Based on https://github.com/tmpvar/jsdom/blob/aa85b2abf07766ff7bf5c1f6daafb3726f2f2db5/lib/jsdom/living/blob.js +// (MIT licensed) + +export const BUFFER = Symbol('buffer'); +const TYPE = Symbol('type'); +const CLOSED = Symbol('closed'); + +export default class Blob { + constructor() { + Object.defineProperty(this, Symbol.toStringTag, { + value: 'Blob', + writable: false, + enumerable: false, + configurable: true + }); + + this[CLOSED] = false; + this[TYPE] = ''; + + const blobParts = arguments[0]; + const options = arguments[1]; + + const buffers = []; + + if (blobParts) { + const a = blobParts; + const length = Number(a.length); + for (let i = 0; i < length; i++) { + const element = a[i]; + let buffer; + if (element instanceof Buffer) { + buffer = element; + } else if (ArrayBuffer.isView(element)) { + buffer = new Buffer(new Uint8Array(element.buffer, element.byteOffset, element.byteLength)); + } else if (element instanceof ArrayBuffer) { + buffer = new Buffer(new Uint8Array(element)); + } else if (element instanceof Blob) { + buffer = element[BUFFER]; + } else { + buffer = new Buffer(typeof element === 'string' ? element : String(element)); + } + buffers.push(buffer); + } + } + + this[BUFFER] = Buffer.concat(buffers); + + let type = options && options.type !== undefined && String(options.type).toLowerCase(); + if (type && !/[^\u0020-\u007E]/.test(type)) { + this[TYPE] = type; + } + } + get size() { + return this[CLOSED] ? 0 : this[BUFFER].length; + } + get type() { + return this[TYPE]; + } + get isClosed() { + return this[CLOSED]; + } + slice() { + const size = this.size; + + const start = arguments[0]; + const end = arguments[1]; + let relativeStart, relativeEnd; + if (start === undefined) { + relativeStart = 0; + } else if (start < 0) { + relativeStart = Math.max(size + start, 0); + } else { + relativeStart = Math.min(start, size); + } + if (end === undefined) { + relativeEnd = size; + } else if (end < 0) { + relativeEnd = Math.max(size + end, 0); + } else { + relativeEnd = Math.min(end, size); + } + const span = Math.max(relativeEnd - relativeStart, 0); + + const buffer = this[BUFFER]; + const slicedBuffer = buffer.slice( + relativeStart, + relativeStart + span + ); + const blob = new Blob([], { type: arguments[2] }); + blob[BUFFER] = slicedBuffer; + blob[CLOSED] = this[CLOSED]; + return blob; + } + close() { + this[CLOSED] = true; + } +} + +Object.defineProperty(Blob.prototype, Symbol.toStringTag, { + value: 'BlobPrototype', + writable: false, + enumerable: false, + configurable: true +}); diff --git a/src/body.js b/src/body.js index 9f3e1f1eb..0ff922ea1 100644 --- a/src/body.js +++ b/src/body.js @@ -9,6 +9,7 @@ import {convert} from 'encoding'; import bodyStream from 'is-stream'; import toArrayBuffer from 'buffer-to-arraybuffer'; import {PassThrough} from 'stream'; +import Blob, {BUFFER} from './blob.js'; import FetchError from './fetch-error.js'; const DISTURBED = Symbol('disturbed'); @@ -26,6 +27,9 @@ export default class Body { size = 0, timeout = 0 } = {}) { + if (body instanceof Blob) { + body = body[BUFFER]; + } this.body = body; this[DISTURBED] = false; this.size = size; @@ -45,6 +49,24 @@ export default class Body { return this[CONSUME_BODY]().then(buf => toArrayBuffer(buf)); } + /** + * Return raw response as Blob + * + * @return Promise + */ + blob() { + let ct = this.headers && this.headers.get('content-type') || ''; + return this[CONSUME_BODY]().then(buf => Object.assign( + // Prevent copying + new Blob([], { + type: ct.toLowerCase() + }), + { + [BUFFER]: buf + } + )); + } + /** * Decode response as json * diff --git a/test/test.js b/test/test.js index 892f1b4e4..4204e1958 100644 --- a/test/test.js +++ b/test/test.js @@ -26,6 +26,7 @@ import Headers from '../src/headers.js'; import Response from '../src/response.js'; import Request from '../src/request.js'; import Body from '../src/body.js'; +import Blob from '../src/blob.js'; import FetchError from '../src/fetch-error.js'; // test with native promise on node 0.11, and bluebird for node 0.10 fetch.Promise = fetch.Promise || bluebird; @@ -1398,6 +1399,20 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { }); }); + it('should support blob round-trip', function() { + url = `${base}hello`; + + return fetch(url).then(res => res.blob()).then(blob => { + url = `${base}inspect`; + return fetch(url, { + method: 'POST', + body: blob + }); + }).then(res => res.json()).then(({body}) => { + expect(body).to.equal('world'); + }); + }); + it('should support wrapping Request instance', function() { url = `${base}hello`; @@ -1494,6 +1509,25 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { }); }); + it('should support blob() method in Request constructor', function() { + const res = new Response('a=1', { + headers: { + 'Content-Type': 'text/plain' + } + }); + return res.blob().then(function(result) { + expect(result).to.be.an.instanceOf(Blob); + expect(result.isClosed).to.be.false; + expect(result.size).to.equal(3); + expect(result.type).to.equal('text/plain'); + + result.close(); + expect(result.isClosed).to.be.true; + expect(result.size).to.equal(0); + expect(result.type).to.equal('text/plain'); + }); + }); + it('should support clone() method in Response constructor', function() { let body = resumer().queue('a=1').end(); body = body.pipe(new stream.PassThrough()); @@ -1620,6 +1654,28 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { }); }); + it('should support blob() method in Request constructor', function() { + url = base; + var req = new Request(url, { + body: 'a=1', + headers: { + 'Content-Type': 'text/plain' + } + }); + expect(req.url).to.equal(url); + return req.blob().then(function(result) { + expect(result).to.be.an.instanceOf(Blob); + expect(result.isClosed).to.be.false; + expect(result.size).to.equal(3); + expect(result.type).to.equal('text/plain'); + + result.close(); + expect(result.isClosed).to.be.true; + expect(result.size).to.equal(0); + expect(result.type).to.equal('text/plain'); + }); + }); + it('should support arbitrary url in Request constructor', function() { url = 'anything'; const req = new Request(url); @@ -1660,9 +1716,10 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { }); }); - it('should support arrayBuffer(), text(), json() and buffer() method in Body constructor', function() { + it('should support arrayBuffer(), blob(), text(), json() and buffer() method in Body constructor', function() { const body = new Body('a=1'); expect(body).to.have.property('arrayBuffer'); + expect(body).to.have.property('blob'); expect(body).to.have.property('text'); expect(body).to.have.property('json'); expect(body).to.have.property('buffer'); From 4d944365dfeff3ab0cb8b978c1e6fa9e3b640e2e Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Mon, 5 Dec 2016 15:21:19 -0800 Subject: [PATCH 049/223] Fix tests added in the last commit --- test/test.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/test.js b/test/test.js index 4204e1958..2512308fc 100644 --- a/test/test.js +++ b/test/test.js @@ -1509,8 +1509,9 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { }); }); - it('should support blob() method in Request constructor', function() { + it('should support blob() method in Response constructor', function() { const res = new Response('a=1', { + method: 'POST', headers: { 'Content-Type': 'text/plain' } @@ -1657,6 +1658,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { it('should support blob() method in Request constructor', function() { url = base; var req = new Request(url, { + method: 'POST', body: 'a=1', headers: { 'Content-Type': 'text/plain' From 552c1a601de6ed62175165d08b65a7d8c8b08438 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Mon, 5 Dec 2016 18:46:02 -0800 Subject: [PATCH 050/223] Bring coverage up to 100% --- .babelrc | 7 ++++++- src/headers.js | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.babelrc b/.babelrc index 50318ff6d..3cd9d25c1 100644 --- a/.babelrc +++ b/.babelrc @@ -13,7 +13,12 @@ [ "es2015", { "loose": true } ] ], "plugins": [ - "istanbul" + [ "istanbul", { + "exclude": [ + "src/blob.js", + "test" + ] + } ] ] }, "rollup": { diff --git a/src/headers.js b/src/headers.js index 1edf19066..eb47f1c71 100644 --- a/src/headers.js +++ b/src/headers.js @@ -245,6 +245,7 @@ function createHeadersIterator(target, kind) { const HeadersIteratorPrototype = Object.setPrototypeOf({ next() { + // istanbul ignore if if (!this || Object.getPrototypeOf(this) !== HeadersIteratorPrototype) { throw new TypeError('Value of `this` is not a HeadersIterator'); From 385ca6b2b010f3bcca6557de64536f2718d97030 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Mon, 5 Dec 2016 19:35:23 -0800 Subject: [PATCH 051/223] To 100% branches coverage --- package.json | 1 + test/test.js | 59 ++++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index d1ff6693b..ecae2d3ca 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "chai": "^3.5.0", "chai-as-promised": "^6.0.0", "chai-iterator": "^1.1.1", + "chai-string": "^1.3.0", "codecov": "^1.0.1", "cross-env": "2.0.1", "form-data": ">=1.0.0", diff --git a/test/test.js b/test/test.js index 2512308fc..97ebb700e 100644 --- a/test/test.js +++ b/test/test.js @@ -4,6 +4,7 @@ import repeat from 'babel-runtime/core-js/string/repeat'; import chai from 'chai'; import chaiPromised from 'chai-as-promised'; import chaiIterator from 'chai-iterator'; +import chaiString from 'chai-string'; import bluebird from 'bluebird'; import then from 'promise'; import {spawn} from 'child_process'; @@ -16,6 +17,7 @@ import * as fs from 'fs'; chai.use(chaiPromised); chai.use(chaiIterator); +chai.use(chaiString); const expect = chai.expect; import TestServer from './server'; @@ -630,6 +632,44 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { }); }); + it('should set default User-Agent', function () { + url = `${base}inspect`; + fetch(url).then(res => res.json()).then(res => { + expect(res.headers['user-agent']).to.startWith('node-fetch/'); + }); + }); + + it('should allow setting User-Agent', function () { + url = `${base}inspect`; + opts = { + headers: { + 'user-agent': 'faked' + } + }; + fetch(url, opts).then(res => res.json()).then(res => { + expect(res.headers['user-agent']).to.equal('faked'); + }); + }); + + it('should set default Accept header', function () { + url = `${base}inspect`; + fetch(url).then(res => res.json()).then(res => { + expect(res.headers.accept).to.equal('*/*'); + }); + }); + + it('should allow setting Accept header', function () { + url = `${base}inspect`; + opts = { + headers: { + 'accept': 'application/json' + } + }; + fetch(url, opts).then(res => res.json()).then(res => { + expect(res.headers.accept).to.equal('application/json'); + }); + }); + it('should allow POST request', function() { url = `${base}inspect`; opts = { @@ -640,6 +680,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { }).then(res => { expect(res.method).to.equal('POST'); expect(res.headers['transfer-encoding']).to.be.undefined; + expect(res.headers['content-type']).to.be.undefined; expect(res.headers['content-length']).to.equal('0'); }); }); @@ -656,6 +697,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { expect(res.method).to.equal('POST'); expect(res.body).to.equal('a=1'); expect(res.headers['transfer-encoding']).to.be.undefined; + expect(res.headers['content-type']).to.equal('text/plain;charset=UTF-8'); expect(res.headers['content-length']).to.equal('3'); }); }); @@ -672,6 +714,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { expect(res.method).to.equal('POST'); expect(res.body).to.equal('a=1'); expect(res.headers['transfer-encoding']).to.equal('chunked'); + expect(res.headers['content-type']).to.be.undefined; expect(res.headers['content-length']).to.be.undefined; }); }); @@ -691,6 +734,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { expect(res.method).to.equal('POST'); expect(res.body).to.equal('a=1'); expect(res.headers['transfer-encoding']).to.equal('chunked'); + expect(res.headers['content-type']).to.be.undefined; expect(res.headers['content-length']).to.be.undefined; }); }); @@ -708,7 +752,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { return res.json(); }).then(res => { expect(res.method).to.equal('POST'); - expect(res.headers['content-type']).to.contain('multipart/form-data'); + expect(res.headers['content-type']).to.startWith('multipart/form-data;boundary='); expect(res.headers['content-length']).to.be.a('string'); expect(res.body).to.equal('a=1'); }); @@ -728,7 +772,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { return res.json(); }).then(res => { expect(res.method).to.equal('POST'); - expect(res.headers['content-type']).to.contain('multipart/form-data'); + expect(res.headers['content-type']).to.startWith('multipart/form-data;boundary='); expect(res.headers['content-length']).to.be.undefined; expect(res.body).to.contain('my_field='); }); @@ -751,7 +795,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { return res.json(); }).then(res => { expect(res.method).to.equal('POST'); - expect(res.headers['content-type']).to.contain('multipart/form-data'); + expect(res.headers['content-type']).to.startWith('multipart/form-data; boundary='); expect(res.headers['content-length']).to.be.a('string'); expect(res.headers.b).to.equal('2'); expect(res.body).to.equal('a=1'); @@ -1659,22 +1703,19 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { url = base; var req = new Request(url, { method: 'POST', - body: 'a=1', - headers: { - 'Content-Type': 'text/plain' - } + body: new Buffer('a=1') }); expect(req.url).to.equal(url); return req.blob().then(function(result) { expect(result).to.be.an.instanceOf(Blob); expect(result.isClosed).to.be.false; expect(result.size).to.equal(3); - expect(result.type).to.equal('text/plain'); + expect(result.type).to.equal(''); result.close(); expect(result.isClosed).to.be.true; expect(result.size).to.equal(0); - expect(result.type).to.equal('text/plain'); + expect(result.type).to.equal(''); }); }); From a604069860d4629ff54cfaebce697e9ea968f34c Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Mon, 5 Dec 2016 20:25:13 -0800 Subject: [PATCH 052/223] More exact content-type and content-length Set content-type of requests with body being objects to text/plain --- src/body.js | 97 +++++++++++++++++++++++++++++++++++++++++++------- src/index.js | 19 ++-------- src/request.js | 2 +- test/test.js | 10 +++++- 4 files changed, 97 insertions(+), 31 deletions(-) diff --git a/src/body.js b/src/body.js index 0ff922ea1..890f2fca6 100644 --- a/src/body.js +++ b/src/body.js @@ -27,8 +27,21 @@ export default class Body { size = 0, timeout = 0 } = {}) { - if (body instanceof Blob) { - body = body[BUFFER]; + if (body == null) { + // body is undefined or null + body = null; + } else if (typeof body === 'string') { + // body is string + } else if (body instanceof Blob) { + // body is blob + } else if (Buffer.isBuffer(body)) { + // body is buffer + } else if (bodyStream(body)) { + // body is stream + } else { + // none of the above + // coerce to string + body = String(body); } this.body = body; this[DISTURBED] = false; @@ -117,7 +130,7 @@ export default class Body { this[DISTURBED] = true; // body is null - if (!this.body) { + if (this.body === null) { return Body.Promise.resolve(new Buffer(0)); } @@ -126,11 +139,21 @@ export default class Body { return Body.Promise.resolve(new Buffer(this.body)); } + // body is blob + if (this.body instanceof Blob) { + return Body.Promise.resolve(this.body[BUFFER]); + } + // body is buffer if (Buffer.isBuffer(this.body)) { return Body.Promise.resolve(this.body); } + // should never happen + if (!bodyStream(this.body)) { + return Body.Promise.resolve(new Buffer(0)); + } + // body is stream // get ready to actually consume the body let accum = []; @@ -281,30 +304,80 @@ export function clone(instance) { * @param Mixed instance Response or Request instance */ export function extractContentType(instance) { - // detect form data input from form-data module - if (typeof instance.body.getBoundary === 'function') { - return `multipart/form-data;boundary=${instance.body.getBoundary()}`; - } + const {body} = instance; - if (typeof instance.body === 'string') { + if (body === null) { + // body is null + return null; + } else if (typeof body === 'string') { + // body is string return 'text/plain;charset=UTF-8'; + } else if (body instanceof Blob) { + // body is blob + return body.type || null; + } else if (Buffer.isBuffer(body)) { + // body is buffer + return null; + } else if (typeof body.getBoundary === 'function') { + // detect form data input from form-data module + return `multipart/form-data;boundary=${body.getBoundary()}`; + } else { + // body is stream + // can't really do much about this + return null; } } export function getTotalBytes(instance) { const {body} = instance; - if (typeof body === 'string') { + if (body === null) { + // body is null + return 0; + } else if (typeof body === 'string') { + // body is string return Buffer.byteLength(body); + } else if (body instanceof Blob) { + // body is blob + return body.size; } else if (body && typeof body.getLengthSync === 'function') { // detect form data input from form-data module if (body._lengthRetrievers && body._lengthRetrievers.length == 0 || // 1.x body.hasKnownLength && body.hasKnownLength()) { // 2.x return body.getLengthSync(); } - } else if (body === undefined || body === null) { - // this is only necessary for older nodejs releases (before iojs merge) - return 0; + return null; + } else { + // body is stream + // can't really do much about this + return null; + } +} + +export function writeToStream(dest, instance) { + const {body} = instance; + + if (body === null) { + // body is null + dest.end(); + } else if (typeof body === 'string') { + // body is string + dest.write(body); + dest.end(); + } else if (body instanceof Blob) { + // body is blob + dest.write(body[BUFFER]); + dest.end(); + } else if (Buffer.isBuffer(body)) { + // body is buffer + dest.write(body); + dest.end() + } else if (bodyStream(body)) { + // body is stream + body.pipe(dest); + } else { + // should never happen + dest.end(); } } diff --git a/src/index.js b/src/index.js index 9fb24403c..e9f2d557b 100644 --- a/src/index.js +++ b/src/index.js @@ -11,7 +11,7 @@ import * as https from 'https'; import * as zlib from 'zlib'; import {PassThrough} from 'stream'; -import Body from './body'; +import Body, { writeToStream } from './body'; import Response from './response'; import Headers from './headers'; import Request, { getNodeRequestOptions } from './request'; @@ -184,22 +184,7 @@ function fetch(url, opts) { return; }); - // accept string, buffer, readable stream or null as body - // per spec we will call tostring on non-stream objects - if (typeof request.body === 'string') { - req.write(request.body); - req.end(); - } else if (request.body instanceof Buffer) { - req.write(request.body); - req.end() - } else if (request.body && typeof request.body === 'object' && request.body.pipe) { - request.body.pipe(req); - } else if (request.body && typeof request.body === 'object') { - req.write(request.body.toString()); - req.end(); - } else { - req.end(); - } + writeToStream(req, request); }); }; diff --git a/src/request.js b/src/request.js index 0241c702d..83fb33615 100644 --- a/src/request.js +++ b/src/request.js @@ -63,7 +63,7 @@ export default class Request extends Body { if (init.body != null) { const contentType = extractContentType(this); - if (contentType && !this.headers.has('Content-Type')) { + if (contentType !== null && !this.headers.has('Content-Type')) { this.headers.append('Content-Type', contentType); } } diff --git a/test/test.js b/test/test.js index 97ebb700e..2c0617655 100644 --- a/test/test.js +++ b/test/test.js @@ -814,6 +814,8 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { }).then(res => { expect(res.method).to.equal('POST'); expect(res.body).to.equal('[object Object]'); + expect(res.headers['content-type']).to.equal('text/plain;charset=UTF-8'); + expect(res.headers['content-length']).to.equal('15'); }); }); @@ -1446,14 +1448,20 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { it('should support blob round-trip', function() { url = `${base}hello`; + let length, type; + return fetch(url).then(res => res.blob()).then(blob => { url = `${base}inspect`; + length = blob.size; + type = blob.type; return fetch(url, { method: 'POST', body: blob }); - }).then(res => res.json()).then(({body}) => { + }).then(res => res.json()).then(({body, headers}) => { expect(body).to.equal('world'); + expect(headers['content-type']).to.equal(type); + expect(headers['content-length']).to.equal(String(length)); }); }); From 90d3bc443669b8a1cfe57e0972992a110c7413de Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Mon, 5 Dec 2016 20:30:00 -0800 Subject: [PATCH 053/223] Set content-length for buffer bodies --- src/body.js | 3 +++ test/test.js | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/body.js b/src/body.js index 890f2fca6..edd15edc2 100644 --- a/src/body.js +++ b/src/body.js @@ -340,6 +340,9 @@ export function getTotalBytes(instance) { } else if (body instanceof Blob) { // body is blob return body.size; + } else if (Buffer.isBuffer(body)) { + // body is buffer + return body.length; } else if (body && typeof body.getLengthSync === 'function') { // detect form data input from form-data module if (body._lengthRetrievers && body._lengthRetrievers.length == 0 || // 1.x diff --git a/test/test.js b/test/test.js index 2c0617655..11f5eb6b8 100644 --- a/test/test.js +++ b/test/test.js @@ -713,9 +713,9 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { }).then(res => { expect(res.method).to.equal('POST'); expect(res.body).to.equal('a=1'); - expect(res.headers['transfer-encoding']).to.equal('chunked'); + expect(res.headers['transfer-encoding']).to.be.undefined; expect(res.headers['content-type']).to.be.undefined; - expect(res.headers['content-length']).to.be.undefined; + expect(res.headers['content-length']).to.equal('3'); }); }); From 4a153ff0042c0e4dcdb7df9398c965ba5695337c Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Mon, 5 Dec 2016 20:40:23 -0800 Subject: [PATCH 054/223] Update changelog See #209. --- CHANGELOG.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 857fc8d49..20659abd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,33 @@ Changelog ========= +# 2.x release + +## v2.0.0-alpha.1 (UNRELEASED) + +- Major: Node.js 0.10.x support is dropped +- Major: rewrite in transpiled ES2015 +- Major: internal methods are no longer exposed +- Major: throw error when a GET/HEAD Request is constructed with a non-null body (per spec) +- Major: `response.text()` no longer attempts to detect encoding, instead always opting for UTF-8 (per spec); use `response.textConverted()` for the old behavior +- Major: make `response.json()` throw error instead of returning an empty object on 204 no-content respose (per spec; reverts behavior set in v1.6.2) +- Major: arrays as parameters to `headers.append` and `headers.set` are joined as a string (per spec) +- Enhance: start testing on Node.js 4, 6, 7 +- Enhance: use Rollup to produce a distributed bundle (less memory overhead and faster startup) +- Enhance: make `toString()` on Headers, Requests, and Responses return correct IDL class strings +- Enhance: add an option to conform to latest spec at the expense of reduced compatibility +- Enhance: set `Content-Length` header for Buffers as well +- Enhance: add `response.arrayBuffer()` (also applies to Requests) +- Enhance: add experimental `response.blob()` (also applies to Requests) +- Enhance: make Headers iterable +- Enhance: make Headers constructor accept an array of tuples +- Enhance: make sure header names and values are valid in HTTP +- Fix: coerce Headers prototype function parameters to strings, where applicable +- Fix: fix Request and Response with `null` body +- Fix: support WHATWG URL objects, created by `whatwg-url` package or `require('url').URL` in Node.js 7+ +- Other: use Codecov for code coverage tracking + + # 1.x release ## v1.6.3 From 0f7e6c15d3ec340b4e769ab48b421fa2b3393e2d Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Mon, 5 Dec 2016 21:09:54 -0800 Subject: [PATCH 055/223] Back to 100% --- src/body.js | 9 ++++----- test/test.js | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/src/body.js b/src/body.js index edd15edc2..c017b69de 100644 --- a/src/body.js +++ b/src/body.js @@ -149,7 +149,7 @@ export default class Body { return Body.Promise.resolve(this.body); } - // should never happen + // istanbul ignore if: should never happen if (!bodyStream(this.body)) { return Body.Promise.resolve(new Buffer(0)); } @@ -306,6 +306,8 @@ export function clone(instance) { export function extractContentType(instance) { const {body} = instance; + // istanbul ignore if: Currently, because of a guard in Request, body + // can never be null. Included here for completeness. if (body === null) { // body is null return null; @@ -375,12 +377,9 @@ export function writeToStream(dest, instance) { // body is buffer dest.write(body); dest.end() - } else if (bodyStream(body)) { + } else { // body is stream body.pipe(dest); - } else { - // should never happen - dest.end(); } } diff --git a/test/test.js b/test/test.js index 11f5eb6b8..1c2604430 100644 --- a/test/test.js +++ b/test/test.js @@ -719,6 +719,42 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { }); }); + it('should allow POST request with blob body without type', function() { + url = `${base}inspect`; + opts = { + method: 'POST' + , body: new Blob(['a=1']) + }; + return fetch(url, opts).then(res => { + return res.json(); + }).then(res => { + expect(res.method).to.equal('POST'); + expect(res.body).to.equal('a=1'); + expect(res.headers['transfer-encoding']).to.be.undefined; + expect(res.headers['content-type']).to.be.undefined; + expect(res.headers['content-length']).to.equal('3'); + }); + }); + + it('should allow POST request with blob body with type', function() { + url = `${base}inspect`; + opts = { + method: 'POST', + body: new Blob(['a=1'], { + type: 'text/plain;charset=UTF-8' + }) + }; + return fetch(url, opts).then(res => { + return res.json(); + }).then(res => { + expect(res.method).to.equal('POST'); + expect(res.body).to.equal('a=1'); + expect(res.headers['transfer-encoding']).to.be.undefined; + expect(res.headers['content-type']).to.equal('text/plain;charset=utf-8'); + expect(res.headers['content-length']).to.equal('3'); + }); + }); + it('should allow POST request with readable stream as body', function() { let body = resumer().queue('a=1').end(); body = body.pipe(new stream.PassThrough()); @@ -1628,6 +1664,13 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { }); }); + it('should support blob as body in Response constructor', function() { + const res = new Response(new Blob(['a=1'])); + return res.text().then(result => { + expect(result).to.equal('a=1'); + }); + }); + it('should default to null as body', function() { const res = new Response(); expect(res.body).to.equal(null); From 030bf279438e203f8ff05cffaf8b8ed7e88dff77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jimmy=20Karl=20Roland=20W=C3=A4rting?= Date: Fri, 9 Dec 2016 04:18:06 +0100 Subject: [PATCH 056/223] Documentation update for v2 (#214) --- LIMITS.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/LIMITS.md b/LIMITS.md index d0d41fcbf..f49fe285d 100644 --- a/LIMITS.md +++ b/LIMITS.md @@ -2,7 +2,7 @@ Known differences ================= -*As of 1.x release* +*As of 2.x release* - Topics such as Cross-Origin, Content Security Policy, Mixed Content, Service Workers are ignored, given our server-side context. @@ -16,12 +16,10 @@ Known differences - Also, you can handle rejected fetch requests through checking `err.type` and `err.code`. -- Only support `res.text()`, `res.json()`, `res.buffer()` at the moment, until there are good use-cases for blob/arrayBuffer. +- Only support `res.text()`, `res.json()`, `res.blob()`, `res.arraybuffer()`, `res.buffer()` - There is currently no built-in caching, as server-side caching varies by use-cases. - Current implementation lacks server-side cookie store, you will need to extract `Set-Cookie` headers manually. - If you are using `res.clone()` and writing an isomorphic app, note that stream on Node.js have a smaller internal buffer size (16Kb, aka `highWaterMark`) from client-side browsers (>1Mb, not consistent across browsers). - -- ES6 features such as `headers.entries()` are missing at the moment, but you can use `headers.raw()` to retrieve the raw headers object. From 9351084a98f15f1f412c2a84d193bf27a2a35244 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Mon, 12 Dec 2016 12:54:53 -0800 Subject: [PATCH 057/223] Update README for ES2015 --- README.md | 110 +++++++++++++++++++++++------------------------------- 1 file changed, 46 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index ecb5b816d..02cf186dd 100644 --- a/README.md +++ b/README.md @@ -41,44 +41,40 @@ See Matt Andrews' [isomorphic-fetch](https://github.com/matthew-andrews/isomorph # Usage ```javascript -var fetch = require('node-fetch'); +import fetch from 'node-fetch'; +// or +// const fetch = require('node-fetch'); -// if you are on node v0.10, set a Promise library first, eg. -// fetch.Promise = require('bluebird'); +// if you are using your own Promise library, set it through fetch.Promise. Eg. + +// import Bluebird from 'bluebird'; +// fetch.Promise = Bluebird; // plain text or html fetch('https://github.com/') - .then(function(res) { - return res.text(); - }).then(function(body) { - console.log(body); - }); + .then(res => res.text()) + .then(body => console.log(body)); // json fetch('https://api.github.com/users/github') - .then(function(res) { - return res.json(); - }).then(function(json) { - console.log(json); - }); + .then(res => res.json()) + .then(json => console.log(json)); // catching network error // 3xx-5xx responses are NOT network errors, and should be handled in then() // you only need one catch() at the end of your promise chain fetch('http://domain.invalid/') - .catch(function(err) { - console.log(err); - }); + .catch(err => console.error(err)); // stream // the node.js way is to use stream when possible fetch('https://assets-cdn.github.com/images/modules/logos_page/Octocat.png') - .then(function(res) { - var dest = fs.createWriteStream('./octocat.png'); + .then(res => { + const dest = fs.createWriteStream('./octocat.png'); res.body.pipe(dest); }); @@ -86,18 +82,17 @@ fetch('https://assets-cdn.github.com/images/modules/logos_page/Octocat.png') // if you prefer to cache binary data in full, use buffer() // note that buffer() is a node-fetch only API -var fileType = require('file-type'); +import fileType from 'file-type'; + fetch('https://assets-cdn.github.com/images/modules/logos_page/Octocat.png') - .then(function(res) { - return res.buffer(); - }).then(function(buffer) { - fileType(buffer); - }); + .then(res => res.buffer()) + .then(buffer => fileType(buffer)) + .then(type => { /* ... */ }); // meta fetch('https://github.com/') - .then(function(res) { + .then(res => { console.log(res.ok); console.log(res.status); console.log(res.statusText); @@ -108,22 +103,17 @@ fetch('https://github.com/') // post fetch('http://httpbin.org/post', { method: 'POST', body: 'a=1' }) - .then(function(res) { - return res.json(); - }).then(function(json) { - console.log(json); - }); + .then(res => res.json()) + .then(json => console.log(json)); -// post with stream from resumer +// post with stream from file -var resumer = require('resumer'); -var stream = resumer().queue('a=1').end(); +import { createReadStream } from 'fs'; + +const stream = createReadStream('input.txt'); fetch('http://httpbin.org/post', { method: 'POST', body: stream }) - .then(function(res) { - return res.json(); - }).then(function(json) { - console.log(json); - }); + .then(res => res.json()) + .then(json => console.log(json)); // post with JSON @@ -133,45 +123,37 @@ fetch('http://httpbin.org/post', { body: JSON.stringify(body), headers: { 'Content-Type': 'application/json' }, }) - .then(function(res) { - return res.json(); - }).then(function(json) { - console.log(json); - }); + .then(res => res.json()) + .then(json => console.log(json)); // post with form-data (detect multipart) -var FormData = require('form-data'); -var form = new FormData(); +import FormData from 'form-data'; + +const form = new FormData(); form.append('a', 1); fetch('http://httpbin.org/post', { method: 'POST', body: form }) - .then(function(res) { - return res.json(); - }).then(function(json) { - console.log(json); - }); + .then(res => res.json()) + .then(json => console.log(json)); // post with form-data (custom headers) // note that getHeaders() is non-standard API -var FormData = require('form-data'); -var form = new FormData(); +import FormData from 'form-data'; + +const form = new FormData(); form.append('a', 1); fetch('http://httpbin.org/post', { method: 'POST', body: form, headers: form.getHeaders() }) - .then(function(res) { - return res.json(); - }).then(function(json) { - console.log(json); - }); + .then(res => res.json()) + .then(json => console.log(json)); -// node 0.12+, yield with co +// node 7+ with async function -var co = require('co'); -co(function *() { - var res = yield fetch('https://api.github.com/users/github'); - var json = yield res.json(); - console.log(res); -}); +(async function () { + const res = await fetch('https://api.github.com/users/github'); + const json = await res.json(); + console.log(json); +})(); ``` See [test cases](https://github.com/bitinn/node-fetch/blob/master/test/test.js) for more examples. From 29e9f5eef69caeeff4702056595fc55a61c8301b Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Mon, 12 Dec 2016 13:50:05 -0800 Subject: [PATCH 058/223] Add guide to upgrade to v2 --- UPGRADE-GUIDE.md | 82 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 UPGRADE-GUIDE.md diff --git a/UPGRADE-GUIDE.md b/UPGRADE-GUIDE.md new file mode 100644 index 000000000..79308f375 --- /dev/null +++ b/UPGRADE-GUIDE.md @@ -0,0 +1,82 @@ +# Upgrade to node-fetch v2 + +node-fetch v2 brings about many changes that increase the compliance of +WHATWG's Fetch Standard. However, many of these changes meant that apps written +for node-fetch v1 needs to be updated to work with node-fetch v2 and be +conformant with the Fetch Standard. + +## `.text()` no longer tries to detect encoding + +Currently, `response.text()` attempts to guess the text encoding of the input +material and decode it for the user. However, it runs counter to the Fetch +Standard which demands `.text()` to always use UTF-8. + +In "response" to that, we have changed `.text()` to use UTF-8. A new function +**`response.textConverted()`** is created that maintains the behavior of +`.text()` last year. + +## Internal methods hidden + +Currently, the user can access internal methods such as `_clone()`, +`_decode()`, and `_convert()` on the `response` object. While these methods +should never have been used, node-fetch v2 makes these functions completely +inaccessible, and may break your app. + +If you have a use case that requires these methods to be available, feel free +to file an issue and we will be happy to help you solve the problem. + +## Headers + +The `Headers` class has gotten a lot of updates to make it spec-compliant. + +```js +////////////////////////////////////////////////////////////////////////////// +// If you are using an object as the initializer, all arrays will be reduced +// to a string. +const headers = new Headers({ + 'Abc': 'string', + 'Multi': [ 'header1', 'header2' ] +}); + +// before after +headers.get('Multi') => headers.get('Multi') => + 'header1'; 'header1,header2'; +headers.getAll('Multi') => headers.getAll('Multi') => + [ 'header1', 'header2' ]; [ 'header1,header2' ]; + +// Instead, to preserve the older behavior, you can use the header pair array +// syntax. +const headers = new Headers([ + [ 'Abc', 'string' ], + [ 'Multi', 'header1' ], + [ 'Multi', 'header2' ] +]); + + +////////////////////////////////////////////////////////////////////////////// +// All method parameters are now stringified. +const headers = new Headers(); +headers.set('null-header', null); +headers.set('undefined', undefined); + +// before after +headers.get('null-header') headers.get('null-header') + => null => 'null' +headers.get(undefined) headers.get(undefined) + => throws => 'undefined' + + +////////////////////////////////////////////////////////////////////////////// +// Invalid HTTP header names and values are now rejected outright. +const headers = new Headers(); +headers.set('Héy', 'ok'); // now throws +headers.get('Héy'); // now throws +new Headers({ 'Héy': 'ok' }); // now throws +``` + +## 0.10.x support dropped + +If you are still using Node.js v0.10, upgrade ASAP. Not only has Node.js +dropped support for that release branch, it has become too much work for us to +maintain. Therefore, we have dropped official support for v0.10 (it may still +work but don't expect them to do so). From 01bf40d5a4114c3c84954b24e5af8f57d25330bb Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Wed, 14 Dec 2016 13:51:43 -0800 Subject: [PATCH 059/223] Slightly expand upgrade guide --- UPGRADE-GUIDE.md | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/UPGRADE-GUIDE.md b/UPGRADE-GUIDE.md index 79308f375..592b2edb8 100644 --- a/UPGRADE-GUIDE.md +++ b/UPGRADE-GUIDE.md @@ -1,13 +1,13 @@ # Upgrade to node-fetch v2 node-fetch v2 brings about many changes that increase the compliance of -WHATWG's Fetch Standard. However, many of these changes meant that apps written -for node-fetch v1 needs to be updated to work with node-fetch v2 and be -conformant with the Fetch Standard. +WHATWG's [Fetch Standard][whatwg-fetch]. However, many of these changes meant +that apps written for node-fetch v1 needs to be updated to work with node-fetch +v2 and be conformant with the Fetch Standard. ## `.text()` no longer tries to detect encoding -Currently, `response.text()` attempts to guess the text encoding of the input +In v1, `response.text()` attempts to guess the text encoding of the input material and decode it for the user. However, it runs counter to the Fetch Standard which demands `.text()` to always use UTF-8. @@ -17,22 +17,29 @@ In "response" to that, we have changed `.text()` to use UTF-8. A new function ## Internal methods hidden -Currently, the user can access internal methods such as `_clone()`, -`_decode()`, and `_convert()` on the `response` object. While these methods -should never have been used, node-fetch v2 makes these functions completely -inaccessible, and may break your app. +In v1, the user can access internal methods such as `_clone()`, `_decode()`, +and `_convert()` on the `response` object. While these methods should never +have been used, node-fetch v2 makes these functions completely inaccessible. +If your app makes use of these functions, it may break when upgrading to v2. If you have a use case that requires these methods to be available, feel free to file an issue and we will be happy to help you solve the problem. ## Headers -The `Headers` class has gotten a lot of updates to make it spec-compliant. +The main goal we have for the `Headers` class in v2 is to make it completely +spec-compliant. However, due to changes in the Fetch Standard itself, total +spec compliance would mean incompatibility with all current major browser +implementations. + +Therefore, in v2, only a limited set of changes was applied to preserve +compatibility with browsers by default. See [#181] for more information on why +a feature is enabled or disabled. ```js ////////////////////////////////////////////////////////////////////////////// -// If you are using an object as the initializer, all arrays will be reduced -// to a string. +// If you are using an object as the initializer, all values will be +// stringified. For arrays, the members will be joined with a comma. const headers = new Headers({ 'Abc': 'string', 'Multi': [ 'header1', 'header2' ] @@ -78,5 +85,11 @@ new Headers({ 'Héy': 'ok' }); // now throws If you are still using Node.js v0.10, upgrade ASAP. Not only has Node.js dropped support for that release branch, it has become too much work for us to -maintain. Therefore, we have dropped official support for v0.10 (it may still -work but don't expect them to do so). +maintain. Therefore, we have dropped official support for v0.10. + +That being said, node-fetch may still work with v0.10, but as we are not +actively trying to support that version, it is in the user's best interest to +upgrade. + +[whatwg-fetch]: https://fetch.spec.whatwg.org/ +[#181]: https://github.com/bitinn/node-fetch/issues/181 From 7f928254118d0ac87dae5f90a20161c6c84c9f6a Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Wed, 14 Dec 2016 13:54:31 -0800 Subject: [PATCH 060/223] Upgrade Rollup --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ecae2d3ca..d69dd4888 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "parted": "^0.1.1", "promise": "^7.1.1", "resumer": "0.0.0", - "rollup": "^0.36.4", + "rollup": "^0.37.0", "rollup-plugin-babel": "^2.6.1", "rollup-plugin-node-resolve": "^2.0.0", "whatwg-url": "^4.0.0" From fa225291280e161dedca5b923c73f82cda23c3fe Mon Sep 17 00:00:00 2001 From: Nathan Rajlich Date: Wed, 14 Dec 2016 14:09:27 -0800 Subject: [PATCH 061/223] proper stack first line for FetchError instances (#215) The `.stack` property gets cached in the `captureStackTrace()` call, so whatever is set as the `name` and `message` at that time will be used for the first line of the stack trace. Before this patch, FetchError's stack would just say "Error" as the first line. Now they correctly display the "${name}: ${message}" of the error instances. Test case included. Signed-off-by: Timothy Gu --- src/fetch-error.js | 6 +++--- test/test.js | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/fetch-error.js b/src/fetch-error.js index a41791955..4d4b9321c 100644 --- a/src/fetch-error.js +++ b/src/fetch-error.js @@ -16,9 +16,6 @@ export default function FetchError(message, type, systemError) { Error.call(this, message); - // hide custom error implementation details from end-users - Error.captureStackTrace(this, this.constructor); - this.message = message; this.type = type; @@ -27,7 +24,10 @@ export default function FetchError(message, type, systemError) { this.code = this.errno = systemError.code; } + // hide custom error implementation details from end-users + Error.captureStackTrace(this, this.constructor); } FetchError.prototype = Object.create(Error.prototype); +FetchError.prototype.constructor = FetchError; FetchError.prototype.name = 'FetchError'; diff --git a/test/test.js b/test/test.js index 1c2604430..ae4972b77 100644 --- a/test/test.js +++ b/test/test.js @@ -1819,7 +1819,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { expect(body).to.have.property('buffer'); }); - it('should create custom FetchError', function() { + it('should create custom FetchError', function funcName() { const systemError = new Error('system'); systemError.code = 'ESOMEERROR'; @@ -1831,6 +1831,8 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { expect(err.type).to.equal('test-error'); expect(err.code).to.equal('ESOMEERROR'); expect(err.errno).to.equal('ESOMEERROR'); + expect(err.stack).to.include('funcName') + .and.to.startWith(`${err.name}: ${err.message}`); }); it('should support https request', function() { From 79704910008a6833f8354420d5caf69d001dff29 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Wed, 14 Dec 2016 15:44:30 -0800 Subject: [PATCH 062/223] Use template literals in Request Closes #217. --- src/request.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/request.js b/src/request.js index 83fb33615..50d711768 100644 --- a/src/request.js +++ b/src/request.js @@ -31,7 +31,7 @@ export default class Request extends Body { parsedURL = parse_url(input.href); } else { // coerce input to a string before attempting to parse - parsedURL = parse_url(input + ''); + parsedURL = parse_url(`${input}`); } input = {}; } else { From 151de2bdfb39bc4786b7a07e1e116a5690df6b94 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sat, 14 Jan 2017 20:50:10 -0800 Subject: [PATCH 063/223] Use ES2015 export syntax (#212) * Implement Rollup's external module check * Use ES2015 export syntax More friendly to ES2015 environments. --- .babelrc | 12 +++++------ build/babel-plugin.js | 48 ++++++++++++++++++++++++++++++++++++++++++ build/rollup-plugin.js | 16 ++++++++++++++ package.json | 3 ++- rollup.config.js | 14 ++++++++++-- src/index.js | 12 +++++------ test/test.js | 18 ++++++++++------ 7 files changed, 101 insertions(+), 22 deletions(-) create mode 100644 build/babel-plugin.js create mode 100644 build/rollup-plugin.js diff --git a/.babelrc b/.babelrc index 3cd9d25c1..f2845f258 100644 --- a/.babelrc +++ b/.babelrc @@ -6,6 +6,10 @@ "test": { "presets": [ [ "es2015", { "loose": true } ] + ], + "plugins": [ + "transform-runtime", + "./build/babel-plugin" ] }, "coverage": { @@ -13,12 +17,8 @@ [ "es2015", { "loose": true } ] ], "plugins": [ - [ "istanbul", { - "exclude": [ - "src/blob.js", - "test" - ] - } ] + [ "istanbul", { "exclude": [ "src/blob.js", "build", "test" ] } ], + "./build/babel-plugin" ] }, "rollup": { diff --git a/build/babel-plugin.js b/build/babel-plugin.js new file mode 100644 index 000000000..08efac90b --- /dev/null +++ b/build/babel-plugin.js @@ -0,0 +1,48 @@ +// This Babel plugin makes it possible to do CommonJS-style function exports + +const walked = Symbol('walked'); + +module.exports = ({ types: t }) => ({ + visitor: { + Program: { + exit(program) { + if (program[walked]) { + return; + } + + for (let path of program.get('body')) { + if (path.isExpressionStatement()) { + const expr = path.get('expression'); + if (expr.isAssignmentExpression() && + expr.get('left').matchesPattern('exports.*')) { + const prop = expr.get('left').get('property'); + if (prop.isIdentifier({ name: 'default' })) { + program.unshiftContainer('body', [ + t.expressionStatement( + t.assignmentExpression('=', + t.identifier('exports'), + t.assignmentExpression('=', + t.memberExpression( + t.identifier('module'), t.identifier('exports') + ), + expr.node.right + ) + ) + ), + t.expressionStatement( + t.assignmentExpression('=', + expr.node.left, t.identifier('exports') + ) + ) + ]); + path.remove(); + } + } + } + } + + program[walked] = true; + } + } + } +}); diff --git a/build/rollup-plugin.js b/build/rollup-plugin.js new file mode 100644 index 000000000..411d677e9 --- /dev/null +++ b/build/rollup-plugin.js @@ -0,0 +1,16 @@ +export default function tweakDefault() { + return { + transformBundle: function (source) { + var lines = source.split('\n'); + for (var i = 0; i < lines.length; i++) { + var line = lines[i]; + var matches = /^exports\['default'] = (.*);$/.exec(line); + if (matches) { + lines[i] = 'module.exports = exports = ' + matches[1] + ';'; + break; + } + } + return lines.join('\n'); + } + }; +} diff --git a/package.json b/package.json index d69dd4888..84a890958 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lib/index.es.js" ], "scripts": { - "build": "rollup -c", + "build": "cross-env BABEL_ENV=rollup rollup -c", "prepublish": "npm run build", "test": "cross-env BABEL_ENV=test mocha --compilers js:babel-register test/test.js", "report": "cross-env BABEL_ENV=coverage nyc --reporter lcov --reporter text mocha -R spec test/test.js", @@ -43,6 +43,7 @@ "codecov": "^1.0.1", "cross-env": "2.0.1", "form-data": ">=1.0.0", + "is-builtin-module": "^1.0.0", "mocha": "^3.1.2", "nyc": "^10.0.0", "parted": "^0.1.1", diff --git a/rollup.config.js b/rollup.config.js index 4a57423de..3e3ed20b4 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -1,5 +1,7 @@ +import isBuiltin from 'is-builtin-module'; import babel from 'rollup-plugin-babel'; import resolve from 'rollup-plugin-node-resolve'; +import tweakDefault from './build/rollup-plugin'; process.env.BABEL_ENV = 'rollup'; @@ -8,10 +10,18 @@ export default { plugins: [ babel({ runtimeHelpers: true - }) + }), + tweakDefault() ], targets: [ { dest: 'lib/index.js', format: 'cjs' }, { dest: 'lib/index.es.js', format: 'es' } - ] + ], + external: function (id) { + if (isBuiltin(id)) { + return true; + } + id = id.split('/').slice(0, id[0] === '@' ? 2 : 1).join('/'); + return !!require('./package.json').dependencies[id]; + } }; diff --git a/src/index.js b/src/index.js index e9f2d557b..ef668b87b 100644 --- a/src/index.js +++ b/src/index.js @@ -24,7 +24,7 @@ import FetchError from './fetch-error'; * @param Object opts Fetch options * @return Promise */ -function fetch(url, opts) { +export default function fetch(url, opts) { // allow custom promise if (!fetch.Promise) { @@ -189,8 +189,6 @@ function fetch(url, opts) { }; -module.exports = fetch; - /** * Redirect code matching * @@ -210,6 +208,8 @@ fetch.Promise = global.Promise; * objects; existing objects are not affected. */ fetch.FOLLOW_SPEC = false; -fetch.Response = Response; -fetch.Headers = Headers; -fetch.Request = Request; +export { + Headers, + Request, + Response +}; diff --git a/test/test.js b/test/test.js index ae4972b77..df0c6ffc0 100644 --- a/test/test.js +++ b/test/test.js @@ -23,10 +23,14 @@ const expect = chai.expect; import TestServer from './server'; // test subjects -import fetch from '../src/index.js'; -import Headers from '../src/headers.js'; -import Response from '../src/response.js'; -import Request from '../src/request.js'; +import fetch, { + Headers, + Request, + Response +} from '../src/'; +import HeadersOrig from '../src/headers.js'; +import RequestOrig from '../src/request.js'; +import ResponseOrig from '../src/response.js'; import Body from '../src/body.js'; import Blob from '../src/blob.js'; import FetchError from '../src/fetch-error.js'; @@ -93,9 +97,9 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { }); it('should expose Headers, Response and Request constructors', function() { - expect(fetch.Headers).to.equal(Headers); - expect(fetch.Response).to.equal(Response); - expect(fetch.Request).to.equal(Request); + expect(Headers).to.equal(HeadersOrig); + expect(Response).to.equal(ResponseOrig); + expect(Request).to.equal(RequestOrig); }); (supportToString ? it : it.skip)('should support proper toString output for Headers, Response and Request objects', function() { From f198f937677bafa54eaaed4b5635410c456da4ae Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sat, 14 Jan 2017 20:56:26 -0800 Subject: [PATCH 064/223] test: remove fallbacks for Node.js 0.10 --- test/test.js | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/test/test.js b/test/test.js index df0c6ffc0..e58819617 100644 --- a/test/test.js +++ b/test/test.js @@ -46,7 +46,6 @@ try { const supportToString = ({ [Symbol.toStringTag]: 'z' }).toString() === '[object z]'; -const supportIterator = !!(global.Symbol && global.Symbol.iterator); const local = new TestServer(); const base = `http://${local.hostname}:${local.port}/`; @@ -1242,9 +1241,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { ['a', '1'] ]); headers.append('b', '3'); - if (supportIterator) { - expect(headers).to.be.iterable; - } + expect(headers).to.be.iterable; const result = []; for (let pair of headers) { @@ -1262,7 +1259,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { ]); }); - (supportIterator ? it : it.skip)('should allow iterating through all headers with entries()', function() { + it('should allow iterating through all headers with entries()', function() { const headers = new Headers([ ['b', '2'], ['c', '4'], @@ -1283,7 +1280,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { ]); }); - (supportIterator ? it : it.skip)('should allow iterating through all headers with keys()', function() { + it('should allow iterating through all headers with keys()', function() { const headers = new Headers([ ['b', '2'], ['c', '4'], @@ -1295,7 +1292,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { .and.to.iterate.over(Headers.FOLLOW_SPEC ? ['a', 'b', 'c'] : ['b', 'b', 'c', 'a']); }); - (supportIterator ? it : it.skip)('should allow iterating through all headers with values()', function() { + it('should allow iterating through all headers with values()', function() { const headers = new Headers([ ['b', '2'], ['c', '4'], From 502b6042081a6c564dd2125a1dd5c4e3864b99dd Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sat, 14 Jan 2017 21:11:30 -0800 Subject: [PATCH 065/223] Fix Headers iterable initializer handling --- src/headers.js | 10 ++++++++-- test/test.js | 22 ++++++++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/headers.js b/src/headers.js index eb47f1c71..020ca7054 100644 --- a/src/headers.js +++ b/src/headers.js @@ -44,10 +44,14 @@ export default class Headers { this.append(name, value); } } - } else if (Array.isArray(headers)) { + } else if (typeof headers === 'object' && headers[Symbol.iterator]) { // array of tuples for (let el of headers) { - if (!Array.isArray(el) || el.length !== 2) { + if (typeof el !== 'object' || !el[Symbol.iterator]) { + throw new TypeError('Header pairs must be an iterable object'); + } + el = Array.from(el); + if (el.length !== 2) { throw new TypeError('Header pairs must contain exactly two items'); } this.append(el[0], el[1]); @@ -59,6 +63,8 @@ export default class Headers { // will handle it. this.append(prop, headers[prop]); } + } else if (headers != null) { + throw new TypeError('Provided initializer must be an object'); } Object.defineProperty(this, Symbol.toStringTag, { diff --git a/test/test.js b/test/test.js index e58819617..129ed0397 100644 --- a/test/test.js +++ b/test/test.js @@ -1435,19 +1435,37 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { expect(h3Raw['b']).to.include('1'); }); - it('should accept headers as an array of tuples', function() { - const headers = new Headers([ + it('should accept headers as an iterable of tuples', function() { + let headers; + + headers = new Headers([ ['a', '1'], ['b', '2'], ['a', '3'] ]); expect(headers.getAll('a')).to.deep.equal(['1', '3']); expect(headers.getAll('b')).to.deep.equal(['2']); + + headers = new Headers([ + new Set(['a', '1']), + ['b', '2'], + new Map([['a', null], ['3', null]]).keys() + ]); + expect(headers.getAll('a')).to.deep.equal(['1', '3']); + expect(headers.getAll('b')).to.deep.equal(['2']); + + headers = new Headers(new Map([ + ['a', '1'], + ['b', '2'] + ])); + expect(headers.getAll('a')).to.deep.equal(['1']); + expect(headers.getAll('b')).to.deep.equal(['2']); }); it('should throw a TypeError if non-tuple exists in a headers initializer', function() { expect(() => new Headers([ ['b', '2', 'huh?'] ])).to.throw(TypeError); expect(() => new Headers([ 'b2' ])).to.throw(TypeError); + expect(() => new Headers('b2')).to.throw(TypeError); }); it('should support fetch with Request instance', function() { From 908f661664e51fd847a5f69eb31724bbf2b3724b Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sat, 14 Jan 2017 21:36:08 -0800 Subject: [PATCH 066/223] 2.0.0-alpha.1 Fixes #8. Fixes #209. Fixes #222. --- CHANGELOG.md | 4 +++- package.json | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20659abd0..5e2c4040e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,9 @@ Changelog # 2.x release -## v2.0.0-alpha.1 (UNRELEASED) +## v2.0.0-alpha.1 + +This is a major release. Check [our upgrade guide](https://github.com/bitinn/node-fetch/blob/master/UPGRADE-GUIDE.md) for an overview on some key differences between v1 and v2. - Major: Node.js 0.10.x support is dropped - Major: rewrite in transpiled ES2015 diff --git a/package.json b/package.json index 84a890958..8660cfb62 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "node-fetch", - "version": "1.6.3", - "description": "A light-weight module that brings window.fetch to node.js and io.js", + "version": "2.0.0-alpha.1", + "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", "jsnext:main": "lib/index.es.js", "files": [ From 7539115bccbabacb254e420e6f47186536703ec7 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 22 Jan 2017 17:08:11 -0800 Subject: [PATCH 067/223] Silent rollup warning --- rollup.config.js | 1 + 1 file changed, 1 insertion(+) diff --git a/rollup.config.js b/rollup.config.js index 3e3ed20b4..d2123b971 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -7,6 +7,7 @@ process.env.BABEL_ENV = 'rollup'; export default { entry: 'src/index.js', + exports: 'named', plugins: [ babel({ runtimeHelpers: true From 95bf356c4d4348b3d8a2812bca62e61b2d265340 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 22 Jan 2017 17:14:55 -0800 Subject: [PATCH 068/223] Use babel-preset-env --- .babelrc | 6 +++--- package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.babelrc b/.babelrc index f2845f258..9456f9e1e 100644 --- a/.babelrc +++ b/.babelrc @@ -5,7 +5,7 @@ "env": { "test": { "presets": [ - [ "es2015", { "loose": true } ] + [ "env", { "loose": true, "targets": { "node": 0.12 } } ] ], "plugins": [ "transform-runtime", @@ -14,7 +14,7 @@ }, "coverage": { "presets": [ - [ "es2015", { "loose": true } ] + [ "env", { "loose": true, "targets": { "node": 0.12 } } ] ], "plugins": [ [ "istanbul", { "exclude": [ "src/blob.js", "build", "test" ] } ], @@ -23,7 +23,7 @@ }, "rollup": { "presets": [ - [ "es2015", { "loose": true, "modules": false } ] + [ "env", { "loose": true, "targets": { "node": 0.12 }, "modules": false } ] ] } } diff --git a/package.json b/package.json index 8660cfb62..068b73b76 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "devDependencies": { "babel-plugin-istanbul": "^3.0.0", "babel-plugin-transform-runtime": "^6.15.0", - "babel-preset-es2015": "^6.16.0", + "babel-preset-env": "^1.1.8", "babel-register": "^6.16.3", "bluebird": "^3.3.4", "chai": "^3.5.0", From e9db8695232fa502557e0e710c4a02a52d50bc01 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Mon, 23 Jan 2017 07:54:28 -0800 Subject: [PATCH 069/223] Remove FOLLOW_SPEC option; make it the default behavior (#225) * Remove !FOLLOW_SPEC mode * Update UPGRADE-GUIDE * Add CHANGELOG entry --- CHANGELOG.md | 1 + UPGRADE-GUIDE.md | 42 ++++++++++++++----------- src/headers.js | 56 ++++++---------------------------- src/index.js | 10 ------ test/test.js | 79 +++++++++--------------------------------------- 5 files changed, 48 insertions(+), 140 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e2c4040e..011fe65ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ This is a major release. Check [our upgrade guide](https://github.com/bitinn/nod - Major: Node.js 0.10.x support is dropped - Major: rewrite in transpiled ES2015 - Major: internal methods are no longer exposed +- Major: remove `headers.getAll()`; make `get()` return all headers (per spec) - Major: throw error when a GET/HEAD Request is constructed with a non-null body (per spec) - Major: `response.text()` no longer attempts to detect encoding, instead always opting for UTF-8 (per spec); use `response.textConverted()` for the old behavior - Major: make `response.json()` throw error instead of returning an empty object on 204 no-content respose (per spec; reverts behavior set in v1.6.2) diff --git a/UPGRADE-GUIDE.md b/UPGRADE-GUIDE.md index 592b2edb8..3ea134248 100644 --- a/UPGRADE-GUIDE.md +++ b/UPGRADE-GUIDE.md @@ -28,36 +28,42 @@ to file an issue and we will be happy to help you solve the problem. ## Headers The main goal we have for the `Headers` class in v2 is to make it completely -spec-compliant. However, due to changes in the Fetch Standard itself, total -spec compliance would mean incompatibility with all current major browser -implementations. - -Therefore, in v2, only a limited set of changes was applied to preserve -compatibility with browsers by default. See [#181] for more information on why -a feature is enabled or disabled. +spec-compliant. ```js ////////////////////////////////////////////////////////////////////////////// -// If you are using an object as the initializer, all values will be -// stringified. For arrays, the members will be joined with a comma. +// `get()` now returns **all** headers, joined by a comma, instead of only the +// first one. Its original behavior can be emulated using +// `get().split(',')[0]`. + const headers = new Headers({ 'Abc': 'string', 'Multi': [ 'header1', 'header2' ] }); // before after +headers.get('Abc') => headers.get('Abc') => + 'string' 'string' headers.get('Multi') => headers.get('Multi') => 'header1'; 'header1,header2'; + headers.get('Multi').split(',')[0] => + 'header1'; + + +////////////////////////////////////////////////////////////////////////////// +// `getAll()` is removed. Its behavior in v1 can be emulated with +// `get().split(',')`. + +const headers = new Headers({ + 'Abc': 'string', + 'Multi': [ 'header1', 'header2' ] +}); + +// before after headers.getAll('Multi') => headers.getAll('Multi') => - [ 'header1', 'header2' ]; [ 'header1,header2' ]; - -// Instead, to preserve the older behavior, you can use the header pair array -// syntax. -const headers = new Headers([ - [ 'Abc', 'string' ], - [ 'Multi', 'header1' ], - [ 'Multi', 'header2' ] -]); + [ 'header1', 'header2' ]; throws ReferenceError + headers.get('Multi').split(',') => + [ 'header1', 'header2' ]; ////////////////////////////////////////////////////////////////////////////// diff --git a/src/headers.js b/src/headers.js index 020ca7054..39f839a54 100644 --- a/src/headers.js +++ b/src/headers.js @@ -23,8 +23,7 @@ function sanitizeValue(value) { return value; } -export const MAP = Symbol('map'); -const FOLLOW_SPEC = Symbol('followSpec'); +const MAP = Symbol('map'); export default class Headers { /** * Headers class @@ -34,17 +33,8 @@ export default class Headers { */ constructor(headers) { this[MAP] = Object.create(null); - this[FOLLOW_SPEC] = Headers.FOLLOW_SPEC; - // Headers - if (headers instanceof Headers) { - let init = headers.raw(); - for (let name of Object.keys(init)) { - for (let value of init[name]) { - this.append(name, value); - } - } - } else if (typeof headers === 'object' && headers[Symbol.iterator]) { + if (typeof headers === 'object' && headers[Symbol.iterator]) { // array of tuples for (let el of headers) { if (typeof el !== 'object' || !el[Symbol.iterator]) { @@ -87,21 +77,7 @@ export default class Headers { return null; } - return this[FOLLOW_SPEC] ? list.join(',') : list[0]; - } - - /** - * Return all header values given name - * - * @param String name Header name - * @return Array - */ - getAll(name) { - if (!this.has(name)) { - return []; - } - - return this[MAP][sanitizeName(name)]; + return list.join(','); } /** @@ -217,24 +193,12 @@ Object.defineProperty(Headers.prototype, Symbol.toStringTag, { }); function getHeaderPairs(headers, kind) { - if (headers[FOLLOW_SPEC]) { - const keys = Object.keys(headers[MAP]).sort(); - return keys.map( - kind === 'key' ? - k => [k] : - k => [k, headers.get(k)] - ); - } - - const values = []; - - for (let name in headers[MAP]) { - for (let value of headers[MAP][name]) { - values.push([name, value]); - } - } - - return values; + const keys = Object.keys(headers[MAP]).sort(); + return keys.map( + kind === 'key' ? + k => [k] : + k => [k, headers.get(k)] + ); } const INTERNAL = Symbol('internal'); @@ -305,5 +269,3 @@ Object.defineProperty(HeadersIteratorPrototype, Symbol.toStringTag, { enumerable: false, configurable: true }); - -Headers.FOLLOW_SPEC = false; diff --git a/src/index.js b/src/index.js index ef668b87b..ebd0dcbc7 100644 --- a/src/index.js +++ b/src/index.js @@ -32,7 +32,6 @@ export default function fetch(url, opts) { } Body.Promise = fetch.Promise; - Headers.FOLLOW_SPEC = fetch.FOLLOW_SPEC; // wrap http.request into fetch return new fetch.Promise((resolve, reject) => { @@ -199,15 +198,6 @@ fetch.isRedirect = code => code === 301 || code === 302 || code === 303 || code // expose Promise fetch.Promise = global.Promise; -/** - * Option to make newly constructed Headers objects conformant to the - * **latest** version of the Fetch Standard. Note, that most other - * implementations of fetch() have not yet been updated to the latest - * version, so enabling this option almost certainly breaks any isomorphic - * attempt. Also, changing this variable will only affect new Headers - * objects; existing objects are not affected. - */ -fetch.FOLLOW_SPEC = false; export { Headers, Request, diff --git a/test/test.js b/test/test.js index 129ed0397..0d5a365d5 100644 --- a/test/test.js +++ b/test/test.js @@ -59,16 +59,7 @@ after(done => { local.stop(done); }); -(runner => { - runner(false); - runner(true); -})(defaultFollowSpec => { - -describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { - before(() => { - fetch.FOLLOW_SPEC = Headers.FOLLOW_SPEC = defaultFollowSpec; - }); - +describe('node-fetch', () => { it('should return a promise', function() { url = 'http://example.com/'; const p = fetch(url); @@ -1199,11 +1190,9 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { it('should allow get all responses of a header', function() { url = `${base}cookie`; return fetch(url).then(res => { - const expected = fetch.FOLLOW_SPEC ? 'a=1,b=1' : 'a=1'; + const expected = 'a=1,b=1'; expect(res.headers.get('set-cookie')).to.equal(expected); expect(res.headers.get('Set-Cookie')).to.equal(expected); - expect(res.headers.getAll('set-cookie')).to.deep.equal(['a=1', 'b=1']); - expect(res.headers.getAll('Set-Cookie')).to.deep.equal(['a=1', 'b=1']); }); }); @@ -1221,17 +1210,11 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { result.push([key, val]); }); - const expected = Headers.FOLLOW_SPEC ? [ + expect(result).to.deep.equal([ ["a", "1"] , ["b", "2,3"] , ["c", "4"] - ] : [ - ["b", "2"] - , ["b", "3"] - , ["c", "4"] - , ["a", "1"] - ]; - expect(result).to.deep.equal(expected); + ]); }); it('should allow iterating through all headers with for-of loop', function() { @@ -1247,15 +1230,10 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { for (let pair of headers) { result.push(pair); } - expect(result).to.deep.equal(Headers.FOLLOW_SPEC ? [ + expect(result).to.deep.equal([ ['a', '1'], ['b', '2,3'], ['c', '4'] - ] : [ - ['b', '2'], - ['b', '3'], - ['c', '4'], - ['a', '1'], ]); }); @@ -1268,15 +1246,10 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { headers.append('b', '3'); expect(headers.entries()).to.be.iterable - .and.to.deep.iterate.over(Headers.FOLLOW_SPEC ? [ + .and.to.deep.iterate.over([ ['a', '1'], ['b', '2,3'], ['c', '4'] - ] : [ - ['b', '2'], - ['b', '3'], - ['c', '4'], - ['a', '1'], ]); }); @@ -1289,7 +1262,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { headers.append('b', '3'); expect(headers.keys()).to.be.iterable - .and.to.iterate.over(Headers.FOLLOW_SPEC ? ['a', 'b', 'c'] : ['b', 'b', 'c', 'a']); + .and.to.iterate.over(['a', 'b', 'c']); }); it('should allow iterating through all headers with values()', function() { @@ -1301,27 +1274,7 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { headers.append('b', '3'); expect(headers.values()).to.be.iterable - .and.to.iterate.over(Headers.FOLLOW_SPEC ? ['1', '2,3', '4'] : ['2', '3', '4', '1']); - }); - - it('should only apply FOLLOW_SPEC when it is requested', function () { - Headers.FOLLOW_SPEC = true; - - const src = [ - ['b', '2'], - ['b', '3'] - ]; - - let headers = new Headers(src); - expect(headers.get('b')).to.equal('2,3'); - - Headers.FOLLOW_SPEC = false; - expect(headers.get('b')).to.equal('2,3'); - - headers = new Headers(src); - expect(headers.get('b')).to.equal('2'); - - Headers.FOLLOW_SPEC = defaultFollowSpec; + .and.to.iterate.over(['1', '2,3', '4']); }); it('should allow deleting header', function() { @@ -1329,7 +1282,6 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { return fetch(url).then(res => { res.headers.delete('set-cookie'); expect(res.headers.get('set-cookie')).to.be.null; - expect(res.headers.getAll('set-cookie')).to.be.empty; }); }); @@ -1341,7 +1293,6 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { expect(() => headers.append('Hé-y', 'ok')) .to.throw(TypeError); expect(() => headers.delete('Hé-y')) .to.throw(TypeError); expect(() => headers.get('Hé-y')) .to.throw(TypeError); - expect(() => headers.getAll('Hé-y')) .to.throw(TypeError); expect(() => headers.has('Hé-y')) .to.throw(TypeError); expect(() => headers.set('Hé-y', 'ok')) .to.throw(TypeError); @@ -1443,23 +1394,23 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { ['b', '2'], ['a', '3'] ]); - expect(headers.getAll('a')).to.deep.equal(['1', '3']); - expect(headers.getAll('b')).to.deep.equal(['2']); + expect(headers.get('a')).to.equal('1,3'); + expect(headers.get('b')).to.equal('2'); headers = new Headers([ new Set(['a', '1']), ['b', '2'], new Map([['a', null], ['3', null]]).keys() ]); - expect(headers.getAll('a')).to.deep.equal(['1', '3']); - expect(headers.getAll('b')).to.deep.equal(['2']); + expect(headers.get('a')).to.equal('1,3'); + expect(headers.get('b')).to.equal('2'); headers = new Headers(new Map([ ['a', '1'], ['b', '2'] ])); - expect(headers.getAll('a')).to.deep.equal(['1']); - expect(headers.getAll('b')).to.deep.equal(['2']); + expect(headers.get('a')).to.equal('1'); + expect(headers.get('b')).to.equal('2'); }); it('should throw a TypeError if non-tuple exists in a headers initializer', function() { @@ -1868,8 +1819,6 @@ describe(`node-fetch with FOLLOW_SPEC = ${defaultFollowSpec}`, () => { }); -}); - function streamToPromise(stream, dataHandler) { return new Promise((resolve, reject) => { stream.on('data', (...args) => { From db733374dd09b97b5ba33d739cbb6356c15027a9 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sat, 28 Jan 2017 08:41:59 -0800 Subject: [PATCH 070/223] Fix CHANGELOG for v2.0.0-alpha.2 [ci skip] --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 011fe65ff..b2a9caa4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ Changelog # 2.x release +## v2.0.0-alpha.2 + +- Major: remove `headers.getAll()`; make `get()` return all headers delimited by commas (per spec) +- Major: remove undocumented `FOLLOW_SPEC` switch -- it is now the default + ## v2.0.0-alpha.1 This is a major release. Check [our upgrade guide](https://github.com/bitinn/node-fetch/blob/master/UPGRADE-GUIDE.md) for an overview on some key differences between v1 and v2. @@ -12,7 +17,6 @@ This is a major release. Check [our upgrade guide](https://github.com/bitinn/nod - Major: Node.js 0.10.x support is dropped - Major: rewrite in transpiled ES2015 - Major: internal methods are no longer exposed -- Major: remove `headers.getAll()`; make `get()` return all headers (per spec) - Major: throw error when a GET/HEAD Request is constructed with a non-null body (per spec) - Major: `response.text()` no longer attempts to detect encoding, instead always opting for UTF-8 (per spec); use `response.textConverted()` for the old behavior - Major: make `response.json()` throw error instead of returning an empty object on 204 no-content respose (per spec; reverts behavior set in v1.6.2) From ac8ddaccb81f896d2e6223fb9605fb5a30dd914a Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 29 Jan 2017 08:58:16 -0800 Subject: [PATCH 071/223] Improve Headers constructor argument processing --- src/headers.js | 51 ++++++++++++++++++++++++++++++++------------------ test/test.js | 1 + 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/src/headers.js b/src/headers.js index 39f839a54..dc31ee112 100644 --- a/src/headers.js +++ b/src/headers.js @@ -31,29 +31,44 @@ export default class Headers { * @param Object headers Response headers * @return Void */ - constructor(headers) { + constructor(init = undefined) { this[MAP] = Object.create(null); - if (typeof headers === 'object' && headers[Symbol.iterator]) { - // array of tuples - for (let el of headers) { - if (typeof el !== 'object' || !el[Symbol.iterator]) { - throw new TypeError('Header pairs must be an iterable object'); + // We don't worry about converting prop to ByteString here as append() + // will handle it. + if (init == null) { + // no op + } else if (typeof init === 'object') { + const method = init[Symbol.iterator]; + if (method != null) { + if (typeof method !== 'function') { + throw new TypeError('Header pairs must be iterable'); } - el = Array.from(el); - if (el.length !== 2) { - throw new TypeError('Header pairs must contain exactly two items'); + + // sequence> + // Note: per spec we have to first exhaust the lists then process them + const pairs = []; + for (const pair of init) { + if (typeof pair !== 'object' || typeof pair[Symbol.iterator] !== 'function') { + throw new TypeError('Each header pair must be iterable'); + } + pairs.push(Array.from(pair)); + } + + for (const pair of pairs) { + if (pair.length !== 2) { + throw new TypeError('Each header pair must be a name/value tuple'); + } + this.append(pair[0], pair[1]); + } + } else { + // record + for (const key of Object.keys(init)) { + const value = init[key]; + this.append(key, value); } - this.append(el[0], el[1]); - } - } else if (typeof headers === 'object') { - // plain object - for (const prop of Object.keys(headers)) { - // We don't worry about converting prop to ByteString here as append() - // will handle it. - this.append(prop, headers[prop]); } - } else if (headers != null) { + } else { throw new TypeError('Provided initializer must be an object'); } diff --git a/test/test.js b/test/test.js index 0d5a365d5..f62361644 100644 --- a/test/test.js +++ b/test/test.js @@ -1417,6 +1417,7 @@ describe('node-fetch', () => { expect(() => new Headers([ ['b', '2', 'huh?'] ])).to.throw(TypeError); expect(() => new Headers([ 'b2' ])).to.throw(TypeError); expect(() => new Headers('b2')).to.throw(TypeError); + expect(() => new Headers({ [Symbol.iterator]: 42 })).to.throw(TypeError); }); it('should support fetch with Request instance', function() { From 11a94814051919d7787494af0a0dbbbce6143f8f Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 29 Jan 2017 09:56:19 -0800 Subject: [PATCH 072/223] request: overwrite Content-Length if possible Also reorganize header normalization --- src/index.js | 4 ---- src/request.js | 53 +++++++++++++++++++++++++++++++------------------- test/test.js | 33 ++++++++++++++++++------------- 3 files changed, 52 insertions(+), 38 deletions(-) diff --git a/src/index.js b/src/index.js index ebd0dcbc7..fbd0cdebe 100644 --- a/src/index.js +++ b/src/index.js @@ -44,10 +44,6 @@ export default function fetch(url, opts) { throw new Error('only absolute urls are supported'); } - if (options.protocol !== 'http:' && options.protocol !== 'https:') { - throw new Error('only http(s) protocols are supported'); - } - const send = (options.protocol === 'https:' ? https : http).request; // http.request only support string as host header, this hack make custom host header possible diff --git a/src/request.js b/src/request.js index 50d711768..f9393cacb 100644 --- a/src/request.js +++ b/src/request.js @@ -57,7 +57,7 @@ export default class Request extends Body { }); // fetch spec options - this.method = method; + this.method = method.toUpperCase(); this.redirect = init.redirect || input.redirect || 'follow'; this.headers = new Headers(init.headers || input.headers || {}); @@ -79,7 +79,6 @@ export default class Request extends Body { this.agent = init.agent || input.agent; this[PARSED_URL] = parsedURL; - Object.defineProperty(this, Symbol.toStringTag, { value: 'Request', writable: false, @@ -109,39 +108,53 @@ Object.defineProperty(Request.prototype, Symbol.toStringTag, { configurable: true }); -function normalizeHeaders(request) { +export function getNodeRequestOptions(request) { const headers = new Headers(request.headers); - if (request.compress) { - headers.set('accept-encoding', 'gzip,deflate'); - } - - if (!headers.has('user-agent')) { - headers.set('user-agent', 'node-fetch/1.0 (+https://github.com/bitinn/node-fetch)'); + // fetch step 3 + if (!headers.has('Accept')) { + headers.set('Accept', '*/*'); } - if (!headers.has('connection') && !request.agent) { - headers.set('connection', 'close'); + // Basic fetch + if (!/^https?:$/.test(request[PARSED_URL].protocol)) { + throw new Error('only http(s) protocols are supported'); } - if (!headers.has('accept')) { - headers.set('accept', '*/*'); + // HTTP-network-or-cache fetch steps 5-9 + let contentLengthValue = null; + if (request.body == null && /^(POST|PUT)$/i.test(request.method)) { + contentLengthValue = '0'; } - - if (!headers.has('content-length') && /post|put|patch|delete/i.test(request.method)) { + if (request.body != null) { const totalBytes = getTotalBytes(request); if (typeof totalBytes === 'number') { - headers.set('content-length', totalBytes); + contentLengthValue = String(totalBytes); } } + if (contentLengthValue) { + headers.set('Content-Length', contentLengthValue); + } - return headers; -} + // HTTP-network-or-cache fetch step 12 + if (!headers.has('User-Agent')) { + headers.set('User-Agent', 'node-fetch/1.0 (+https://github.com/bitinn/node-fetch)'); + } + + // HTTP-network-or-cache fetch step 16 + if (request.compress) { + headers.set('Accept-Encoding', 'gzip,deflate'); + } + if (!headers.has('Connection') && !request.agent) { + headers.set('Connection', 'close'); + } + + // HTTP-network fetch step 4 + // chunked encoding is handled by Node.js -export function getNodeRequestOptions(request) { return Object.assign({}, request[PARSED_URL], { method: request.method, - headers: normalizeHeaders(request).raw(), + headers: headers.raw(), agent: request.agent }); } diff --git a/test/test.js b/test/test.js index f62361644..b57dad8e8 100644 --- a/test/test.js +++ b/test/test.js @@ -849,45 +849,50 @@ describe('node-fetch', () => { }); }); - it('should allow PUT request', function() { + it('should overwrite Content-Length if possible', function() { url = `${base}inspect`; + // note that fetch simply calls tostring on an object opts = { - method: 'PUT' - , body: 'a=1' + method: 'POST', + headers: { + 'Content-Length': '1000' + }, + body: 'a=1' }; return fetch(url, opts).then(res => { return res.json(); }).then(res => { - expect(res.method).to.equal('PUT'); + expect(res.method).to.equal('POST'); expect(res.body).to.equal('a=1'); + expect(res.headers['transfer-encoding']).to.be.undefined; + expect(res.headers['content-type']).to.equal('text/plain;charset=UTF-8'); + expect(res.headers['content-length']).to.equal('3'); }); }); - it('should allow DELETE request', function() { + it('should allow PUT request', function() { url = `${base}inspect`; opts = { - method: 'DELETE' + method: 'PUT' + , body: 'a=1' }; return fetch(url, opts).then(res => { return res.json(); }).then(res => { - expect(res.method).to.equal('DELETE'); + expect(res.method).to.equal('PUT'); + expect(res.body).to.equal('a=1'); }); }); - it('should allow POST request with string body', function() { + it('should allow DELETE request', function() { url = `${base}inspect`; opts = { - method: 'POST' - , body: 'a=1' + method: 'DELETE' }; return fetch(url, opts).then(res => { return res.json(); }).then(res => { - expect(res.method).to.equal('POST'); - expect(res.body).to.equal('a=1'); - expect(res.headers['transfer-encoding']).to.be.undefined; - expect(res.headers['content-length']).to.equal('3'); + expect(res.method).to.equal('DELETE'); }); }); From 560d5d6a023f069c37497f633fe65955182fa3d5 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 29 Jan 2017 10:34:42 -0800 Subject: [PATCH 073/223] 2.0.0-alpha.3 --- CHANGELOG.md | 5 +++++ package.json | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2a9caa4f..72ecf5112 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ Changelog # 2.x release +## v2.0.0-alpha.3 + +- Major: overwrite user's `Content-Length` if we can be sure our information is correct +- Fix: exhaust list in `Headers` constructor before processing + ## v2.0.0-alpha.2 - Major: remove `headers.getAll()`; make `get()` return all headers delimited by commas (per spec) diff --git a/package.json b/package.json index 068b73b76..edb770135 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,9 @@ { "name": "node-fetch", - "version": "2.0.0-alpha.1", + "version": "2.0.0-alpha.3", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", - "jsnext:main": "lib/index.es.js", + "module": "lib/index.es.js", "files": [ "lib/index.js", "lib/index.es.js" From 673ebb2089f3966abd8900a53fc07be2553d7989 Mon Sep 17 00:00:00 2001 From: Ahmad Nassri Date: Tue, 21 Feb 2017 15:38:54 -0500 Subject: [PATCH 074/223] docs(defaults): list of default header values (#237) Fixes #226. --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 02cf186dd..3d4ae4300 100644 --- a/README.md +++ b/README.md @@ -187,6 +187,18 @@ Note that only `method`, `headers`, `redirect` and `body` are allowed in `window } ``` +#### Default Headers + +If no values are set, the following request headers will be sent automatically: + +Header | Value +----------------- | -------------------------------------------------------- +`Accept-Encoding` | `gzip,deflate` _(when `options.compress === true`)_ +`Accept` | `*/*` +`Connection` | `close` _(when no `options.agent` is present)_ +`Content-Length` | _(automatically calculated, if possible)_ +`User-Agent` | `node-fetch/1.0 (+https://github.com/bitinn/node-fetch)` + # License From edb28a6c50da805dfe6e05894c2360eb88160cb5 Mon Sep 17 00:00:00 2001 From: Tino Vyatkin Date: Sun, 26 Feb 2017 17:17:47 -0400 Subject: [PATCH 075/223] Drop Node.js v0.12 support (#240) --- .babelrc | 6 +++--- .travis.yml | 1 - package.json | 5 ++++- src/headers.js | 9 +-------- 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/.babelrc b/.babelrc index 9456f9e1e..7b0e36281 100644 --- a/.babelrc +++ b/.babelrc @@ -5,7 +5,7 @@ "env": { "test": { "presets": [ - [ "env", { "loose": true, "targets": { "node": 0.12 } } ] + [ "env", { "loose": true, "targets": { "node": 4 } } ] ], "plugins": [ "transform-runtime", @@ -14,7 +14,7 @@ }, "coverage": { "presets": [ - [ "env", { "loose": true, "targets": { "node": 0.12 } } ] + [ "env", { "loose": true, "targets": { "node": 4 } } ] ], "plugins": [ [ "istanbul", { "exclude": [ "src/blob.js", "build", "test" ] } ], @@ -23,7 +23,7 @@ }, "rollup": { "presets": [ - [ "env", { "loose": true, "targets": { "node": 0.12 }, "modules": false } ] + [ "env", { "loose": true, "targets": { "node": 4 }, "modules": false } ] ] } } diff --git a/.travis.yml b/.travis.yml index 23a2e4adc..4a706efab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: node_js node_js: - - "0.12" - "4" - "6" - "node" diff --git a/package.json b/package.json index edb770135..b17c8d487 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,9 @@ "lib/index.js", "lib/index.es.js" ], + "engines": { + "node": ">=4" + }, "scripts": { "build": "cross-env BABEL_ENV=rollup rollup -c", "prepublish": "npm run build", @@ -33,7 +36,7 @@ "devDependencies": { "babel-plugin-istanbul": "^3.0.0", "babel-plugin-transform-runtime": "^6.15.0", - "babel-preset-env": "^1.1.8", + "babel-preset-env": "^1.1.10", "babel-register": "^6.16.3", "bluebird": "^3.3.4", "chai": "^3.5.0", diff --git a/src/headers.js b/src/headers.js index dc31ee112..77e02d4de 100644 --- a/src/headers.js +++ b/src/headers.js @@ -5,7 +5,7 @@ * Headers class offers convenient helpers */ -import { checkIsHttpToken, checkInvalidHeaderChar } from './common.js'; +import { checkInvalidHeaderChar, checkIsHttpToken } from './common.js'; function sanitizeName(name) { name += ''; @@ -271,13 +271,6 @@ const HeadersIteratorPrototype = Object.setPrototypeOf({ Object.getPrototypeOf([][Symbol.iterator]()) )); -// On Node.js v0.12 the %IteratorPrototype% object is broken -if (typeof HeadersIteratorPrototype[Symbol.iterator] !== 'function') { - HeadersIteratorPrototype[Symbol.iterator] = function () { - return this; - }; -} - Object.defineProperty(HeadersIteratorPrototype, Symbol.toStringTag, { value: 'HeadersIterator', writable: false, From d23204b4ecb46ebc93c0e3d5dfe71dea82b9ac0c Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 26 Feb 2017 13:18:41 -0800 Subject: [PATCH 076/223] README: remove trailing whitespace --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3d4ae4300..bc4a5d7dd 100644 --- a/README.md +++ b/README.md @@ -191,12 +191,12 @@ Note that only `method`, `headers`, `redirect` and `body` are allowed in `window If no values are set, the following request headers will be sent automatically: -Header | Value +Header | Value ----------------- | -------------------------------------------------------- -`Accept-Encoding` | `gzip,deflate` _(when `options.compress === true`)_ -`Accept` | `*/*` -`Connection` | `close` _(when no `options.agent` is present)_ -`Content-Length` | _(automatically calculated, if possible)_ +`Accept-Encoding` | `gzip,deflate` _(when `options.compress === true`)_ +`Accept` | `*/*` +`Connection` | `close` _(when no `options.agent` is present)_ +`Content-Length` | _(automatically calculated, if possible)_ `User-Agent` | `node-fetch/1.0 (+https://github.com/bitinn/node-fetch)` From 67678a9a54f9715f56e9f3d7fe34bfbe0493e964 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 26 Feb 2017 13:45:25 -0800 Subject: [PATCH 077/223] Update UPGRADE-GUIDE --- UPGRADE-GUIDE.md | 50 ++++++++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/UPGRADE-GUIDE.md b/UPGRADE-GUIDE.md index 3ea134248..22aab748b 100644 --- a/UPGRADE-GUIDE.md +++ b/UPGRADE-GUIDE.md @@ -1,34 +1,41 @@ -# Upgrade to node-fetch v2 +# Upgrade to node-fetch v2.x -node-fetch v2 brings about many changes that increase the compliance of -WHATWG's [Fetch Standard][whatwg-fetch]. However, many of these changes meant -that apps written for node-fetch v1 needs to be updated to work with node-fetch -v2 and be conformant with the Fetch Standard. +node-fetch v2.x brings about many changes that increase the compliance of +WHATWG's [Fetch Standard][whatwg-fetch]. However, many of these changes mean +that apps written for node-fetch v1.x needs to be updated to work with +node-fetch v2.x and be conformant with the Fetch Standard. This document helps +you make this transition. + +Note that this document is not an exhaustive list of all changes made in v2.x, +but rather that of the most important breaking changes. See our [changelog] for +other comparatively minor modifications. ## `.text()` no longer tries to detect encoding -In v1, `response.text()` attempts to guess the text encoding of the input +In v1.x, `response.text()` attempts to guess the text encoding of the input material and decode it for the user. However, it runs counter to the Fetch Standard which demands `.text()` to always use UTF-8. In "response" to that, we have changed `.text()` to use UTF-8. A new function **`response.textConverted()`** is created that maintains the behavior of -`.text()` last year. +`.text()` in v1.x. ## Internal methods hidden -In v1, the user can access internal methods such as `_clone()`, `_decode()`, +In v1.x, the user can access internal methods such as `_clone()`, `_decode()`, and `_convert()` on the `response` object. While these methods should never -have been used, node-fetch v2 makes these functions completely inaccessible. -If your app makes use of these functions, it may break when upgrading to v2. +have been used, node-fetch v2.x makes these functions completely inaccessible. +If your app makes use of these functions, it may break when upgrading to v2.x. If you have a use case that requires these methods to be available, feel free to file an issue and we will be happy to help you solve the problem. ## Headers -The main goal we have for the `Headers` class in v2 is to make it completely -spec-compliant. +The main goal we have for the `Headers` class in v2.x is to make it completely +spec-compliant. These changes are done in conjunction with GitHub's +[`whatwg-fetch`][gh-fetch] polyfill, [Chrome][chrome-headers], and +[Firefox][firefox-headers]. ```js ////////////////////////////////////////////////////////////////////////////// @@ -87,15 +94,16 @@ headers.get('Héy'); // now throws new Headers({ 'Héy': 'ok' }); // now throws ``` -## 0.10.x support dropped - -If you are still using Node.js v0.10, upgrade ASAP. Not only has Node.js -dropped support for that release branch, it has become too much work for us to -maintain. Therefore, we have dropped official support for v0.10. +## Node.js v0.x support dropped -That being said, node-fetch may still work with v0.10, but as we are not -actively trying to support that version, it is in the user's best interest to -upgrade. +If you are still using Node.js v0.10 or v0.12, upgrade ASAP. Not only has it +become too much work for us to maintain, Node.js has also dropped support for +those release branches in 2016. Check out Node.js' official [LTS plan] for more +information on Node.js' support lifetime. [whatwg-fetch]: https://fetch.spec.whatwg.org/ -[#181]: https://github.com/bitinn/node-fetch/issues/181 +[LTS plan]: https://github.com/nodejs/LTS#lts-plan +[gh-fetch]: https://github.com/github/fetch +[chrome-headers]: https://crbug.com/645492 +[firefox-headers]: https://bugzilla.mozilla.org/show_bug.cgi?id=1278275 +[changelog]: CHANGELOG.md From a10f5786bd78419a98ab51372161d3449ceb3ebb Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 26 Feb 2017 13:50:38 -0800 Subject: [PATCH 078/223] Consolidate CHANGELOG --- CHANGELOG.md | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72ecf5112..45c248583 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,40 +5,30 @@ Changelog # 2.x release -## v2.0.0-alpha.3 - -- Major: overwrite user's `Content-Length` if we can be sure our information is correct -- Fix: exhaust list in `Headers` constructor before processing - -## v2.0.0-alpha.2 - -- Major: remove `headers.getAll()`; make `get()` return all headers delimited by commas (per spec) -- Major: remove undocumented `FOLLOW_SPEC` switch -- it is now the default - -## v2.0.0-alpha.1 +## v2.0.0 This is a major release. Check [our upgrade guide](https://github.com/bitinn/node-fetch/blob/master/UPGRADE-GUIDE.md) for an overview on some key differences between v1 and v2. -- Major: Node.js 0.10.x support is dropped -- Major: rewrite in transpiled ES2015 +- Major: Node.js 0.10.x and 0.12.x support is dropped +- Major: overwrite user's `Content-Length` if we can be sure our information is correct (per spec) +- Major: `response.text()` no longer attempts to detect encoding, instead always opting for UTF-8 (per spec); use `response.textConverted()` for the v1 behavior +- Major: make `response.json()` throw error instead of returning an empty object on 204 no-content respose (per spec; reverts behavior changed in v1.6.2) +- Major: remove `headers.getAll()`; make `get()` return all headers delimited by commas (per spec) - Major: internal methods are no longer exposed -- Major: throw error when a GET/HEAD Request is constructed with a non-null body (per spec) -- Major: `response.text()` no longer attempts to detect encoding, instead always opting for UTF-8 (per spec); use `response.textConverted()` for the old behavior -- Major: make `response.json()` throw error instead of returning an empty object on 204 no-content respose (per spec; reverts behavior set in v1.6.2) -- Major: arrays as parameters to `headers.append` and `headers.set` are joined as a string (per spec) +- Major: throw error when a `GET` or `HEAD` Request is constructed with a non-null body (per spec) - Enhance: start testing on Node.js 4, 6, 7 - Enhance: use Rollup to produce a distributed bundle (less memory overhead and faster startup) - Enhance: make `toString()` on Headers, Requests, and Responses return correct IDL class strings -- Enhance: add an option to conform to latest spec at the expense of reduced compatibility -- Enhance: set `Content-Length` header for Buffers as well - Enhance: add `response.arrayBuffer()` (also applies to Requests) - Enhance: add experimental `response.blob()` (also applies to Requests) - Enhance: make Headers iterable - Enhance: make Headers constructor accept an array of tuples - Enhance: make sure header names and values are valid in HTTP +- Enhance: add a list of default headers in README - Fix: coerce Headers prototype function parameters to strings, where applicable - Fix: fix Request and Response with `null` body - Fix: support WHATWG URL objects, created by `whatwg-url` package or `require('url').URL` in Node.js 7+ +- Other: rewrite in ES2015 using Babel and Rollup - Other: use Codecov for code coverage tracking From 56f786f896b7550b5c17c4b479d6b30cf509e3f3 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 26 Feb 2017 14:12:45 -0800 Subject: [PATCH 079/223] Update deps and remove 0.12 compat --- package.json | 7 +++---- rollup.config.js | 1 - test/server.js | 5 ++--- test/test.js | 5 ++--- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/package.json b/package.json index b17c8d487..0fa2de617 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ }, "homepage": "https://github.com/bitinn/node-fetch", "devDependencies": { - "babel-plugin-istanbul": "^3.0.0", + "babel-plugin-istanbul": "^4.0.0", "babel-plugin-transform-runtime": "^6.15.0", "babel-preset-env": "^1.1.10", "babel-register": "^6.16.3", @@ -44,7 +44,7 @@ "chai-iterator": "^1.1.1", "chai-string": "^1.3.0", "codecov": "^1.0.1", - "cross-env": "2.0.1", + "cross-env": "^3.1.4", "form-data": ">=1.0.0", "is-builtin-module": "^1.0.0", "mocha": "^3.1.2", @@ -52,9 +52,8 @@ "parted": "^0.1.1", "promise": "^7.1.1", "resumer": "0.0.0", - "rollup": "^0.37.0", + "rollup": "^0.41.4", "rollup-plugin-babel": "^2.6.1", - "rollup-plugin-node-resolve": "^2.0.0", "whatwg-url": "^4.0.0" }, "dependencies": { diff --git a/rollup.config.js b/rollup.config.js index d2123b971..17e129272 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -1,6 +1,5 @@ import isBuiltin from 'is-builtin-module'; import babel from 'rollup-plugin-babel'; -import resolve from 'rollup-plugin-node-resolve'; import tweakDefault from './build/rollup-plugin'; process.env.BABEL_ENV = 'rollup'; diff --git a/test/server.js b/test/server.js index 804603a04..d6016d0ad 100644 --- a/test/server.js +++ b/test/server.js @@ -1,4 +1,3 @@ -import repeat from 'babel-runtime/core-js/string/repeat'; import * as http from 'http'; import { parse } from 'url'; import * as zlib from 'zlib'; @@ -188,7 +187,7 @@ export default class TestServer { res.statusCode = 200; res.setHeader('Content-Type', 'text/html'); res.setHeader('Transfer-Encoding', 'chunked'); - res.write(repeat('a', 10)); + res.write('a'.repeat(10)); res.end(convert('
日本語
', 'Shift_JIS')); } @@ -196,7 +195,7 @@ export default class TestServer { res.statusCode = 200; res.setHeader('Content-Type', 'text/html'); res.setHeader('Transfer-Encoding', 'chunked'); - res.write(repeat('a', 1200)); + res.write('a'.repeat(1200)); res.end(convert('中文', 'gbk')); } diff --git a/test/test.js b/test/test.js index b57dad8e8..31fa0322e 100644 --- a/test/test.js +++ b/test/test.js @@ -1,6 +1,5 @@ // test tools -import repeat from 'babel-runtime/core-js/string/repeat'; import chai from 'chai'; import chaiPromised from 'chai-as-promised'; import chaiIterator from 'chai-iterator'; @@ -1093,7 +1092,7 @@ describe('node-fetch', () => { url = `${base}encoding/chunked`; return fetch(url).then(res => { expect(res.status).to.equal(200); - const padding = repeat('a', 10); + const padding = 'a'.repeat(10); return res.textConverted().then(result => { expect(result).to.equal(`${padding}
日本語
`); }); @@ -1104,7 +1103,7 @@ describe('node-fetch', () => { url = `${base}encoding/invalid`; return fetch(url).then(res => { expect(res.status).to.equal(200); - const padding = repeat('a', 1200); + const padding = 'a'.repeat(1200); return res.textConverted().then(result => { expect(result).to.not.equal(`${padding}中文`); }); From f08b1207718ddd1d7a9055d39eeeba36b0dec17a Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 26 Feb 2017 14:29:40 -0800 Subject: [PATCH 080/223] Do not inherit from body Per spec, make Body a proper mixin --- src/body.js | 224 ++++++++++++++++++++++++------------------------ src/request.js | 11 ++- src/response.js | 11 ++- 3 files changed, 131 insertions(+), 115 deletions(-) diff --git a/src/body.js b/src/body.js index c017b69de..7cf2eb1ca 100644 --- a/src/body.js +++ b/src/body.js @@ -13,45 +13,46 @@ import Blob, {BUFFER} from './blob.js'; import FetchError from './fetch-error.js'; const DISTURBED = Symbol('disturbed'); -const CONSUME_BODY = Symbol('consumeBody'); /** * Body class * + * Cannot use ES6 class because Body must be called with .call(). + * * @param Stream body Readable stream * @param Object opts Response options * @return Void */ -export default class Body { - constructor(body, { - size = 0, - timeout = 0 - } = {}) { - if (body == null) { - // body is undefined or null - body = null; - } else if (typeof body === 'string') { - // body is string - } else if (body instanceof Blob) { - // body is blob - } else if (Buffer.isBuffer(body)) { - // body is buffer - } else if (bodyStream(body)) { - // body is stream - } else { - // none of the above - // coerce to string - body = String(body); - } - this.body = body; - this[DISTURBED] = false; - this.size = size; - this.timeout = timeout; +export default function Body(body, { + size = 0, + timeout = 0 +} = {}) { + if (body == null) { + // body is undefined or null + body = null; + } else if (typeof body === 'string') { + // body is string + } else if (body instanceof Blob) { + // body is blob + } else if (Buffer.isBuffer(body)) { + // body is buffer + } else if (bodyStream(body)) { + // body is stream + } else { + // none of the above + // coerce to string + body = String(body); } + this.body = body; + this[DISTURBED] = false; + this.size = size; + this.timeout = timeout; +} +Body.prototype = { get bodyUsed() { return this[DISTURBED]; - } + }, /** * Decode response as ArrayBuffer @@ -59,8 +60,8 @@ export default class Body { * @return Promise */ arrayBuffer() { - return this[CONSUME_BODY]().then(buf => toArrayBuffer(buf)); - } + return consumeBody.call(this).then(buf => toArrayBuffer(buf)); + }, /** * Return raw response as Blob @@ -69,7 +70,7 @@ export default class Body { */ blob() { let ct = this.headers && this.headers.get('content-type') || ''; - return this[CONSUME_BODY]().then(buf => Object.assign( + return consumeBody.call(this).then(buf => Object.assign( // Prevent copying new Blob([], { type: ct.toLowerCase() @@ -78,7 +79,7 @@ export default class Body { [BUFFER]: buf } )); - } + }, /** * Decode response as json @@ -86,8 +87,8 @@ export default class Body { * @return Promise */ json() { - return this[CONSUME_BODY]().then(buffer => JSON.parse(buffer.toString())); - } + return consumeBody.call(this).then(buffer => JSON.parse(buffer.toString())); + }, /** * Decode response as text @@ -95,8 +96,8 @@ export default class Body { * @return Promise */ text() { - return this[CONSUME_BODY]().then(buffer => buffer.toString()); - } + return consumeBody.call(this).then(buffer => buffer.toString()); + }, /** * Decode response as buffer (non-spec api) @@ -104,8 +105,8 @@ export default class Body { * @return Promise */ buffer() { - return this[CONSUME_BODY](); - } + return consumeBody.call(this); + }, /** * Decode response as text, while automatically detecting the encoding and @@ -114,94 +115,95 @@ export default class Body { * @return Promise */ textConverted() { - return this[CONSUME_BODY]().then(buffer => convertBody(buffer, this.headers)); + return consumeBody.call(this).then(buffer => convertBody(buffer, this.headers)); + }, + + +}; + +/** + * Decode buffers into utf-8 string + * + * @return Promise + */ +function consumeBody(body) { + if (this[DISTURBED]) { + return Body.Promise.reject(new Error(`body used already for: ${this.url}`)); } - /** - * Decode buffers into utf-8 string - * - * @return Promise - */ - [CONSUME_BODY]() { - if (this[DISTURBED]) { - return Body.Promise.reject(new Error(`body used already for: ${this.url}`)); - } + this[DISTURBED] = true; - this[DISTURBED] = true; + // body is null + if (this.body === null) { + return Body.Promise.resolve(new Buffer(0)); + } - // body is null - if (this.body === null) { - return Body.Promise.resolve(new Buffer(0)); - } + // body is string + if (typeof this.body === 'string') { + return Body.Promise.resolve(new Buffer(this.body)); + } - // body is string - if (typeof this.body === 'string') { - return Body.Promise.resolve(new Buffer(this.body)); - } + // body is blob + if (this.body instanceof Blob) { + return Body.Promise.resolve(this.body[BUFFER]); + } - // body is blob - if (this.body instanceof Blob) { - return Body.Promise.resolve(this.body[BUFFER]); - } + // body is buffer + if (Buffer.isBuffer(this.body)) { + return Body.Promise.resolve(this.body); + } - // body is buffer - if (Buffer.isBuffer(this.body)) { - return Body.Promise.resolve(this.body); - } + // istanbul ignore if: should never happen + if (!bodyStream(this.body)) { + return Body.Promise.resolve(new Buffer(0)); + } - // istanbul ignore if: should never happen - if (!bodyStream(this.body)) { - return Body.Promise.resolve(new Buffer(0)); + // body is stream + // get ready to actually consume the body + let accum = []; + let accumBytes = 0; + let abort = false; + + return new Body.Promise((resolve, reject) => { + let resTimeout; + + // allow timeout on slow response body + if (this.timeout) { + resTimeout = setTimeout(() => { + abort = true; + reject(new FetchError(`Response timeout while trying to fetch ${this.url} (over ${this.timeout}ms)`, 'body-timeout')); + }, this.timeout); } - // body is stream - // get ready to actually consume the body - let accum = []; - let accumBytes = 0; - let abort = false; - - return new Body.Promise((resolve, reject) => { - let resTimeout; - - // allow timeout on slow response body - if (this.timeout) { - resTimeout = setTimeout(() => { - abort = true; - reject(new FetchError(`Response timeout while trying to fetch ${this.url} (over ${this.timeout}ms)`, 'body-timeout')); - }, this.timeout); + // handle stream error, such as incorrect content-encoding + this.body.on('error', err => { + reject(new FetchError(`Invalid response body while trying to fetch ${this.url}: ${err.message}`, 'system', err)); + }); + + this.body.on('data', chunk => { + if (abort || chunk === null) { + return; } - // handle stream error, such as incorrect content-encoding - this.body.on('error', err => { - reject(new FetchError(`Invalid response body while trying to fetch ${this.url}: ${err.message}`, 'system', err)); - }); - - this.body.on('data', chunk => { - if (abort || chunk === null) { - return; - } - - if (this.size && accumBytes + chunk.length > this.size) { - abort = true; - reject(new FetchError(`content size at ${this.url} over limit: ${this.size}`, 'max-size')); - return; - } - - accumBytes += chunk.length; - accum.push(chunk); - }); - - this.body.on('end', () => { - if (abort) { - return; - } - - clearTimeout(resTimeout); - resolve(Buffer.concat(accum)); - }); + if (this.size && accumBytes + chunk.length > this.size) { + abort = true; + reject(new FetchError(`content size at ${this.url} over limit: ${this.size}`, 'max-size')); + return; + } + + accumBytes += chunk.length; + accum.push(chunk); }); - } + this.body.on('end', () => { + if (abort) { + return; + } + + clearTimeout(resTimeout); + resolve(Buffer.concat(accum)); + }); + }); } /** diff --git a/src/request.js b/src/request.js index f9393cacb..9c53fec0f 100644 --- a/src/request.js +++ b/src/request.js @@ -18,7 +18,7 @@ const PARSED_URL = Symbol('url'); * @param Object init Custom options * @return Void */ -export default class Request extends Body { +export default class Request { constructor(input, init = {}) { let parsedURL; @@ -51,7 +51,7 @@ export default class Request extends Body { clone(input) : null; - super(inputBody, { + Body.call(this, inputBody, { timeout: init.timeout || input.timeout || 0, size: init.size || input.size || 0 }); @@ -101,6 +101,13 @@ export default class Request extends Body { } } +for (const name of Object.getOwnPropertyNames(Body.prototype)) { + if (!(name in Request.prototype)) { + const desc = Object.getOwnPropertyDescriptor(Body.prototype, name); + Object.defineProperty(Request.prototype, name, desc); + } +} + Object.defineProperty(Request.prototype, Symbol.toStringTag, { value: 'RequestPrototype', writable: false, diff --git a/src/response.js b/src/response.js index 85b1820ef..6659bdee4 100644 --- a/src/response.js +++ b/src/response.js @@ -16,9 +16,9 @@ import Body, { clone } from './body'; * @param Object opts Response options * @return Void */ -export default class Response extends Body { +export default class Response { constructor(body = null, opts = {}) { - super(body, opts); + Body.call(this, body, opts); this.url = opts.url; this.status = opts.status || 200; @@ -58,6 +58,13 @@ export default class Response extends Body { } } +for (const name of Object.getOwnPropertyNames(Body.prototype)) { + if (!(name in Response.prototype)) { + const desc = Object.getOwnPropertyDescriptor(Body.prototype, name); + Object.defineProperty(Response.prototype, name, desc); + } +} + Object.defineProperty(Response.prototype, Symbol.toStringTag, { value: 'ResponsePrototype', writable: false, From 9e140039acbe3ea43e085832ea25fddff355b30b Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 26 Feb 2017 14:42:09 -0800 Subject: [PATCH 081/223] Factor out Body mixin --- src/body.js | 10 ++++++++++ src/request.js | 7 +------ src/response.js | 7 +------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/body.js b/src/body.js index 7cf2eb1ca..cf64852d9 100644 --- a/src/body.js +++ b/src/body.js @@ -121,6 +121,16 @@ Body.prototype = { }; +Body.mixIn = function (proto) { + for (const name of Object.getOwnPropertyNames(Body.prototype)) { + // istanbul ignore else: future proof + if (!(name in proto)) { + const desc = Object.getOwnPropertyDescriptor(Body.prototype, name); + Object.defineProperty(proto, name, desc); + } + } +}; + /** * Decode buffers into utf-8 string * diff --git a/src/request.js b/src/request.js index 9c53fec0f..dfadbf547 100644 --- a/src/request.js +++ b/src/request.js @@ -101,12 +101,7 @@ export default class Request { } } -for (const name of Object.getOwnPropertyNames(Body.prototype)) { - if (!(name in Request.prototype)) { - const desc = Object.getOwnPropertyDescriptor(Body.prototype, name); - Object.defineProperty(Request.prototype, name, desc); - } -} +Body.mixIn(Request.prototype); Object.defineProperty(Request.prototype, Symbol.toStringTag, { value: 'RequestPrototype', diff --git a/src/response.js b/src/response.js index 6659bdee4..0735e3c5b 100644 --- a/src/response.js +++ b/src/response.js @@ -58,12 +58,7 @@ export default class Response { } } -for (const name of Object.getOwnPropertyNames(Body.prototype)) { - if (!(name in Response.prototype)) { - const desc = Object.getOwnPropertyDescriptor(Body.prototype, name); - Object.defineProperty(Response.prototype, name, desc); - } -} +Body.mixIn(Response.prototype); Object.defineProperty(Response.prototype, Symbol.toStringTag, { value: 'ResponsePrototype', From b899649c32f78cb114afb85106a3b5fa47018173 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 26 Feb 2017 14:42:46 -0800 Subject: [PATCH 082/223] Remove bluebird --- package.json | 1 - test/test.js | 22 +++++++--------------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index 0fa2de617..220ee9d9d 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,6 @@ "babel-plugin-transform-runtime": "^6.15.0", "babel-preset-env": "^1.1.10", "babel-register": "^6.16.3", - "bluebird": "^3.3.4", "chai": "^3.5.0", "chai-as-promised": "^6.0.0", "chai-iterator": "^1.1.1", diff --git a/test/test.js b/test/test.js index 31fa0322e..fc0e0b6dc 100644 --- a/test/test.js +++ b/test/test.js @@ -4,13 +4,13 @@ import chai from 'chai'; import chaiPromised from 'chai-as-promised'; import chaiIterator from 'chai-iterator'; import chaiString from 'chai-string'; -import bluebird from 'bluebird'; import then from 'promise'; import {spawn} from 'child_process'; import * as stream from 'stream'; import resumer from 'resumer'; import FormData from 'form-data'; import {parse as parseURL} from 'url'; +import {URL} from 'whatwg-url'; import * as http from 'http'; import * as fs from 'fs'; @@ -33,14 +33,6 @@ import ResponseOrig from '../src/response.js'; import Body from '../src/body.js'; import Blob from '../src/blob.js'; import FetchError from '../src/fetch-error.js'; -// test with native promise on node 0.11, and bluebird for node 0.10 -fetch.Promise = fetch.Promise || bluebird; - -let URL; -// whatwg-url doesn't support old Node.js, so make it optional -try { - URL = require('whatwg-url').URL; -} catch (err) {} const supportToString = ({ [Symbol.toStringTag]: 'z' @@ -62,16 +54,16 @@ describe('node-fetch', () => { it('should return a promise', function() { url = 'http://example.com/'; const p = fetch(url); - expect(p).to.be.an.instanceof(fetch.Promise); + expect(p).to.be.an.instanceof(Promise); expect(p).to.have.property('then'); }); it('should allow custom promise', function() { url = 'http://example.com/'; - const old = fetch.Promise; + const old = Promise; fetch.Promise = then; expect(fetch(url)).to.be.an.instanceof(then); - expect(fetch(url)).to.not.be.an.instanceof(bluebird); + expect(fetch(url)).to.not.be.an.instanceof(old); fetch.Promise = old; }); @@ -1147,7 +1139,7 @@ describe('node-fetch', () => { url = `${base}json`; return fetch(url).then(res => { const r1 = res.clone(); - return fetch.Promise.all([res.json(), r1.text()]).then(results => { + return Promise.all([res.json(), r1.text()]).then(results => { expect(results[0]).to.deep.equal({name: 'value'}); expect(results[1]).to.equal('{"name":"value"}'); }); @@ -1445,7 +1437,7 @@ describe('node-fetch', () => { }); }); - (URL ? it : it.skip)('should support fetch with WHATWG URL object', function() { + it('should support fetch with WHATWG URL object', function() { url = `${base}hello`; const urlObj = new URL(url); const req = new Request(urlObj); @@ -1779,7 +1771,7 @@ describe('node-fetch', () => { expect(cl.agent).to.equal(agent); // clone body shouldn't be the same body expect(cl.body).to.not.equal(body); - return fetch.Promise.all([cl.text(), req.text()]).then(results => { + return Promise.all([cl.text(), req.text()]).then(results => { expect(results[0]).to.equal('a=1'); expect(results[1]).to.equal('a=1'); }); From 55e573b741b5784afd3aa8aab0702de4c471476c Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 26 Feb 2017 15:30:43 -0800 Subject: [PATCH 083/223] Back to 100% coverage --- src/body.js | 1 + src/index.js | 5 ----- src/request.js | 11 ++++++++--- test/test.js | 6 +++--- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/body.js b/src/body.js index cf64852d9..8b7ed40b7 100644 --- a/src/body.js +++ b/src/body.js @@ -345,6 +345,7 @@ export function extractContentType(instance) { export function getTotalBytes(instance) { const {body} = instance; + // istanbul ignore if: included for completion if (body === null) { // body is null return 0; diff --git a/src/index.js b/src/index.js index fbd0cdebe..4b68db39a 100644 --- a/src/index.js +++ b/src/index.js @@ -37,13 +37,8 @@ export default function fetch(url, opts) { return new fetch.Promise((resolve, reject) => { // build request object const request = new Request(url, opts); - const options = getNodeRequestOptions(request); - if (!options.protocol || !options.hostname) { - throw new Error('only absolute urls are supported'); - } - const send = (options.protocol === 'https:' ? https : http).request; // http.request only support string as host header, this hack make custom host header possible diff --git a/src/request.js b/src/request.js index dfadbf547..51b7ec124 100644 --- a/src/request.js +++ b/src/request.js @@ -111,6 +111,7 @@ Object.defineProperty(Request.prototype, Symbol.toStringTag, { }); export function getNodeRequestOptions(request) { + const parsedURL = request[PARSED_URL]; const headers = new Headers(request.headers); // fetch step 3 @@ -119,8 +120,12 @@ export function getNodeRequestOptions(request) { } // Basic fetch - if (!/^https?:$/.test(request[PARSED_URL].protocol)) { - throw new Error('only http(s) protocols are supported'); + if (!parsedURL.protocol || !parsedURL.hostname) { + throw new TypeError('Only absolute URLs are supported'); + } + + if (!/^https?:$/.test(parsedURL.protocol)) { + throw new TypeError('Only HTTP(S) protocols are supported'); } // HTTP-network-or-cache fetch steps 5-9 @@ -154,7 +159,7 @@ export function getNodeRequestOptions(request) { // HTTP-network fetch step 4 // chunked encoding is handled by Node.js - return Object.assign({}, request[PARSED_URL], { + return Object.assign({}, parsedURL, { method: request.method, headers: headers.raw(), agent: request.agent diff --git a/test/test.js b/test/test.js index fc0e0b6dc..43e38fb10 100644 --- a/test/test.js +++ b/test/test.js @@ -91,17 +91,17 @@ describe('node-fetch', () => { it('should reject with error if url is protocol relative', function() { url = '//example.com/'; - return expect(fetch(url)).to.eventually.be.rejectedWith(Error); + return expect(fetch(url)).to.eventually.be.rejectedWith(TypeError, 'Only absolute URLs are supported'); }); it('should reject with error if url is relative path', function() { url = '/some/path'; - return expect(fetch(url)).to.eventually.be.rejectedWith(Error); + return expect(fetch(url)).to.eventually.be.rejectedWith(TypeError, 'Only absolute URLs are supported'); }); it('should reject with error if protocol is unsupported', function() { url = 'ftp://example.com/'; - return expect(fetch(url)).to.eventually.be.rejectedWith(Error); + return expect(fetch(url)).to.eventually.be.rejectedWith(TypeError, 'Only HTTP(S) protocols are supported'); }); it('should reject with error on network failure', function() { From 25e43a8f3eea097bc6b65d2a73f81dded0e6fe99 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 26 Feb 2017 16:45:11 -0800 Subject: [PATCH 084/223] Export FetchError and update ERROR-HANDLING.md --- ERROR-HANDLING.md | 8 +++++--- src/index.js | 3 ++- test/test.js | 4 +++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ERROR-HANDLING.md b/ERROR-HANDLING.md index 0e4025d14..630032598 100644 --- a/ERROR-HANDLING.md +++ b/ERROR-HANDLING.md @@ -6,16 +6,18 @@ Because `window.fetch` isn't designed to transparent about the cause of request The basics: -- All [operational errors](https://www.joyent.com/node-js/production/design/errors) are rejected as [FetchError](https://github.com/bitinn/node-fetch/blob/master/lib/fetch-error.js), you can handle them all through promise `catch` clause. +- All [operational errors][joyent-guide] are rejected as [FetchError](https://github.com/bitinn/node-fetch/blob/master/README.md#class-fetcherror), you can handle them all through promise `catch` clause. - All errors comes with `err.message` detailing the cause of errors. - All errors originated from `node-fetch` are marked with custom `err.type`. -- All errors originated from Node.js core are marked with `err.type = system`, and contains addition `err.code` and `err.errno` for error handling, they are alias to error codes thrown by Node.js core. +- All errors originated from Node.js core are marked with `err.type = 'system'`, and contains addition `err.code` and `err.errno` for error handling, they are alias to error codes thrown by Node.js core. -- [Programmer errors](https://www.joyent.com/node-js/production/design/errors) are either thrown as soon as possible, or rejected with default `Error` with `err.message` for ease of troubleshooting. +- [Programmer errors][joyent-guide] are either thrown as soon as possible, or rejected with default `Error` with `err.message` for ease of troubleshooting. List of error types: - Because we maintain 100% coverage, see [test.js](https://github.com/bitinn/node-fetch/blob/master/test/test.js) for a full list of custom `FetchError` types, as well as some of the common errors from Node.js + +[joyent-guide]: https://www.joyent.com/node-js/production/design/errors#operational-errors-vs-programmer-errors diff --git a/src/index.js b/src/index.js index 4b68db39a..8b2f66de9 100644 --- a/src/index.js +++ b/src/index.js @@ -192,5 +192,6 @@ fetch.Promise = global.Promise; export { Headers, Request, - Response + Response, + FetchError }; diff --git a/test/test.js b/test/test.js index 43e38fb10..eecd10f6b 100644 --- a/test/test.js +++ b/test/test.js @@ -23,16 +23,17 @@ import TestServer from './server'; // test subjects import fetch, { + FetchError, Headers, Request, Response } from '../src/'; +import FetchErrorOrig from '../src/fetch-error.js'; import HeadersOrig from '../src/headers.js'; import RequestOrig from '../src/request.js'; import ResponseOrig from '../src/response.js'; import Body from '../src/body.js'; import Blob from '../src/blob.js'; -import FetchError from '../src/fetch-error.js'; const supportToString = ({ [Symbol.toStringTag]: 'z' @@ -78,6 +79,7 @@ describe('node-fetch', () => { }); it('should expose Headers, Response and Request constructors', function() { + expect(FetchError).to.equal(FetchErrorOrig); expect(Headers).to.equal(HeadersOrig); expect(Response).to.equal(ResponseOrig); expect(Request).to.equal(RequestOrig); From a345c398b30d2bccf54abc9c45071dece69a7bba Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 26 Feb 2017 16:52:32 -0800 Subject: [PATCH 085/223] More documentation --- CHANGELOG.md | 34 ++++++-- LIMITS.md | 13 ++- README.md | 227 ++++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 232 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45c248583..5504645e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,27 +9,43 @@ Changelog This is a major release. Check [our upgrade guide](https://github.com/bitinn/node-fetch/blob/master/UPGRADE-GUIDE.md) for an overview on some key differences between v1 and v2. +### General changes + - Major: Node.js 0.10.x and 0.12.x support is dropped +- Major: `require('node-fetch/lib/response')` etc. is now unsupported; use `require('node-fetch').Response` or ES6 module imports +- Enhance: start testing on Node.js 4, 6, 7 +- Enhance: use Rollup to produce a distributed bundle (less memory overhead and faster startup) +- Enhance: make `Object.prototype.toString()` on Headers, Requests, and Responses return correct class strings +- Other: rewrite in ES2015 using Babel +- Other: use Codecov for code coverage tracking + +### HTTP requests + - Major: overwrite user's `Content-Length` if we can be sure our information is correct (per spec) +- Fix: support WHATWG URL objects, created by `whatwg-url` package or `require('url').URL` in Node.js 7+ + +### Response and Request classes + - Major: `response.text()` no longer attempts to detect encoding, instead always opting for UTF-8 (per spec); use `response.textConverted()` for the v1 behavior - Major: make `response.json()` throw error instead of returning an empty object on 204 no-content respose (per spec; reverts behavior changed in v1.6.2) -- Major: remove `headers.getAll()`; make `get()` return all headers delimited by commas (per spec) - Major: internal methods are no longer exposed - Major: throw error when a `GET` or `HEAD` Request is constructed with a non-null body (per spec) -- Enhance: start testing on Node.js 4, 6, 7 -- Enhance: use Rollup to produce a distributed bundle (less memory overhead and faster startup) -- Enhance: make `toString()` on Headers, Requests, and Responses return correct IDL class strings - Enhance: add `response.arrayBuffer()` (also applies to Requests) - Enhance: add experimental `response.blob()` (also applies to Requests) +- Fix: fix Request and Response with `null` body + +### Headers class + +- Major: remove `headers.getAll()`; make `get()` return all headers delimited by commas (per spec) - Enhance: make Headers iterable - Enhance: make Headers constructor accept an array of tuples - Enhance: make sure header names and values are valid in HTTP -- Enhance: add a list of default headers in README - Fix: coerce Headers prototype function parameters to strings, where applicable -- Fix: fix Request and Response with `null` body -- Fix: support WHATWG URL objects, created by `whatwg-url` package or `require('url').URL` in Node.js 7+ -- Other: rewrite in ES2015 using Babel and Rollup -- Other: use Codecov for code coverage tracking + +### Documentation + +- Enhance: more comprehensive API docs +- Enhance: add a list of default headers in README # 1.x release diff --git a/LIMITS.md b/LIMITS.md index f49fe285d..bdcf66a55 100644 --- a/LIMITS.md +++ b/LIMITS.md @@ -8,13 +8,15 @@ Known differences - URL input must be an absolute URL, using either `http` or `https` as scheme. -- On the upside, there are no forbidden headers, and `res.url` contains the final url when following redirects. +- On the upside, there are no forbidden headers. -- For convenience, `res.body` is a transform stream, so decoding can be handled independently. +- `res.url` contains the final url when following redirects. -- Similarly, `req.body` can either be a string, a buffer or a readable stream. +- For convenience, `res.body` is a Node.js [Readable stream][readable-stream], so decoding can be handled independently. -- Also, you can handle rejected fetch requests through checking `err.type` and `err.code`. +- Similarly, `req.body` can either be `null`, a string, a buffer or a Readable stream. + +- Also, you can handle rejected fetch requests through checking `err.type` and `err.code`. See [ERROR-HANDLING.md][] for more info. - Only support `res.text()`, `res.json()`, `res.blob()`, `res.arraybuffer()`, `res.buffer()` @@ -23,3 +25,6 @@ Known differences - Current implementation lacks server-side cookie store, you will need to extract `Set-Cookie` headers manually. - If you are using `res.clone()` and writing an isomorphic app, note that stream on Node.js have a smaller internal buffer size (16Kb, aka `highWaterMark`) from client-side browsers (>1Mb, not consistent across browsers). + +[readable-stream]: https://nodejs.org/api/stream.html#stream_readable_streams +[ERROR-HANDLING.md]: https://github.com/bitinn/node-fetch/blob/master/ERROR-HANDLING.md diff --git a/README.md b/README.md index bc4a5d7dd..3b7d5b7c6 100644 --- a/README.md +++ b/README.md @@ -9,36 +9,38 @@ node-fetch A light-weight module that brings `window.fetch` to Node.js -# Motivation +## Motivation -Instead of implementing `XMLHttpRequest` in Node.js to run browser-specific [Fetch polyfill](https://github.com/github/fetch), why not go from native `http` to `Fetch` API directly? Hence `node-fetch`, minimal code for a `window.fetch` compatible API on Node.js runtime. +Instead of implementing `XMLHttpRequest` in Node.js to run browser-specific [Fetch polyfill](https://github.com/github/fetch), why not go from native `http` to `fetch` API directly? Hence `node-fetch`, minimal code for a `window.fetch` compatible API on Node.js runtime. See Matt Andrews' [isomorphic-fetch](https://github.com/matthew-andrews/isomorphic-fetch) for isomorphic usage (exports `node-fetch` for server-side, `whatwg-fetch` for client-side). -# Features +## Features - Stay consistent with `window.fetch` API. -- Make conscious trade-off when following [whatwg fetch spec](https://fetch.spec.whatwg.org/) and [stream spec](https://streams.spec.whatwg.org/) implementation details, document known difference. +- Make conscious trade-off when following [whatwg fetch spec][whatwg-fetch] and [stream spec](https://streams.spec.whatwg.org/) implementation details, document known difference. - Use native promise, but allow substituting it with [insert your favorite promise library]. - Use native stream for body, on both request and response. - Decode content encoding (gzip/deflate) properly, and convert string output (such as `res.text()` and `res.json()`) to UTF-8 automatically. -- Useful extensions such as timeout, redirect limit, response size limit, [explicit errors](https://github.com/bitinn/node-fetch/blob/master/ERROR-HANDLING.md) for troubleshooting. +- Useful extensions such as timeout, redirect limit, response size limit, [explicit errors][] for troubleshooting. -# Difference from client-side fetch +## Difference from client-side fetch - See [Known Differences](https://github.com/bitinn/node-fetch/blob/master/LIMITS.md) for details. - If you happen to use a missing feature that `window.fetch` offers, feel free to open an issue. - Pull requests are welcomed too! -# Install +## Install -`npm install node-fetch --save` +```sh +$ npm install node-fetch --save +``` -# Usage +## Usage ```javascript import fetch from 'node-fetch'; @@ -159,35 +161,41 @@ fetch('http://httpbin.org/post', { method: 'POST', body: form, headers: form.get See [test cases](https://github.com/bitinn/node-fetch/blob/master/test/test.js) for more examples. -# API +## API -## fetch(url, options) +### fetch(url[, options]) -Returns a `Promise` +- `url` A string representing the URL for fetching +- `options` [Options](#fetch-options) for the HTTP(S) request +- Returns: Promise<[Response](#class-response)> -### Url +Perform an HTTP(S) fetch. -Should be an absolute url, eg `http://example.com/` +`url` should be an absolute url, such as `http://example.com/`. A path-relative URL (`/file/under/root`) or protocol-relative URL (`//can-be-http-or-https.com/`) will result in a rejected promise. -### Options + +#### Options -Note that only `method`, `headers`, `redirect` and `body` are allowed in `window.fetch`. Other options are node.js extensions. The default values are shown after each option key. +The default values are shown after each option key. -``` +```js { - method: 'GET' - , headers: {} // request header. format {a:'1'} or {b:['1','2','3']} - , redirect: 'follow' // set to `manual` to extract redirect headers, `error` to reject redirect - , follow: 20 // maximum redirect count. 0 to not follow redirect - , timeout: 0 // req/res timeout in ms, it resets on redirect. 0 to disable (OS limit applies) - , compress: true // support gzip/deflate content encoding. false to disable - , size: 0 // maximum response body size in bytes. 0 to disable - , body: empty // request body. can be a string, buffer, readable stream - , agent: null // http.Agent instance, allows custom proxy, certificate etc. + // These properties are part of the Fetch Standard + method: 'GET', + headers: {}, // request headers. format is the identical to that accepted by the Headers constructor (see below) + body: null, // request body. can be null, a string, a Buffer, a Blob, or a Node.js Readable stream + redirect: 'follow', // set to `manual` to extract redirect headers, `error` to reject redirect + + // The following properties are node-fetch extensions + follow: 20, // maximum redirect count. 0 to not follow redirect + timeout: 0, // req/res timeout in ms, it resets on redirect. 0 to disable (OS limit applies) + compress: true, // support gzip/deflate content encoding. false to disable + size: 0, // maximum response body size in bytes. 0 to disable + agent: null // http(s).Agent instance, allows custom proxy, certificate etc. } ``` -#### Default Headers +##### Default Headers If no values are set, the following request headers will be sent automatically: @@ -199,13 +207,169 @@ Header | Value `Content-Length` | _(automatically calculated, if possible)_ `User-Agent` | `node-fetch/1.0 (+https://github.com/bitinn/node-fetch)` + +### Class: Request + +An HTTP(S) request containing information about URL, method, headers, and the body. This class implements the [Body](#iface-body) interface. + +Due to the nature of Node.js, the following properties are not implemented at this moment: + +- `type` +- `destination` +- `referrer` +- `referrerPolicy` +- `mode` +- `credentials` +- `cache` +- `integrity` +- `keepalive` + +The following node-fetch extension properties are provided: + +- `follow` +- `compress` +- `counter` +- `agent` + +See [options](#fetch-options) for exact meaning of these extensions. + +#### new Request(input[, options]) + +*(spec-compliant)* + +- `input` A string representing a URL, or another `Request` (which will be cloned) +- `options` [Options][#fetch-options] for the HTTP(S) request + +Constructs a new `Request` object. The constructor is identical to that in the [browser](https://developer.mozilla.org/en-US/docs/Web/API/Request/Request). + +In most cases, directly `fetch(url, options)` is simpler than creating a `Request` object. + + +### Class: Response + +An HTTP(S) response. This class implements the [Body](#iface-body) interface. + +The following properties are not implemented in node-fetch at this moment: + +- `Response.error()` +- `Response.redirect()` +- `type` +- `redirected` +- `trailer` + +#### new Response([body[, options]]) + +*(spec-compliant)* + +- `body` A string or [Readable stream][node-readable] +- `options` A [`ResponseInit`][response-init] options dictionary + +Constructs a new `Response` object. The constructor is identical to that in the [browser](https://developer.mozilla.org/en-US/docs/Web/API/Response/Response). + +Because Node.js does not implement service workers (for which this class was designed), one rarely has to construct a `Response` directly. + + +### Class: Headers + +This class allows manipulating and iterating over a set of HTTP headers. All methods specified in the [Fetch Standard][whatwg-fetch] are implemented. + +#### new Headers([init]) + +*(spec-compliant)* + +- `init` Optional argument to pre-fill the `Headers` object + +Construct a new `Headers` object. `init` can be either `null`, a `Headers` object, an key-value map object, or any iterable object. + +```js +// Example adapted from https://fetch.spec.whatwg.org/#example-headers-class + +const meta = { + 'Content-Type': 'text/xml', + 'Breaking-Bad': '<3' +}; +const headers = new Headers(meta); + +// The above is equivalent to +const meta = [ + [ 'Content-Type', 'text/xml' ], + [ 'Breaking-Bad', '<3' ] +]; +const headers = new Headers(meta); + +// You can in fact use any iterable objects, like a Map or even another Headers +const meta = new Map(); +meta.set('Content-Type', 'text/xml'); +meta.set('Breaking-Bad', '<3'); +const headers = new Headers(meta); +const copyOfHeaders = new Headers(headers); +``` + + +### Interface: Body + +`Body` is an abstract interface with methods that are applicable to both `Request` and `Response` classes. + +The following methods are not yet implemented in node-fetch at this moment: + +- `formData()` + +#### body.body + +*(deviation from spec)* + +* Node.js [`Readable` stream][node-readable] + +The data encapsulated in the `Body` object. Note that while the [Fetch Standard][whatwg-fetch] requires the property to always be a WHATWG `ReadableStream`, in node-fetch it is a Node.js [`Readable` stream][node-readable]. + +#### body.bodyUsed + +*(spec-compliant)* + +* `Boolean` + +A boolean property for if this body has been consumed. Per spec, a consumed body cannot be used again. + +#### body.arrayBuffer() +#### body.blob() +#### body.json() +#### body.text() + +*(spec-compliant)* + +* Returns: Promise + +Consume the body and return a promise that will resolve to one of these formats. + +#### body.buffer() + +*(node-fetch extension)* + +* Returns: Promise<Buffer> + +Consume the body and return a promise that will resolve to a Buffer. + +#### body.textConverted() + +*(node-fetch extension)* + +* Returns: Promise<String> + +Identical to `body.text()`, except instead of always converting to UTF-8, encoding sniffing will be performed and text converted to UTF-8, if possible. + + +### Class: FetchError + +*(node-fetch extension)* + +An operational error in the fetching process. See [ERROR-HANDLING.md][] for more info. -# License +## License MIT -# Acknowledgement +## Acknowledgement Thanks to [github/fetch](https://github.com/github/fetch) for providing a solid implementation reference. @@ -216,3 +380,8 @@ Thanks to [github/fetch](https://github.com/github/fetch) for providing a solid [travis-url]: https://travis-ci.org/bitinn/node-fetch [codecov-image]: https://img.shields.io/codecov/c/github/bitinn/node-fetch.svg?style=flat-square [codecov-url]: https://codecov.io/gh/bitinn/node-fetch +[ERROR-HANDLING.md]: https://github.com/bitinn/node-fetch/blob/master/ERROR-HANDLING.md +[whatwg-fetch]: https://fetch.spec.whatwg.org/ +[response-init]: https://fetch.spec.whatwg.org/#responseinit +[node-readable]: https://nodejs.org/api/stream.html#stream_readable_streams +[mdn-headers]: https://developer.mozilla.org/en-US/docs/Web/API/Headers From e284841db961d9886c0e0be596bcc81874e9b99a Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 26 Feb 2017 17:00:01 -0800 Subject: [PATCH 086/223] test: use fetch.Promise consistently On Node.js v4, Babel polyfills `Promise`. --- test/test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test.js b/test/test.js index eecd10f6b..7f6e75571 100644 --- a/test/test.js +++ b/test/test.js @@ -55,13 +55,13 @@ describe('node-fetch', () => { it('should return a promise', function() { url = 'http://example.com/'; const p = fetch(url); - expect(p).to.be.an.instanceof(Promise); + expect(p).to.be.an.instanceof(fetch.Promise); expect(p).to.have.property('then'); }); it('should allow custom promise', function() { url = 'http://example.com/'; - const old = Promise; + const old = fetch.Promise; fetch.Promise = then; expect(fetch(url)).to.be.an.instanceof(then); expect(fetch(url)).to.not.be.an.instanceof(old); From f0d0675d00845c59ac4bac58d8c9b89d2a39d38e Mon Sep 17 00:00:00 2001 From: Tino Vyatkin Date: Sun, 26 Feb 2017 22:29:57 -0400 Subject: [PATCH 087/223] Remove unneeded is-stream and buffer-to-arraybuffer (#241) --- package.json | 4 +--- src/body.js | 12 +++++------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index 220ee9d9d..5f9ea00b0 100644 --- a/package.json +++ b/package.json @@ -57,8 +57,6 @@ }, "dependencies": { "babel-runtime": "^6.11.6", - "buffer-to-arraybuffer": "0.0.4", - "encoding": "^0.1.11", - "is-stream": "^1.0.1" + "encoding": "^0.1.11" } } diff --git a/src/body.js b/src/body.js index 8b7ed40b7..1a35fa653 100644 --- a/src/body.js +++ b/src/body.js @@ -6,9 +6,7 @@ */ import {convert} from 'encoding'; -import bodyStream from 'is-stream'; -import toArrayBuffer from 'buffer-to-arraybuffer'; -import {PassThrough} from 'stream'; +import Stream, {PassThrough} from 'stream'; import Blob, {BUFFER} from './blob.js'; import FetchError from './fetch-error.js'; @@ -36,7 +34,7 @@ export default function Body(body, { // body is blob } else if (Buffer.isBuffer(body)) { // body is buffer - } else if (bodyStream(body)) { + } else if (body instanceof Stream) { // body is stream } else { // none of the above @@ -60,7 +58,7 @@ Body.prototype = { * @return Promise */ arrayBuffer() { - return consumeBody.call(this).then(buf => toArrayBuffer(buf)); + return consumeBody.call(this).then(buf => buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength)); }, /** @@ -164,7 +162,7 @@ function consumeBody(body) { } // istanbul ignore if: should never happen - if (!bodyStream(this.body)) { + if (!(this.body instanceof Stream)) { return Body.Promise.resolve(new Buffer(0)); } @@ -292,7 +290,7 @@ export function clone(instance) { // check that body is a stream and not form-data object // note: we can't clone the form-data object without having it as a dependency - if (bodyStream(body) && typeof body.getBoundary !== 'function') { + if ((body instanceof Stream) && (typeof body.getBoundary !== 'function')) { // tee instance body p1 = new PassThrough(); p2 = new PassThrough(); From 397eab775767f9da8dce426b7bc6f95d7ed3ff88 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 26 Feb 2017 21:27:27 -0800 Subject: [PATCH 088/223] compress: use spec nomenclature --- src/index.js | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/src/index.js b/src/index.js index 8b2f66de9..1062fb272 100644 --- a/src/index.js +++ b/src/index.js @@ -125,33 +125,31 @@ export default function fetch(url, opts) { , timeout: request.timeout }; - // response object - let output; + // HTTP-network fetch step 16.1.2 + const codings = headers.get('Content-Encoding'); + + // HTTP-network fetch step 16.1.3: handle content codings // in following scenarios we ignore compression support // 1. compression support is disabled // 2. HEAD request - // 3. no content-encoding header + // 3. no Content-Encoding header // 4. no content response (204) // 5. content not modified response (304) - if (!request.compress || request.method === 'HEAD' || !headers.has('content-encoding') || res.statusCode === 204 || res.statusCode === 304) { - output = new Response(body, response_options); - resolve(output); + if (!request.compress || request.method === 'HEAD' || codings === null || res.statusCode === 204 || res.statusCode === 304) { + resolve(new Response(body, response_options)); return; } - // otherwise, check for gzip or deflate - let name = headers.get('content-encoding'); - // for gzip - if (name == 'gzip' || name == 'x-gzip') { + if (codings == 'gzip' || codings == 'x-gzip') { body = body.pipe(zlib.createGunzip()); - output = new Response(body, response_options); - resolve(output); + resolve(new Response(body, response_options)); return; + } // for deflate - } else if (name == 'deflate' || name == 'x-deflate') { + if (codings == 'deflate' || codings == 'x-deflate') { // handle the infamous raw deflate response from old servers // a hack for old IIS and Apache servers const raw = res.pipe(new PassThrough()); @@ -162,16 +160,13 @@ export default function fetch(url, opts) { } else { body = body.pipe(zlib.createInflateRaw()); } - output = new Response(body, response_options); - resolve(output); + resolve(new Response(body, response_options)); }); return; } // otherwise, use response as-is - output = new Response(body, response_options); - resolve(output); - return; + resolve(new Response(body, response_options)); }); writeToStream(req, request); From eb1959ed0945ba184a3b0baab20998dcbfc57f6c Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 26 Feb 2017 21:46:53 -0800 Subject: [PATCH 089/223] Drop babel-runtime Fixes #243. --- .babelrc | 36 +++++++++++++++++++++++++++++------- package.json | 1 - 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/.babelrc b/.babelrc index 7b0e36281..ff1d1fc56 100644 --- a/.babelrc +++ b/.babelrc @@ -1,20 +1,33 @@ { - "plugins": [ - "transform-runtime" - ], "env": { "test": { "presets": [ - [ "env", { "loose": true, "targets": { "node": 4 } } ] + [ "env", { + "loose": true, + "targets": { "node": 4 }, + "exclude": [ + // skip some almost-compliant features on Node.js v4.x + "transform-es2015-block-scoping", + "transform-es2015-classes", + "transform-es2015-for-of", + ] + } ] ], "plugins": [ - "transform-runtime", "./build/babel-plugin" ] }, "coverage": { "presets": [ - [ "env", { "loose": true, "targets": { "node": 4 } } ] + [ "env", { + "loose": true, + "targets": { "node": 4 }, + "exclude": [ + "transform-es2015-block-scoping", + "transform-es2015-classes", + "transform-es2015-for-of" + ] + } ] ], "plugins": [ [ "istanbul", { "exclude": [ "src/blob.js", "build", "test" ] } ], @@ -23,7 +36,16 @@ }, "rollup": { "presets": [ - [ "env", { "loose": true, "targets": { "node": 4 }, "modules": false } ] + [ "env", { + "loose": true, + "targets": { "node": 4 }, + "exclude": [ + "transform-es2015-block-scoping", + "transform-es2015-classes", + "transform-es2015-for-of" + ], + "modules": false + } ] ] } } diff --git a/package.json b/package.json index 5f9ea00b0..546e4623a 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,6 @@ "whatwg-url": "^4.0.0" }, "dependencies": { - "babel-runtime": "^6.11.6", "encoding": "^0.1.11" } } From da159b9547d04958ac987196e6e6418614361d33 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 26 Feb 2017 21:51:37 -0800 Subject: [PATCH 090/223] Prettify .babelrc --- .babelrc | 82 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/.babelrc b/.babelrc index ff1d1fc56..6a95c25e7 100644 --- a/.babelrc +++ b/.babelrc @@ -1,51 +1,51 @@ { - "env": { - "test": { - "presets": [ - [ "env", { - "loose": true, - "targets": { "node": 4 }, - "exclude": [ - // skip some almost-compliant features on Node.js v4.x - "transform-es2015-block-scoping", - "transform-es2015-classes", - "transform-es2015-for-of", - ] - } ] + env: { + test: { + presets: [ + [ 'env', { + loose: true, + targets: { node: 4 }, + exclude: [ + // skip some almost-compliant features on Node.js v4.x + 'transform-es2015-block-scoping', + 'transform-es2015-classes', + 'transform-es2015-for-of', + ] + } ] ], - "plugins": [ - "./build/babel-plugin" + plugins: [ + './build/babel-plugin' ] }, - "coverage": { - "presets": [ - [ "env", { - "loose": true, - "targets": { "node": 4 }, - "exclude": [ - "transform-es2015-block-scoping", - "transform-es2015-classes", - "transform-es2015-for-of" - ] - } ] + coverage: { + presets: [ + [ 'env', { + loose: true, + targets: { node: 4 }, + exclude: [ + 'transform-es2015-block-scoping', + 'transform-es2015-classes', + 'transform-es2015-for-of' + ] + } ] ], - "plugins": [ - [ "istanbul", { "exclude": [ "src/blob.js", "build", "test" ] } ], - "./build/babel-plugin" + plugins: [ + [ 'istanbul', { exclude: [ 'src/blob.js', 'build', 'test' ] } ], + './build/babel-plugin' ] }, - "rollup": { - "presets": [ - [ "env", { - "loose": true, - "targets": { "node": 4 }, - "exclude": [ - "transform-es2015-block-scoping", - "transform-es2015-classes", - "transform-es2015-for-of" - ], - "modules": false - } ] + rollup: { + presets: [ + [ 'env', { + loose: true, + targets: { node: 4 }, + exclude: [ + 'transform-es2015-block-scoping', + 'transform-es2015-classes', + 'transform-es2015-for-of' + ], + modules: false + } ] ] } } From 047799b742c49b875f6658b3b9ea4981523c3030 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 26 Feb 2017 21:55:58 -0800 Subject: [PATCH 091/223] Remove babel-plugin-transform-runtime --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index 546e4623a..7c8e226af 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,6 @@ "homepage": "https://github.com/bitinn/node-fetch", "devDependencies": { "babel-plugin-istanbul": "^4.0.0", - "babel-plugin-transform-runtime": "^6.15.0", "babel-preset-env": "^1.1.10", "babel-register": "^6.16.3", "chai": "^3.5.0", From a1e76b97e18f28395bdb46e3a849800f52ee8027 Mon Sep 17 00:00:00 2001 From: Grzegorz Graczyk Date: Wed, 22 Feb 2017 09:05:55 +0100 Subject: [PATCH 092/223] More lenient gzip decompression (#239) Ref: https://github.com/nodejs/node/issues/8701#issuecomment-268224481 Ref: https://github.com/request/request/issues/2482 Ref: https://github.com/request/request/pull/2492 Fixes: http://www.mantovanispa.it Fixes: #139 --- src/index.js | 15 ++++++++++++--- test/server.js | 10 ++++++++++ test/test.js | 11 +++++++++++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/index.js b/src/index.js index 1062fb272..c0b905605 100644 --- a/src/index.js +++ b/src/index.js @@ -141,9 +141,18 @@ export default function fetch(url, opts) { return; } + // Be less strict when decoding compressed responses, since sometimes + // servers send slightly invalid responses that are still accepted + // by common browsers. + // Always using Z_SYNC_FLUSH is what cURL does. + const zlibOptions = { + flush: zlib.Z_SYNC_FLUSH, + finishFlush: zlib.Z_SYNC_FLUSH + }; + // for gzip if (codings == 'gzip' || codings == 'x-gzip') { - body = body.pipe(zlib.createGunzip()); + body = body.pipe(zlib.createGunzip(zlibOptions)); resolve(new Response(body, response_options)); return; } @@ -156,9 +165,9 @@ export default function fetch(url, opts) { raw.once('data', chunk => { // see http://stackoverflow.com/questions/37519828 if ((chunk[0] & 0x0F) === 0x08) { - body = body.pipe(zlib.createInflate()); + body = body.pipe(zlib.createInflate(zlibOptions)); } else { - body = body.pipe(zlib.createInflateRaw()); + body = body.pipe(zlib.createInflateRaw(zlibOptions)); } resolve(new Response(body, response_options)); }); diff --git a/test/server.js b/test/server.js index d6016d0ad..ce392f713 100644 --- a/test/server.js +++ b/test/server.js @@ -70,6 +70,16 @@ export default class TestServer { }); } + if (p === '/gzip-truncated') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/plain'); + res.setHeader('Content-Encoding', 'gzip'); + zlib.gzip('hello world', function(err, buffer) { + // truncate the CRC checksum and size check at the end of the stream + res.end(buffer.slice(0, buffer.length - 8)); + }); + } + if (p === '/deflate') { res.statusCode = 200; res.setHeader('Content-Type', 'text/plain'); diff --git a/test/test.js b/test/test.js index 7f6e75571..30a5888c3 100644 --- a/test/test.js +++ b/test/test.js @@ -513,6 +513,17 @@ describe('node-fetch', () => { }); }); + it('should decompress slightly invalid gzip response', function() { + url = `${base}gzip-truncated`; + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain'); + return res.text().then(result => { + expect(result).to.be.a('string'); + expect(result).to.equal('hello world'); + }); + }); + }); + it('should decompress deflate response', function() { url = `${base}deflate`; return fetch(url).then(res => { From 4804a40c177bab6824c3ede85ebda8303fba5c86 Mon Sep 17 00:00:00 2001 From: Gajus Kuizinas Date: Mon, 20 Mar 2017 16:22:49 +0000 Subject: [PATCH 093/223] Add a special case for constructing Headers with Headers (#253) Fixes: #251. --- src/headers.js | 13 +++++++++++++ src/response.js | 1 + test/test.js | 12 ++++++++++++ 3 files changed, 26 insertions(+) diff --git a/src/headers.js b/src/headers.js index 77e02d4de..e330e933d 100644 --- a/src/headers.js +++ b/src/headers.js @@ -34,6 +34,19 @@ export default class Headers { constructor(init = undefined) { this[MAP] = Object.create(null); + if (init instanceof Headers) { + const rawHeaders = init.raw(); + const headerNames = Object.keys(rawHeaders); + + for (const headerName of headerNames) { + for (const value of rawHeaders[headerName]) { + this.append(headerName, value); + } + } + + return; + } + // We don't worry about converting prop to ByteString here as append() // will handle it. if (init == null) { diff --git a/src/response.js b/src/response.js index 0735e3c5b..68e920593 100644 --- a/src/response.js +++ b/src/response.js @@ -23,6 +23,7 @@ export default class Response { this.url = opts.url; this.status = opts.status || 200; this.statusText = opts.statusText || STATUS_CODES[this.status]; + this.headers = new Headers(opts.headers); Object.defineProperty(this, Symbol.toStringTag, { diff --git a/test/test.js b/test/test.js index 30a5888c3..ed0f78b4f 100644 --- a/test/test.js +++ b/test/test.js @@ -1205,6 +1205,18 @@ describe('node-fetch', () => { }); }); + it('should return all headers using raw()', function() { + url = `${base}cookie`; + return fetch(url).then(res => { + const expected = [ + 'a=1', + 'b=1' + ]; + + expect(res.headers.raw()['set-cookie']).to.deep.equal(expected); + }); + }); + it('should allow iterating through all headers with forEach', function() { const headers = new Headers([ ['b', '2'], From 09f2e305570a4db56b8867aac1a79e6268ba7e46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kat=20March=C3=A1n?= Date: Sun, 2 Apr 2017 08:43:46 -0700 Subject: [PATCH 094/223] headers.get: add space to join (#257) The behavior was changed in whatwg/fetch#504. --- src/headers.js | 2 +- test/test.js | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/headers.js b/src/headers.js index e330e933d..ee4003f15 100644 --- a/src/headers.js +++ b/src/headers.js @@ -105,7 +105,7 @@ export default class Headers { return null; } - return list.join(','); + return list.join(', '); } /** diff --git a/test/test.js b/test/test.js index ed0f78b4f..6f91fb4fa 100644 --- a/test/test.js +++ b/test/test.js @@ -1199,7 +1199,7 @@ describe('node-fetch', () => { it('should allow get all responses of a header', function() { url = `${base}cookie`; return fetch(url).then(res => { - const expected = 'a=1,b=1'; + const expected = 'a=1, b=1'; expect(res.headers.get('set-cookie')).to.equal(expected); expect(res.headers.get('Set-Cookie')).to.equal(expected); }); @@ -1233,7 +1233,7 @@ describe('node-fetch', () => { expect(result).to.deep.equal([ ["a", "1"] - , ["b", "2,3"] + , ["b", "2, 3"] , ["c", "4"] ]); }); @@ -1253,7 +1253,7 @@ describe('node-fetch', () => { } expect(result).to.deep.equal([ ['a', '1'], - ['b', '2,3'], + ['b', '2, 3'], ['c', '4'] ]); }); @@ -1269,7 +1269,7 @@ describe('node-fetch', () => { expect(headers.entries()).to.be.iterable .and.to.deep.iterate.over([ ['a', '1'], - ['b', '2,3'], + ['b', '2, 3'], ['c', '4'] ]); }); @@ -1295,7 +1295,7 @@ describe('node-fetch', () => { headers.append('b', '3'); expect(headers.values()).to.be.iterable - .and.to.iterate.over(['1', '2,3', '4']); + .and.to.iterate.over(['1', '2, 3', '4']); }); it('should allow deleting header', function() { @@ -1415,7 +1415,7 @@ describe('node-fetch', () => { ['b', '2'], ['a', '3'] ]); - expect(headers.get('a')).to.equal('1,3'); + expect(headers.get('a')).to.equal('1, 3'); expect(headers.get('b')).to.equal('2'); headers = new Headers([ @@ -1423,7 +1423,7 @@ describe('node-fetch', () => { ['b', '2'], new Map([['a', null], ['3', null]]).keys() ]); - expect(headers.get('a')).to.equal('1,3'); + expect(headers.get('a')).to.equal('1, 3'); expect(headers.get('b')).to.equal('2'); headers = new Headers(new Map([ From ec29e3d2640586bc07cddf894e2aa10461b20ec0 Mon Sep 17 00:00:00 2001 From: Ahmad Nassri Date: Mon, 3 Apr 2017 02:52:52 -0700 Subject: [PATCH 095/223] Allow passing agent option as an object (#236) --- README.md | 2 +- src/index.js | 9 ++++++++- src/request.js | 2 ++ test/test.js | 12 ++++++++++++ 4 files changed, 23 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3b7d5b7c6..4933dbe86 100644 --- a/README.md +++ b/README.md @@ -191,7 +191,7 @@ The default values are shown after each option key. timeout: 0, // req/res timeout in ms, it resets on redirect. 0 to disable (OS limit applies) compress: true, // support gzip/deflate content encoding. false to disable size: 0, // maximum response body size in bytes. 0 to disable - agent: null // http(s).Agent instance, allows custom proxy, certificate etc. + agent: null // http(s).Agent instance or options object, allows custom proxy, certificate etc. } ``` diff --git a/src/index.js b/src/index.js index c0b905605..d532bb29c 100644 --- a/src/index.js +++ b/src/index.js @@ -39,7 +39,14 @@ export default function fetch(url, opts) { const request = new Request(url, opts); const options = getNodeRequestOptions(request); - const send = (options.protocol === 'https:' ? https : http).request; + const protocol = options.protocol === 'https:' ? https : http; + const { agent } = options; + + if (agent && typeof agent === 'object' && !(agent instanceof http.Agent)) { + options.agent = new protocol.Agent(agent); + } + + const send = protocol.request; // http.request only support string as host header, this hack make custom host header possible if (options.headers.host) { diff --git a/src/request.js b/src/request.js index 51b7ec124..67051077a 100644 --- a/src/request.js +++ b/src/request.js @@ -159,6 +159,8 @@ export function getNodeRequestOptions(request) { // HTTP-network fetch step 4 // chunked encoding is handled by Node.js + // Agent normalization is done in fetch(). + return Object.assign({}, parsedURL, { method: request.method, headers: headers.raw(), diff --git a/test/test.js b/test/test.js index 6f91fb4fa..dfdcbc915 100644 --- a/test/test.js +++ b/test/test.js @@ -1335,6 +1335,18 @@ describe('node-fetch', () => { }); }); + it('should create http.Agent class if options.agent is provided as an object', function() { + url = `${base}inspect`; + opts = { + agent: { + keepAlive: true + } + }; + return fetch(url, opts).then(res => res.json()).then(res => { + expect(res.headers['connection']).to.equal('keep-alive'); + }); + }); + it('should ignore unsupported attributes while reading headers', function() { const FakeHeader = function () {}; // prototypes are currently ignored From e5ff203ef41186954b5eea1ddc3b3b0c31c23593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kat=20March=C3=A1n?= Date: Sat, 8 Apr 2017 18:33:46 -0700 Subject: [PATCH 096/223] Revert "Allow passing agent option as an object (#236)" (#263) This reverts commit ec29e3d2640586bc07cddf894e2aa10461b20ec0. This patch prevents any agent being passed in which is not explicitly an instance of `http.Agent`. This makes `node-fetch` no longer compatible with https://npm.im/proxy-agent, which is one example of a library that does not directly inherit from `http.Agent` directly. Sorry for the revert -- I don't have an alternative patch because I don't believe this is something node-fetch should be doing automatically, because of how much of a limitation this could impose. The original PR stated that this was to prevent requiring http/https, and the effect would effectively be to force creation of an Agent on every call. Note that this is already the behavior specified for http.Agent when http.request received `false`. (See the bottom of the [section on http.Agent in the Node docs](https://nodejs.org/dist/latest-v7.x/docs/api/http.html#http_class_http_agent). Cheers, and with apologies to @ahmadnassri for reverting their PR. --- README.md | 2 +- src/index.js | 9 +-------- src/request.js | 2 -- test/test.js | 12 ------------ 4 files changed, 2 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 4933dbe86..3b7d5b7c6 100644 --- a/README.md +++ b/README.md @@ -191,7 +191,7 @@ The default values are shown after each option key. timeout: 0, // req/res timeout in ms, it resets on redirect. 0 to disable (OS limit applies) compress: true, // support gzip/deflate content encoding. false to disable size: 0, // maximum response body size in bytes. 0 to disable - agent: null // http(s).Agent instance or options object, allows custom proxy, certificate etc. + agent: null // http(s).Agent instance, allows custom proxy, certificate etc. } ``` diff --git a/src/index.js b/src/index.js index d532bb29c..c0b905605 100644 --- a/src/index.js +++ b/src/index.js @@ -39,14 +39,7 @@ export default function fetch(url, opts) { const request = new Request(url, opts); const options = getNodeRequestOptions(request); - const protocol = options.protocol === 'https:' ? https : http; - const { agent } = options; - - if (agent && typeof agent === 'object' && !(agent instanceof http.Agent)) { - options.agent = new protocol.Agent(agent); - } - - const send = protocol.request; + const send = (options.protocol === 'https:' ? https : http).request; // http.request only support string as host header, this hack make custom host header possible if (options.headers.host) { diff --git a/src/request.js b/src/request.js index 67051077a..51b7ec124 100644 --- a/src/request.js +++ b/src/request.js @@ -159,8 +159,6 @@ export function getNodeRequestOptions(request) { // HTTP-network fetch step 4 // chunked encoding is handled by Node.js - // Agent normalization is done in fetch(). - return Object.assign({}, parsedURL, { method: request.method, headers: headers.raw(), diff --git a/test/test.js b/test/test.js index dfdcbc915..6f91fb4fa 100644 --- a/test/test.js +++ b/test/test.js @@ -1335,18 +1335,6 @@ describe('node-fetch', () => { }); }); - it('should create http.Agent class if options.agent is provided as an object', function() { - url = `${base}inspect`; - opts = { - agent: { - keepAlive: true - } - }; - return fetch(url, opts).then(res => res.json()).then(res => { - expect(res.headers['connection']).to.equal('keep-alive'); - }); - }); - it('should ignore unsupported attributes while reading headers', function() { const FakeHeader = function () {}; // prototypes are currently ignored From 5f02f36500d0c2d0800113bb843bf4dbe5a3d404 Mon Sep 17 00:00:00 2001 From: Johnny Hauser Date: Sun, 9 Apr 2017 13:42:50 -0500 Subject: [PATCH 097/223] use browser natives when bundled for the browser --- browser.js | 4 ++++ package.json | 1 + 2 files changed, 5 insertions(+) create mode 100644 browser.js diff --git a/browser.js b/browser.js new file mode 100644 index 000000000..7e24a0735 --- /dev/null +++ b/browser.js @@ -0,0 +1,4 @@ +module.exports = exports = window.fetch; +exports.Headers = window.Headers; +exports.Request = window.Request; +exports.Response = window.Response; diff --git a/package.json b/package.json index 7c8e226af..c3f95b866 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,7 @@ "version": "2.0.0-alpha.3", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", + "browser": "./browser", "module": "lib/index.es.js", "files": [ "lib/index.js", From 70e8483ac69ca847c43e1804d44bf656c60b52cd Mon Sep 17 00:00:00 2001 From: Alexander Tesfamichael Date: Sun, 30 Apr 2017 19:37:39 +0200 Subject: [PATCH 098/223] Document response.ok convenience property --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 3b7d5b7c6..193e934b5 100644 --- a/README.md +++ b/README.md @@ -268,6 +268,10 @@ Constructs a new `Response` object. The constructor is identical to that in the Because Node.js does not implement service workers (for which this class was designed), one rarely has to construct a `Response` directly. +#### response.ok + +Convenience property representing if the request ended normally. Will evaluate to true if the response status was greater than or equal to 200 but smaller than 300. + ### Class: Headers From 1797181febc7984607f829fc997401d682926324 Mon Sep 17 00:00:00 2001 From: Alexander Tesfamichael Date: Thu, 11 May 2017 10:02:28 +0200 Subject: [PATCH 099/223] Small improvements to the error-handling wording --- ERROR-HANDLING.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ERROR-HANDLING.md b/ERROR-HANDLING.md index 630032598..7ff8f5464 100644 --- a/ERROR-HANDLING.md +++ b/ERROR-HANDLING.md @@ -2,17 +2,17 @@ Error handling with node-fetch ============================== -Because `window.fetch` isn't designed to transparent about the cause of request errors, we have to come up with our own solutions. +Because `window.fetch` isn't designed to be transparent about the cause of request errors, we have to come up with our own solutions. The basics: -- All [operational errors][joyent-guide] are rejected as [FetchError](https://github.com/bitinn/node-fetch/blob/master/README.md#class-fetcherror), you can handle them all through promise `catch` clause. +- All [operational errors][joyent-guide] are rejected with a [FetchError](https://github.com/bitinn/node-fetch/blob/master/README.md#class-fetcherror). You can handle them all through the promise `catch` clause. -- All errors comes with `err.message` detailing the cause of errors. +- All errors come with an `err.message` detailing the cause of errors. -- All errors originated from `node-fetch` are marked with custom `err.type`. +- All errors originating from `node-fetch` are marked with a custom `err.type`. -- All errors originated from Node.js core are marked with `err.type = 'system'`, and contains addition `err.code` and `err.errno` for error handling, they are alias to error codes thrown by Node.js core. +- All errors originating from Node.js core are marked with `err.type = 'system'`, and in addition contain an `err.code` and an `err.errno` for error handling. These are aliases for error codes thrown by Node.js core. - [Programmer errors][joyent-guide] are either thrown as soon as possible, or rejected with default `Error` with `err.message` for ease of troubleshooting. From a17029046044b12a6e4e6ca1da63adb122e440ca Mon Sep 17 00:00:00 2001 From: David Frank Date: Fri, 12 May 2017 13:04:51 +0800 Subject: [PATCH 100/223] Use full filename at browser field --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c3f95b866..d6a6d2ab7 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "2.0.0-alpha.3", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", - "browser": "./browser", + "browser": "./browser.js", "module": "lib/index.es.js", "files": [ "lib/index.js", From 770388db1b7f3a651f283c72aca799dfc422bf31 Mon Sep 17 00:00:00 2001 From: David Frank Date: Mon, 15 May 2017 19:45:04 +0800 Subject: [PATCH 101/223] only enable Z_SYNC_FLUSH for gzip response --- src/index.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/index.js b/src/index.js index c0b905605..36ed77758 100644 --- a/src/index.js +++ b/src/index.js @@ -141,6 +141,7 @@ export default function fetch(url, opts) { return; } + // For Node v6+ // Be less strict when decoding compressed responses, since sometimes // servers send slightly invalid responses that are still accepted // by common browsers. @@ -165,9 +166,9 @@ export default function fetch(url, opts) { raw.once('data', chunk => { // see http://stackoverflow.com/questions/37519828 if ((chunk[0] & 0x0F) === 0x08) { - body = body.pipe(zlib.createInflate(zlibOptions)); + body = body.pipe(zlib.createInflate()); } else { - body = body.pipe(zlib.createInflateRaw(zlibOptions)); + body = body.pipe(zlib.createInflateRaw()); } resolve(new Response(body, response_options)); }); From 2d986a79996c0d19631405f25809cdf26dc65d4b Mon Sep 17 00:00:00 2001 From: David Frank Date: Mon, 15 May 2017 19:47:12 +0800 Subject: [PATCH 102/223] get ready for alpha 4 release --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d6a6d2ab7..1c0aa79a0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.0.0-alpha.3", + "version": "2.0.0-alpha.4", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", "browser": "./browser.js", From 38d75d5a1f1c94de7e657d45b34cd32bbc8a4b03 Mon Sep 17 00:00:00 2001 From: David Frank Date: Mon, 15 May 2017 20:24:52 +0800 Subject: [PATCH 103/223] Mention 2.x alpha in the main readme --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 193e934b5..9c683c61d 100644 --- a/README.md +++ b/README.md @@ -35,13 +35,22 @@ See Matt Andrews' [isomorphic-fetch](https://github.com/matthew-andrews/isomorph ## Install +Stable release (`v1.x`) + ```sh $ npm install node-fetch --save ``` +Next release (`v2.x`), currently in alpha + +```sh +$ npm install node-fetch@next --save +``` ## Usage +Note that documentation below is up-to-date with `v2.x` releases, [see `v1.x` readme](https://github.com/bitinn/node-fetch/tree/v1.6.3) and [upgrade guide](https://github.com/bitinn/node-fetch/blob/master/UPGRADE-GUIDE.md) if you want to find out the difference. + ```javascript import fetch from 'node-fetch'; // or From 0fff17fbe9c228c6bb15d535f12bfca7d18e7e36 Mon Sep 17 00:00:00 2001 From: David Frank Date: Sat, 20 May 2017 12:15:24 +0800 Subject: [PATCH 104/223] Update README.md Actually fix the readme this time... --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9c683c61d..6390286c2 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ See Matt Andrews' [isomorphic-fetch](https://github.com/matthew-andrews/isomorph - Use native promise, but allow substituting it with [insert your favorite promise library]. - Use native stream for body, on both request and response. - Decode content encoding (gzip/deflate) properly, and convert string output (such as `res.text()` and `res.json()`) to UTF-8 automatically. -- Useful extensions such as timeout, redirect limit, response size limit, [explicit errors][] for troubleshooting. +- Useful extensions such as timeout, redirect limit, response size limit, [explicit errors](https://github.com/bitinn/node-fetch/blob/master/ERROR-HANDLING.md) for troubleshooting. ## Difference from client-side fetch From b38e4a425776274539c34889848ce707705b7ab5 Mon Sep 17 00:00:00 2001 From: David Frank Date: Wed, 24 May 2017 13:09:32 +0800 Subject: [PATCH 105/223] Update README.md point people to the new 1.x branch readme and changelog --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6390286c2..6642fbb14 100644 --- a/README.md +++ b/README.md @@ -35,13 +35,13 @@ See Matt Andrews' [isomorphic-fetch](https://github.com/matthew-andrews/isomorph ## Install -Stable release (`v1.x`) +Stable release (`1.x`) ```sh $ npm install node-fetch --save ``` -Next release (`v2.x`), currently in alpha +Next release (`2.x`), currently in alpha ```sh $ npm install node-fetch@next --save @@ -49,7 +49,7 @@ $ npm install node-fetch@next --save ## Usage -Note that documentation below is up-to-date with `v2.x` releases, [see `v1.x` readme](https://github.com/bitinn/node-fetch/tree/v1.6.3) and [upgrade guide](https://github.com/bitinn/node-fetch/blob/master/UPGRADE-GUIDE.md) if you want to find out the difference. +Note that documentation below is up-to-date with `2.x` releases, [see `1.x` readme](https://github.com/bitinn/node-fetch/tree/1.x), [changelog](https://github.com/bitinn/node-fetch/blob/1.x/CHANGELOG.md) and [2.x upgrade guide](https://github.com/bitinn/node-fetch/blob/master/UPGRADE-GUIDE.md) if you want to find out the difference. ```javascript import fetch from 'node-fetch'; From c02f09c60d5e865a05405c670b095b685d82e4ca Mon Sep 17 00:00:00 2001 From: David Frank Date: Wed, 24 May 2017 13:13:50 +0800 Subject: [PATCH 106/223] Update CHANGELOG.md mention 1.x changelog, so we don't have to update changelog file on both branch. --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5504645e5..ca69bedcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,10 @@ This is a major release. Check [our upgrade guide](https://github.com/bitinn/nod # 1.x release +## backport releases (v1.7.0 and beyond) + +See [changelog on 1.x branch](https://github.com/bitinn/node-fetch/blob/1.x/CHANGELOG.md) for details. + ## v1.6.3 - Enhance: error handling document to explain `FetchError` design From ed2f241df738d0ec6f43907cc144dda3b7fc36b1 Mon Sep 17 00:00:00 2001 From: David Frank Date: Wed, 24 May 2017 13:15:28 +0800 Subject: [PATCH 107/223] Update README.md point to actual readme instead --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6642fbb14..1ae2371a9 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ $ npm install node-fetch@next --save ## Usage -Note that documentation below is up-to-date with `2.x` releases, [see `1.x` readme](https://github.com/bitinn/node-fetch/tree/1.x), [changelog](https://github.com/bitinn/node-fetch/blob/1.x/CHANGELOG.md) and [2.x upgrade guide](https://github.com/bitinn/node-fetch/blob/master/UPGRADE-GUIDE.md) if you want to find out the difference. +Note that documentation below is up-to-date with `2.x` releases, [see `1.x` readme](https://github.com/bitinn/node-fetch/blob/1.x/README.md), [changelog](https://github.com/bitinn/node-fetch/blob/1.x/CHANGELOG.md) and [2.x upgrade guide](https://github.com/bitinn/node-fetch/blob/master/UPGRADE-GUIDE.md) if you want to find out the difference. ```javascript import fetch from 'node-fetch'; From a2b032c0dfc97347970218c7c79cb20e11db88fe Mon Sep 17 00:00:00 2001 From: David Frank Date: Sat, 3 Jun 2017 17:59:26 +0800 Subject: [PATCH 108/223] switch from prepublish to prepare in package.json, also ignore npm 5 lock file --- .gitignore | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f212ad965..97fd1c698 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,7 @@ node_modules # Babel-compiled files lib + +# Ignore package manager lock files +package-lock.json +yarn.lock diff --git a/package.json b/package.json index 1c0aa79a0..c79115ed5 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ }, "scripts": { "build": "cross-env BABEL_ENV=rollup rollup -c", - "prepublish": "npm run build", + "prepare": "npm run build", "test": "cross-env BABEL_ENV=test mocha --compilers js:babel-register test/test.js", "report": "cross-env BABEL_ENV=coverage nyc --reporter lcov --reporter text mocha -R spec test/test.js", "coverage": "cross-env BABEL_ENV=coverage nyc --reporter json --reporter text mocha -R spec test/test.js && codecov -f coverage/coverage-final.json" From c24d591794958be298a075c7274198f244ade3e6 Mon Sep 17 00:00:00 2001 From: David Frank Date: Sat, 3 Jun 2017 18:01:13 +0800 Subject: [PATCH 109/223] fix test server for node 8, which changes keepalive connection handling --- test/server.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/server.js b/test/server.js index ce392f713..6f331f875 100644 --- a/test/server.js +++ b/test/server.js @@ -10,6 +10,9 @@ export default class TestServer { this.server = http.createServer(this.router); this.port = 30001; this.hostname = 'localhost'; + // node 8 default keepalive timeout is 5000ms + // make it shorter here as we want to close server quickly at the end of tests + this.server.keepAliveTimeout = 1000; this.server.on('error', function(err) { console.log(err.stack); }); From fdc2344b63320f15baf1619c8963a7413a0fbab9 Mon Sep 17 00:00:00 2001 From: David Frank Date: Sat, 3 Jun 2017 18:08:33 +0800 Subject: [PATCH 110/223] changelog update --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca69bedcb..a3d5c0d19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,11 +13,12 @@ This is a major release. Check [our upgrade guide](https://github.com/bitinn/nod - Major: Node.js 0.10.x and 0.12.x support is dropped - Major: `require('node-fetch/lib/response')` etc. is now unsupported; use `require('node-fetch').Response` or ES6 module imports -- Enhance: start testing on Node.js 4, 6, 7 +- Enhance: start testing on Node.js 4, 6, 8 LTS - Enhance: use Rollup to produce a distributed bundle (less memory overhead and faster startup) - Enhance: make `Object.prototype.toString()` on Headers, Requests, and Responses return correct class strings - Other: rewrite in ES2015 using Babel - Other: use Codecov for code coverage tracking +- Other: update package.json script for npm 5 ### HTTP requests From 062763a1108d1a600fec6d83f0aacb8356e5c5bf Mon Sep 17 00:00:00 2001 From: David Frank Date: Sat, 3 Jun 2017 18:47:44 +0800 Subject: [PATCH 111/223] 2.0.0-alpha.5 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c79115ed5..d5606f7f3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.0.0-alpha.4", + "version": "2.0.0-alpha.5", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", "browser": "./browser.js", From 553d50e80455ab35cfe11ec950e00d8ce4d59653 Mon Sep 17 00:00:00 2001 From: Jared Kantrowitz Date: Mon, 12 Jun 2017 01:29:50 -0400 Subject: [PATCH 112/223] Accept URLSearchParams as body (#297) Fixes: #296 --- README.md | 10 ++++++++ package.json | 1 + src/body.js | 37 ++++++++++++++++++++++++++++ test/test.js | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 114 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1ae2371a9..e796a7369 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,16 @@ fetch('http://httpbin.org/post', { .then(res => res.json()) .then(json => console.log(json)); +// post form parameters (x-www-form-urlencoded) + +import { URLSearchParams } from 'url'; + +const params = new URLSearchParams(); +params.append('a', 1); +fetch('http://httpbin.org/post', { method: 'POST', body: params }) + .then(res => res.json()) + .then(json => console.log(json)); + // post with form-data (detect multipart) import FormData from 'form-data'; diff --git a/package.json b/package.json index d5606f7f3..66a3b16ed 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "resumer": "0.0.0", "rollup": "^0.41.4", "rollup-plugin-babel": "^2.6.1", + "url-search-params": "^0.9.0", "whatwg-url": "^4.0.0" }, "dependencies": { diff --git a/src/body.js b/src/body.js index 1a35fa653..2c6ae32c4 100644 --- a/src/body.js +++ b/src/body.js @@ -30,6 +30,8 @@ export default function Body(body, { body = null; } else if (typeof body === 'string') { // body is string + } else if (isURLSearchParams(body)) { + // body is a URLSearchParams } else if (body instanceof Blob) { // body is blob } else if (Buffer.isBuffer(body)) { @@ -273,6 +275,31 @@ function convertBody(buffer, headers) { ).toString(); } +/** + * Detect a URLSearchParams object + * ref: https://github.com/bitinn/node-fetch/issues/296#issuecomment-307598143 + * + * @param Object obj Object to detect by type or brand + * @return String + */ +function isURLSearchParams(obj) { + // Duck-typing as a necessary condition. + if (typeof obj !== 'object' || + typeof obj.append !== 'function' || + typeof obj.delete !== 'function' || + typeof obj.get !== 'function' || + typeof obj.getAll !== 'function' || + typeof obj.has !== 'function' || + typeof obj.set !== 'function') { + return false; + } + + // Brand-checking and more duck-typing as optional condition. + return obj.constructor.name === 'URLSearchParams' || + Object.prototype.toString.call(obj) === '[object URLSearchParams]' || + typeof obj.sort === 'function'; +} + /** * Clone body given Res/Req instance * @@ -324,6 +351,9 @@ export function extractContentType(instance) { } else if (typeof body === 'string') { // body is string return 'text/plain;charset=UTF-8'; + } else if (isURLSearchParams(body)) { + // body is a URLSearchParams + return 'application/x-www-form-urlencoded;charset=UTF-8'; } else if (body instanceof Blob) { // body is blob return body.type || null; @@ -350,6 +380,9 @@ export function getTotalBytes(instance) { } else if (typeof body === 'string') { // body is string return Buffer.byteLength(body); + } else if (isURLSearchParams(body)) { + // body is URLSearchParams + return Buffer.byteLength(String(body)); } else if (body instanceof Blob) { // body is blob return body.size; @@ -380,6 +413,10 @@ export function writeToStream(dest, instance) { // body is string dest.write(body); dest.end(); + } else if (isURLSearchParams(body)) { + // body is URLSearchParams + dest.write(Buffer.from(String(body))); + dest.end(); } else if (body instanceof Blob) { // body is blob dest.write(body[BUFFER]); diff --git a/test/test.js b/test/test.js index 6f91fb4fa..b6cf2ca89 100644 --- a/test/test.js +++ b/test/test.js @@ -9,10 +9,12 @@ import {spawn} from 'child_process'; import * as stream from 'stream'; import resumer from 'resumer'; import FormData from 'form-data'; -import {parse as parseURL} from 'url'; +import URLSearchParams_Polyfill from 'url-search-params'; +import {parse as parseURL, URLSearchParams} from 'url'; import {URL} from 'whatwg-url'; import * as http from 'http'; import * as fs from 'fs'; +import * as path from 'path'; chai.use(chaiPromised); chai.use(chaiIterator); @@ -794,7 +796,7 @@ describe('node-fetch', () => { it('should allow POST request with form-data using stream as body', function() { const form = new FormData(); - form.append('my_field', fs.createReadStream('test/dummy.txt')); + form.append('my_field', fs.createReadStream(path.join(__dirname, 'dummy.txt'))); url = `${base}multipart`; opts = { @@ -853,6 +855,68 @@ describe('node-fetch', () => { }); }); + const itUSP = typeof URLSearchParams === 'function' ? it : it.skip; + itUSP('should allow POST request with URLSearchParams as body', function() { + const params = new URLSearchParams(); + params.append('a','1'); + + url = `${base}inspect`; + opts = { + method: 'POST', + body: params, + }; + return fetch(url, opts).then(res => { + return res.json(); + }).then(res => { + expect(res.method).to.equal('POST'); + expect(res.headers['content-type']).to.equal('application/x-www-form-urlencoded;charset=UTF-8'); + expect(res.headers['content-length']).to.equal('3'); + expect(res.body).to.equal('a=1'); + }); + }); + + itUSP('should still recognize URLSearchParams when extended', function() { + class CustomSearchParams extends URLSearchParams {} + const params = new CustomSearchParams(); + params.append('a','1'); + + url = `${base}inspect`; + opts = { + method: 'POST', + body: params, + }; + return fetch(url, opts).then(res => { + return res.json(); + }).then(res => { + expect(res.method).to.equal('POST'); + expect(res.headers['content-type']).to.equal('application/x-www-form-urlencoded;charset=UTF-8'); + expect(res.headers['content-length']).to.equal('3'); + expect(res.body).to.equal('a=1'); + }); + }); + + /* for 100% code coverage, checks for duck-typing-only detection + * where both constructor.name and brand tests fail */ + it('should still recognize URLSearchParams when extended from polyfill', function() { + class CustomPolyfilledSearchParams extends URLSearchParams_Polyfill {} + const params = new CustomPolyfilledSearchParams(); + params.append('a','1'); + + url = `${base}inspect`; + opts = { + method: 'POST', + body: params, + }; + return fetch(url, opts).then(res => { + return res.json(); + }).then(res => { + expect(res.method).to.equal('POST'); + expect(res.headers['content-type']).to.equal('application/x-www-form-urlencoded;charset=UTF-8'); + expect(res.headers['content-length']).to.equal('3'); + expect(res.body).to.equal('a=1'); + }); + }); + it('should overwrite Content-Length if possible', function() { url = `${base}inspect`; // note that fetch simply calls tostring on an object From 18040417264938935e175a8963f1948110942215 Mon Sep 17 00:00:00 2001 From: jared kantrowitz Date: Mon, 19 Jun 2017 19:19:06 -0400 Subject: [PATCH 113/223] wrap JSON.parse's SynaxError with FetchError --- src/body.js | 9 ++++++++- test/test.js | 16 +++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/body.js b/src/body.js index 2c6ae32c4..c3c193c02 100644 --- a/src/body.js +++ b/src/body.js @@ -87,7 +87,14 @@ Body.prototype = { * @return Promise */ json() { - return consumeBody.call(this).then(buffer => JSON.parse(buffer.toString())); + return consumeBody.call(this).then((buffer) => { + return JSON.parse(buffer.toString()); + }).catch((err) => { + if (err instanceof SyntaxError) { + return Body.Promise.reject(new FetchError(`invalid json response body at ${this.url} reason: ${err.message}`, 'invalid-json')); + } + throw err; + }) }, /** diff --git a/test/test.js b/test/test.js index b6cf2ca89..5ecd36165 100644 --- a/test/test.js +++ b/test/test.js @@ -446,7 +446,9 @@ describe('node-fetch', () => { url = `${base}error/json`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('application/json'); - return expect(res.json()).to.eventually.be.rejectedWith(Error); + return expect(res.json()).to.eventually.be.rejected + .and.be.an.instanceOf(FetchError) + .and.include({ type: 'invalid-json' }); }); }); @@ -463,6 +465,18 @@ describe('node-fetch', () => { }); }); + it('should reject when trying to parse no content response as json', function() { + url = `${base}no-content`; + return fetch(url).then(res => { + expect(res.status).to.equal(204); + expect(res.statusText).to.equal('No Content'); + expect(res.ok).to.be.true; + return expect(res.json()).to.eventually.be.rejected + .and.be.an.instanceOf(FetchError) + .and.include({ type: 'invalid-json' }); + }); + }); + it('should handle no content response with gzip encoding', function() { url = `${base}no-content/gzip`; return fetch(url).then(res => { From 6b990d0631ec130bb27a531b1cfb3f6f1a1dd7aa Mon Sep 17 00:00:00 2001 From: jared kantrowitz Date: Mon, 19 Jun 2017 20:21:46 -0400 Subject: [PATCH 114/223] use synchronous try/catch to fix codecov missing lines --- src/body.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/body.js b/src/body.js index c3c193c02..61fd7267c 100644 --- a/src/body.js +++ b/src/body.js @@ -88,12 +88,11 @@ Body.prototype = { */ json() { return consumeBody.call(this).then((buffer) => { - return JSON.parse(buffer.toString()); - }).catch((err) => { - if (err instanceof SyntaxError) { + try { + return JSON.parse(buffer.toString()); + } catch (err) { return Body.Promise.reject(new FetchError(`invalid json response body at ${this.url} reason: ${err.message}`, 'invalid-json')); } - throw err; }) }, From 76e8ad8b24c61af3cf4d34b87343f99b67e4e8c9 Mon Sep 17 00:00:00 2001 From: Jared Kantrowitz Date: Mon, 19 Jun 2017 20:41:05 -0400 Subject: [PATCH 115/223] refactor deprecated `new Buffer` with Buffer.from and Buffer.alloc (#299) * refactor deprecated `new Buffer` with Buffer.from and Buffer.alloc * don't need new ArrayBuffer instance when already an ArrayBuffer --- src/blob.js | 6 +++--- src/body.js | 6 +++--- test/test.js | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/blob.js b/src/blob.js index 89e6ef2de..36d65bff7 100644 --- a/src/blob.js +++ b/src/blob.js @@ -31,13 +31,13 @@ export default class Blob { if (element instanceof Buffer) { buffer = element; } else if (ArrayBuffer.isView(element)) { - buffer = new Buffer(new Uint8Array(element.buffer, element.byteOffset, element.byteLength)); + buffer = Buffer.from(element.buffer, element.byteOffset, element.byteLength); } else if (element instanceof ArrayBuffer) { - buffer = new Buffer(new Uint8Array(element)); + buffer = Buffer.from(element); } else if (element instanceof Blob) { buffer = element[BUFFER]; } else { - buffer = new Buffer(typeof element === 'string' ? element : String(element)); + buffer = Buffer.from(typeof element === 'string' ? element : String(element)); } buffers.push(buffer); } diff --git a/src/body.js b/src/body.js index 61fd7267c..2d919b7dd 100644 --- a/src/body.js +++ b/src/body.js @@ -151,12 +151,12 @@ function consumeBody(body) { // body is null if (this.body === null) { - return Body.Promise.resolve(new Buffer(0)); + return Body.Promise.resolve(Buffer.alloc(0)); } // body is string if (typeof this.body === 'string') { - return Body.Promise.resolve(new Buffer(this.body)); + return Body.Promise.resolve(Buffer.from(this.body)); } // body is blob @@ -171,7 +171,7 @@ function consumeBody(body) { // istanbul ignore if: should never happen if (!(this.body instanceof Stream)) { - return Body.Promise.resolve(new Buffer(0)); + return Body.Promise.resolve(Buffer.alloc(0)); } // body is stream diff --git a/test/test.js b/test/test.js index 5ecd36165..00731e6db 100644 --- a/test/test.js +++ b/test/test.js @@ -720,7 +720,7 @@ describe('node-fetch', () => { url = `${base}inspect`; opts = { method: 'POST' - , body: new Buffer('a=1', 'utf-8') + , body: Buffer.from('a=1', 'utf-8') }; return fetch(url, opts).then(res => { return res.json(); @@ -1432,7 +1432,7 @@ describe('node-fetch', () => { res.j = NaN; res.k = true; res.l = false; - res.m = new Buffer('test'); + res.m = Buffer.from('test'); const h1 = new Headers(res); h1.set('n', [1, 2]); @@ -1728,7 +1728,7 @@ describe('node-fetch', () => { }); it('should support buffer as body in Response constructor', function() { - const res = new Response(new Buffer('a=1')); + const res = new Response(Buffer.from('a=1')); return res.text().then(result => { expect(result).to.equal('a=1'); }); @@ -1824,7 +1824,7 @@ describe('node-fetch', () => { url = base; var req = new Request(url, { method: 'POST', - body: new Buffer('a=1') + body: Buffer.from('a=1') }); expect(req.url).to.equal(url); return req.blob().then(function(result) { From 432cd8a31e3c3d1a1d640be5aa914aa415f60a3f Mon Sep 17 00:00:00 2001 From: Jared Kantrowitz Date: Sun, 2 Jul 2017 12:32:48 -0400 Subject: [PATCH 116/223] remove `encoding` as a dependency, throw detailed Error if not installed (#302) * remove `encoding` as a dependency, throw descriptive Error if textConverted() is used without it in env * remove rollup ext dep resolution since we don't need it * switch to programmer error, rm unneeded test conditions, bump timeout for slow CI * more kill `encoding` dep PR changes keep blank "dependencies" prop in package.json so rollup's external config func doesn't seize add ext dep checks back to the rollup config no implicit var clarify test comment * [squash] alter travis cfg to test with and without `encoding`, various fix devDeps separate `encoding` tests to their own block * [squash] fixing nits * [squash] ci: full matrix of form-data versions and encoding existence and nits --- .travis.yml | 3 + package.json | 4 +- src/body.js | 8 +- test/server.js | 4 +- test/test.js | 233 +++++++++++++++++++++++++++---------------------- 5 files changed, 144 insertions(+), 108 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4a706efab..1bc83a8f2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,9 +5,12 @@ node_js: - "node" env: - FORMDATA_VERSION=1.0.0 + - FORMDATA_VERSION=1.0.0 ENCODING=yes - FORMDATA_VERSION=2.1.0 + - FORMDATA_VERSION=2.1.0 ENCODING=yes before_script: - 'if [ "$FORMDATA_VERSION" ]; then npm install form-data@^$FORMDATA_VERSION; fi' + - 'if [ "$ENCODING" = "yes" ]; then npm install encoding; fi' before_install: npm install -g npm script: npm run coverage cache: diff --git a/package.json b/package.json index 66a3b16ed..4ff578bd5 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,5 @@ "url-search-params": "^0.9.0", "whatwg-url": "^4.0.0" }, - "dependencies": { - "encoding": "^0.1.11" - } + "dependencies": { } } diff --git a/src/body.js b/src/body.js index 2d919b7dd..3b5696172 100644 --- a/src/body.js +++ b/src/body.js @@ -5,13 +5,15 @@ * Body interface provides common methods for Request and Response */ -import {convert} from 'encoding'; import Stream, {PassThrough} from 'stream'; import Blob, {BUFFER} from './blob.js'; import FetchError from './fetch-error.js'; const DISTURBED = Symbol('disturbed'); +let convert; +try { convert = require('encoding').convert; } catch(e) {} + /** * Body class * @@ -231,6 +233,10 @@ function consumeBody(body) { * @return String */ function convertBody(buffer, headers) { + if (typeof convert !== 'function') { + throw new Error('The package `encoding` must be installed to use the textConverted() function'); + } + const ct = headers.get('content-type'); let charset = 'utf-8'; let res, str; diff --git a/test/server.js b/test/server.js index 6f331f875..6f5b11620 100644 --- a/test/server.js +++ b/test/server.js @@ -2,9 +2,11 @@ import * as http from 'http'; import { parse } from 'url'; import * as zlib from 'zlib'; import * as stream from 'stream'; -import { convert } from 'encoding'; import { multipart as Multipart } from 'parted'; +let convert; +try { convert = require('encoding').convert; } catch(e) {} + export default class TestServer { constructor() { this.server = http.createServer(this.router); diff --git a/test/test.js b/test/test.js index 00731e6db..3a50045e0 100644 --- a/test/test.js +++ b/test/test.js @@ -16,6 +16,9 @@ import * as http from 'http'; import * as fs from 'fs'; import * as path from 'path'; +let convert; +try { convert = require('encoding').convert; } catch(e) { } + chai.use(chaiPromised); chai.use(chaiIterator); chai.use(chaiString); @@ -1090,109 +1093,6 @@ describe('node-fetch', () => { }); }); - it('should only use UTF-8 decoding with text()', function() { - url = `${base}encoding/euc-jp`; - return fetch(url).then(res => { - expect(res.status).to.equal(200); - return res.text().then(result => { - expect(result).to.equal('\ufffd\ufffd\ufffd\u0738\ufffd'); - }); - }); - }); - - it('should support encoding decode, xml dtd detect', function() { - url = `${base}encoding/euc-jp`; - return fetch(url).then(res => { - expect(res.status).to.equal(200); - return res.textConverted().then(result => { - expect(result).to.equal('日本語'); - }); - }); - }); - - it('should support encoding decode, content-type detect', function() { - url = `${base}encoding/shift-jis`; - return fetch(url).then(res => { - expect(res.status).to.equal(200); - return res.textConverted().then(result => { - expect(result).to.equal('
日本語
'); - }); - }); - }); - - it('should support encoding decode, html5 detect', function() { - url = `${base}encoding/gbk`; - return fetch(url).then(res => { - expect(res.status).to.equal(200); - return res.textConverted().then(result => { - expect(result).to.equal('
中文
'); - }); - }); - }); - - it('should support encoding decode, html4 detect', function() { - url = `${base}encoding/gb2312`; - return fetch(url).then(res => { - expect(res.status).to.equal(200); - return res.textConverted().then(result => { - expect(result).to.equal('
中文
'); - }); - }); - }); - - it('should default to utf8 encoding', function() { - url = `${base}encoding/utf8`; - return fetch(url).then(res => { - expect(res.status).to.equal(200); - expect(res.headers.get('content-type')).to.be.null; - return res.textConverted().then(result => { - expect(result).to.equal('中文'); - }); - }); - }); - - it('should support uncommon content-type order, charset in front', function() { - url = `${base}encoding/order1`; - return fetch(url).then(res => { - expect(res.status).to.equal(200); - return res.textConverted().then(result => { - expect(result).to.equal('中文'); - }); - }); - }); - - it('should support uncommon content-type order, end with qs', function() { - url = `${base}encoding/order2`; - return fetch(url).then(res => { - expect(res.status).to.equal(200); - return res.textConverted().then(result => { - expect(result).to.equal('中文'); - }); - }); - }); - - it('should support chunked encoding, html4 detect', function() { - url = `${base}encoding/chunked`; - return fetch(url).then(res => { - expect(res.status).to.equal(200); - const padding = 'a'.repeat(10); - return res.textConverted().then(result => { - expect(result).to.equal(`${padding}
日本語
`); - }); - }); - }); - - it('should only do encoding detection up to 1024 bytes', function() { - url = `${base}encoding/invalid`; - return fetch(url).then(res => { - expect(res.status).to.equal(200); - const padding = 'a'.repeat(1200); - return res.textConverted().then(result => { - expect(result).to.not.equal(`${padding}中文`); - }); - }); - }); - it('should allow piping response body as stream', function() { url = `${base}hello`; return fetch(url).then(res => { @@ -1930,3 +1830,130 @@ function streamToPromise(stream, dataHandler) { stream.on('error', reject); }); } + +describe('external encoding', () => { + const hasEncoding = typeof convert === 'function'; + + describe('with optional `encoding`', function() { + before(function() { + if(!hasEncoding) this.skip(); + }); + + it('should only use UTF-8 decoding with text()', function() { + url = `${base}encoding/euc-jp`; + return fetch(url).then(res => { + expect(res.status).to.equal(200); + return res.text().then(result => { + expect(result).to.equal('\ufffd\ufffd\ufffd\u0738\ufffd'); + }); + }); + }); + + it('should support encoding decode, xml dtd detect', function() { + url = `${base}encoding/euc-jp`; + return fetch(url).then(res => { + expect(res.status).to.equal(200); + return res.textConverted().then(result => { + expect(result).to.equal('日本語'); + }); + }); + }); + + it('should support encoding decode, content-type detect', function() { + url = `${base}encoding/shift-jis`; + return fetch(url).then(res => { + expect(res.status).to.equal(200); + return res.textConverted().then(result => { + expect(result).to.equal('
日本語
'); + }); + }); + }); + + it('should support encoding decode, html5 detect', function() { + url = `${base}encoding/gbk`; + return fetch(url).then(res => { + expect(res.status).to.equal(200); + return res.textConverted().then(result => { + expect(result).to.equal('
中文
'); + }); + }); + }); + + it('should support encoding decode, html4 detect', function() { + url = `${base}encoding/gb2312`; + return fetch(url).then(res => { + expect(res.status).to.equal(200); + return res.textConverted().then(result => { + expect(result).to.equal('
中文
'); + }); + }); + }); + + it('should default to utf8 encoding', function() { + url = `${base}encoding/utf8`; + return fetch(url).then(res => { + expect(res.status).to.equal(200); + expect(res.headers.get('content-type')).to.be.null; + return res.textConverted().then(result => { + expect(result).to.equal('中文'); + }); + }); + }); + + it('should support uncommon content-type order, charset in front', function() { + url = `${base}encoding/order1`; + return fetch(url).then(res => { + expect(res.status).to.equal(200); + return res.textConverted().then(result => { + expect(result).to.equal('中文'); + }); + }); + }); + + it('should support uncommon content-type order, end with qs', function() { + url = `${base}encoding/order2`; + return fetch(url).then(res => { + expect(res.status).to.equal(200); + return res.textConverted().then(result => { + expect(result).to.equal('中文'); + }); + }); + }); + + it('should support chunked encoding, html4 detect', function() { + url = `${base}encoding/chunked`; + return fetch(url).then(res => { + expect(res.status).to.equal(200); + const padding = 'a'.repeat(10); + return res.textConverted().then(result => { + expect(result).to.equal(`${padding}
日本語
`); + }); + }); + }); + + it('should only do encoding detection up to 1024 bytes', function() { + url = `${base}encoding/invalid`; + return fetch(url).then(res => { + expect(res.status).to.equal(200); + const padding = 'a'.repeat(1200); + return res.textConverted().then(result => { + expect(result).to.not.equal(`${padding}中文`); + }); + }); + }); + }); + + describe('without optional `encoding`', function() { + before(function() { + if (hasEncoding) this.skip() + }); + + it('should throw a FetchError if res.textConverted() is called without `encoding` in require cache', () => { + url = `${base}hello`; + return fetch(url).then((res) => { + return expect(res.textConverted()).to.eventually.be.rejected + .and.have.property('message').which.includes('encoding') + }); + }); + }); +}); From 76506a8b2b7b5648c068126c548726c003936895 Mon Sep 17 00:00:00 2001 From: jared kantrowitz Date: Fri, 7 Jul 2017 14:04:30 -0400 Subject: [PATCH 117/223] explicitly exclude node 5 from engines directive --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4ff578bd5..c88be3458 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "lib/index.es.js" ], "engines": { - "node": ">=4" + "node": "4.x || >=6.0.0" }, "scripts": { "build": "cross-env BABEL_ENV=rollup rollup -c", From 2b359c1ea161edf5778490b9303a40ee11fef7ea Mon Sep 17 00:00:00 2001 From: David Frank Date: Sat, 22 Jul 2017 01:59:59 +0800 Subject: [PATCH 118/223] V2 alpha.6 (#313) 2.0.0-alpha.6 * changelog update for new release * version bump --- CHANGELOG.md | 3 +++ package.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3d5c0d19..e44a09296 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ This is a major release. Check [our upgrade guide](https://github.com/bitinn/nod - Other: rewrite in ES2015 using Babel - Other: use Codecov for code coverage tracking - Other: update package.json script for npm 5 +- Other: `encoding` module is now optional ### HTTP requests @@ -33,6 +34,8 @@ This is a major release. Check [our upgrade guide](https://github.com/bitinn/nod - Major: throw error when a `GET` or `HEAD` Request is constructed with a non-null body (per spec) - Enhance: add `response.arrayBuffer()` (also applies to Requests) - Enhance: add experimental `response.blob()` (also applies to Requests) +- Enhance: `URLSearchParams` is now accepted as a body +- Enhance: wrap `response.json()` json parsing error as `FetchError` - Fix: fix Request and Response with `null` body ### Headers class diff --git a/package.json b/package.json index 4ff578bd5..f917bc0c3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.0.0-alpha.5", + "version": "2.0.0-alpha.6", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", "browser": "./browser.js", From 70cd403914db04415a2fe6a3c9715ff61b9dd2be Mon Sep 17 00:00:00 2001 From: David Frank Date: Sat, 22 Jul 2017 21:04:59 +0800 Subject: [PATCH 119/223] do not make request to example.com, use local server instead, avoid UnhandledPromiseRejectionWarning altogether --- package.json | 2 +- test/test.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 75e1e3547..4d28a94a2 100644 --- a/package.json +++ b/package.json @@ -56,5 +56,5 @@ "url-search-params": "^0.9.0", "whatwg-url": "^4.0.0" }, - "dependencies": { } + "dependencies": {} } diff --git a/test/test.js b/test/test.js index 3a50045e0..000594e74 100644 --- a/test/test.js +++ b/test/test.js @@ -58,14 +58,14 @@ after(done => { describe('node-fetch', () => { it('should return a promise', function() { - url = 'http://example.com/'; + url = `${base}hello`; const p = fetch(url); expect(p).to.be.an.instanceof(fetch.Promise); expect(p).to.have.property('then'); }); it('should allow custom promise', function() { - url = 'http://example.com/'; + url = `${base}hello`; const old = fetch.Promise; fetch.Promise = then; expect(fetch(url)).to.be.an.instanceof(then); @@ -74,7 +74,7 @@ describe('node-fetch', () => { }); it('should throw error when no promise implementation are found', function() { - url = 'http://example.com/'; + url = `${base}hello`; const old = fetch.Promise; fetch.Promise = undefined; expect(() => { From 9bd099a7796561326100b8002063bf26a65621b6 Mon Sep 17 00:00:00 2001 From: David Frank Date: Sat, 22 Jul 2017 21:11:25 +0800 Subject: [PATCH 120/223] workaround slow nodejs 8 spawn on travis ci by increasing timeout for test cases --- test/test.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/test.js b/test/test.js index 000594e74..7c6795d5b 100644 --- a/test/test.js +++ b/test/test.js @@ -626,24 +626,24 @@ describe('node-fetch', () => { }); it('should clear internal timeout on fetch response', function (done) { - this.timeout(1000); - spawn('node', ['-e', `require('./')('${base}hello', { timeout: 5000 })`]) + this.timeout(2000); + spawn('node', ['-e', `require('./')('${base}hello', { timeout: 10000 })`]) .on('exit', () => { done(); }); }); it('should clear internal timeout on fetch redirect', function (done) { - this.timeout(1000); - spawn('node', ['-e', `require('./')('${base}redirect/301', { timeout: 5000 })`]) + this.timeout(2000); + spawn('node', ['-e', `require('./')('${base}redirect/301', { timeout: 10000 })`]) .on('exit', () => { done(); }); }); it('should clear internal timeout on fetch error', function (done) { - this.timeout(1000); - spawn('node', ['-e', `require('./')('${base}error/reset', { timeout: 5000 })`]) + this.timeout(2000); + spawn('node', ['-e', `require('./')('${base}error/reset', { timeout: 10000 })`]) .on('exit', () => { done(); }); From 2edb4c026667e5c8e7c808a3973bd94d193b2d0f Mon Sep 17 00:00:00 2001 From: David Frank Date: Tue, 25 Jul 2017 12:10:11 +0800 Subject: [PATCH 121/223] 2.0.0-alpha.7 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4d28a94a2..002a27f66 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.0.0-alpha.6", + "version": "2.0.0-alpha.7", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", "browser": "./browser.js", From 60cf26c2f3baf566c15632b723664b47f5b1f2db Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Thu, 27 Jul 2017 00:13:57 +0800 Subject: [PATCH 122/223] Use require() for Node.js core modules Fixes: #318 --- package.json | 1 - rollup.config.js | 4 ---- src/body.js | 6 ++++-- src/index.js | 12 ++++++------ src/request.js | 3 ++- src/response.js | 3 ++- 6 files changed, 14 insertions(+), 15 deletions(-) diff --git a/package.json b/package.json index 002a27f66..faaed010c 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,6 @@ "codecov": "^1.0.1", "cross-env": "^3.1.4", "form-data": ">=1.0.0", - "is-builtin-module": "^1.0.0", "mocha": "^3.1.2", "nyc": "^10.0.0", "parted": "^0.1.1", diff --git a/rollup.config.js b/rollup.config.js index 17e129272..c165e4aa4 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -1,4 +1,3 @@ -import isBuiltin from 'is-builtin-module'; import babel from 'rollup-plugin-babel'; import tweakDefault from './build/rollup-plugin'; @@ -18,9 +17,6 @@ export default { { dest: 'lib/index.es.js', format: 'es' } ], external: function (id) { - if (isBuiltin(id)) { - return true; - } id = id.split('/').slice(0, id[0] === '@' ? 2 : 1).join('/'); return !!require('./package.json').dependencies[id]; } diff --git a/src/body.js b/src/body.js index 3b5696172..999380858 100644 --- a/src/body.js +++ b/src/body.js @@ -5,10 +5,12 @@ * Body interface provides common methods for Request and Response */ -import Stream, {PassThrough} from 'stream'; -import Blob, {BUFFER} from './blob.js'; +import Blob, { BUFFER } from './blob.js'; import FetchError from './fetch-error.js'; +const Stream = require('stream'); +const { PassThrough } = require('stream'); + const DISTURBED = Symbol('disturbed'); let convert; diff --git a/src/index.js b/src/index.js index 36ed77758..17a272206 100644 --- a/src/index.js +++ b/src/index.js @@ -5,18 +5,18 @@ * a request API compatible with window.fetch */ -import {resolve as resolve_url} from 'url'; -import * as http from 'http'; -import * as https from 'https'; -import * as zlib from 'zlib'; -import {PassThrough} from 'stream'; - import Body, { writeToStream } from './body'; import Response from './response'; import Headers from './headers'; import Request, { getNodeRequestOptions } from './request'; import FetchError from './fetch-error'; +const http = require('http'); +const https = require('https'); +const { PassThrough } = require('stream'); +const { resolve: resolve_url } = require('url'); +const zlib = require('zlib'); + /** * Fetch function * diff --git a/src/request.js b/src/request.js index 51b7ec124..5b1d012e9 100644 --- a/src/request.js +++ b/src/request.js @@ -5,10 +5,11 @@ * Request class contains server only options */ -import { format as format_url, parse as parse_url } from 'url'; import Headers from './headers.js'; import Body, { clone, extractContentType, getTotalBytes } from './body'; +const { format: format_url, parse: parse_url } = require('url'); + const PARSED_URL = Symbol('url'); /** diff --git a/src/response.js b/src/response.js index 68e920593..bd5dd2a54 100644 --- a/src/response.js +++ b/src/response.js @@ -5,10 +5,11 @@ * Response class provides content decoding */ -import { STATUS_CODES } from 'http'; import Headers from './headers.js'; import Body, { clone } from './body'; +const { STATUS_CODES } = require('http'); + /** * Response class * From ec852307f2050a24ffc51bad5e5cd99add54e030 Mon Sep 17 00:00:00 2001 From: David Frank Date: Thu, 27 Jul 2017 00:43:40 +0800 Subject: [PATCH 123/223] prepare next release, and start to take note in changelog on what each alpha release fixes --- CHANGELOG.md | 3 ++- package.json | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e44a09296..f1097c818 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,12 +19,13 @@ This is a major release. Check [our upgrade guide](https://github.com/bitinn/nod - Other: rewrite in ES2015 using Babel - Other: use Codecov for code coverage tracking - Other: update package.json script for npm 5 -- Other: `encoding` module is now optional +- Other: `encoding` module is now optional (alpha.7) ### HTTP requests - Major: overwrite user's `Content-Length` if we can be sure our information is correct (per spec) - Fix: support WHATWG URL objects, created by `whatwg-url` package or `require('url').URL` in Node.js 7+ +- Fix: a regression in older v2 build where `index.es.js` doesn't require https module, causing HTTPS request to fail (alpha.8) ### Response and Request classes diff --git a/package.json b/package.json index faaed010c..70b70953e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.0.0-alpha.7", + "version": "2.0.0-alpha.8", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", "browser": "./browser.js", From e7c1ef88ede64c1a8fa9cfc5a2690d10c11225ce Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Fri, 28 Jul 2017 12:12:54 +0800 Subject: [PATCH 124/223] Update deps (#320) --- package.json | 14 +++++++------- test/test.js | 15 ++++++++------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/package.json b/package.json index 70b70953e..173c4e4ee 100644 --- a/package.json +++ b/package.json @@ -39,21 +39,21 @@ "babel-preset-env": "^1.1.10", "babel-register": "^6.16.3", "chai": "^3.5.0", - "chai-as-promised": "^6.0.0", + "chai-as-promised": "^7.1.1", "chai-iterator": "^1.1.1", "chai-string": "^1.3.0", - "codecov": "^1.0.1", - "cross-env": "^3.1.4", + "codecov": "^2.2.0", + "cross-env": "^5.0.1", "form-data": ">=1.0.0", "mocha": "^3.1.2", - "nyc": "^10.0.0", + "nyc": "^11.1.0", "parted": "^0.1.1", - "promise": "^7.1.1", + "promise": "^8.0.1", "resumer": "0.0.0", - "rollup": "^0.41.4", + "rollup": "^0.45.2", "rollup-plugin-babel": "^2.6.1", "url-search-params": "^0.9.0", - "whatwg-url": "^4.0.0" + "whatwg-url": "^5.0.0" }, "dependencies": {} } diff --git a/test/test.js b/test/test.js index 7c6795d5b..91e9e076e 100644 --- a/test/test.js +++ b/test/test.js @@ -5,16 +5,17 @@ import chaiPromised from 'chai-as-promised'; import chaiIterator from 'chai-iterator'; import chaiString from 'chai-string'; import then from 'promise'; -import {spawn} from 'child_process'; -import * as stream from 'stream'; import resumer from 'resumer'; import FormData from 'form-data'; import URLSearchParams_Polyfill from 'url-search-params'; -import {parse as parseURL, URLSearchParams} from 'url'; -import {URL} from 'whatwg-url'; -import * as http from 'http'; -import * as fs from 'fs'; -import * as path from 'path'; +import { URL } from 'whatwg-url'; + +const { spawn } = require('child_process'); +const http = require('http'); +const fs = require('fs'); +const path = require('path'); +const stream = require('stream'); +const { parse: parseURL, URLSearchParams } = require('url'); let convert; try { convert = require('encoding').convert; } catch(e) { } From e2603d31c767cd5111df2ff6e2977577840656a4 Mon Sep 17 00:00:00 2001 From: Johnny Hauser Date: Wed, 20 Sep 2017 03:18:49 -0500 Subject: [PATCH 125/223] add browser.js to npm published files (#343) --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 173c4e4ee..a12989271 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "module": "lib/index.es.js", "files": [ "lib/index.js", - "lib/index.es.js" + "lib/index.es.js", + "browser.js" ], "engines": { "node": "4.x || >=6.0.0" From cf5975cb5ca7d60cb53ff4791a031141622f0d28 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sat, 23 Sep 2017 22:06:41 -0700 Subject: [PATCH 126/223] Support TypeScript import (#332) Fixes #307. --- src/index.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/index.js b/src/index.js index 17a272206..e6ac30b46 100644 --- a/src/index.js +++ b/src/index.js @@ -192,6 +192,9 @@ export default function fetch(url, opts) { */ fetch.isRedirect = code => code === 301 || code === 302 || code === 303 || code === 307 || code === 308; +// Needed for TypeScript. +fetch.default = fetch; + // expose Promise fetch.Promise = global.Promise; export { From d1a3b1ee34720b0d2674a21741264a888863cb3f Mon Sep 17 00:00:00 2001 From: David Frank Date: Sun, 24 Sep 2017 13:46:17 +0800 Subject: [PATCH 127/223] 2.0.0-alpha.9 * changelog update --- CHANGELOG.md | 2 ++ package.json | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1097c818..15ed0d9ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ This is a major release. Check [our upgrade guide](https://github.com/bitinn/nod - Other: use Codecov for code coverage tracking - Other: update package.json script for npm 5 - Other: `encoding` module is now optional (alpha.7) +- Other: expose browser.js through package.json, avoid bundling mishaps (alpha.9) +- Other: allow TypeScript to `import` node-fetch by exposing default (alpha.9) ### HTTP requests diff --git a/package.json b/package.json index a12989271..7346a5928 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.0.0-alpha.8", + "version": "2.0.0-alpha.9", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", "browser": "./browser.js", From 6192398c46c66275b5859ff908527c90c61411ab Mon Sep 17 00:00:00 2001 From: David Frank Date: Sun, 19 Nov 2017 12:22:28 +0800 Subject: [PATCH 128/223] Travis CI fix (#366) * a quick test to see if we can drop npm install before script * adding node 8 lts --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1bc83a8f2..d16cb5353 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: node_js node_js: - "4" - "6" + - "8" - "node" env: - FORMDATA_VERSION=1.0.0 @@ -11,7 +12,6 @@ env: before_script: - 'if [ "$FORMDATA_VERSION" ]; then npm install form-data@^$FORMDATA_VERSION; fi' - 'if [ "$ENCODING" = "yes" ]; then npm install encoding; fi' -before_install: npm install -g npm script: npm run coverage cache: directories: From 3345b652e4ccf7bd8ff30a0f07315821af7de826 Mon Sep 17 00:00:00 2001 From: David Frank Date: Sun, 19 Nov 2017 12:44:02 +0800 Subject: [PATCH 129/223] fix a small oversight in request class lowercase method name does not trigger type error properly (#362) --- src/request.js | 3 ++- test/test.js | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/request.js b/src/request.js index 5b1d012e9..da94f6262 100644 --- a/src/request.js +++ b/src/request.js @@ -40,6 +40,7 @@ export default class Request { } let method = init.method || input.method || 'GET'; + method = method.toUpperCase(); if ((init.body != null || input instanceof Request && input.body !== null) && (method === 'GET' || method === 'HEAD')) { @@ -58,7 +59,7 @@ export default class Request { }); // fetch spec options - this.method = method.toUpperCase(); + this.method = method; this.redirect = init.redirect || input.redirect || 'follow'; this.headers = new Headers(init.headers || input.headers || {}); diff --git a/test/test.js b/test/test.js index 91e9e076e..14e15aab4 100644 --- a/test/test.js +++ b/test/test.js @@ -1527,6 +1527,10 @@ describe('node-fetch', () => { .to.throw(TypeError); expect(() => new Request('.', { body: 'a', method: 'HEAD' })) .to.throw(TypeError); + expect(() => new Request('.', { body: 'a', method: 'get' })) + .to.throw(TypeError); + expect(() => new Request('.', { body: 'a', method: 'head' })) + .to.throw(TypeError); }); it('should support empty options in Response constructor', function() { From a9c76c19ac518a225bec4f48d8c92cd2eb5729f7 Mon Sep 17 00:00:00 2001 From: David Frank Date: Wed, 22 Nov 2017 17:55:39 +0800 Subject: [PATCH 130/223] adding comment to recommend 2.x branch (#367) --- README.md | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index e796a7369..c532bc6f1 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ node-fetch ========== -[![npm version][npm-image]][npm-url] +[![npm stable version][npm-image]][npm-url] +[![npm next version][npm-next-image]][npm-url] [![build status][travis-image]][travis-url] [![coverage status][codecov-image]][codecov-url] @@ -13,7 +14,7 @@ A light-weight module that brings `window.fetch` to Node.js Instead of implementing `XMLHttpRequest` in Node.js to run browser-specific [Fetch polyfill](https://github.com/github/fetch), why not go from native `http` to `fetch` API directly? Hence `node-fetch`, minimal code for a `window.fetch` compatible API on Node.js runtime. -See Matt Andrews' [isomorphic-fetch](https://github.com/matthew-andrews/isomorphic-fetch) for isomorphic usage (exports `node-fetch` for server-side, `whatwg-fetch` for client-side). +See Matt Andrews' [isomorphic-fetch](https://github.com/matthew-andrews/isomorphic-fetch) or Leonardo Quixada's [cross-fetch](https://github.com/lquixada/cross-fetch) for isomorphic usage (exports `node-fetch` for server-side, `whatwg-fetch` for client-side). ## Features @@ -22,13 +23,13 @@ See Matt Andrews' [isomorphic-fetch](https://github.com/matthew-andrews/isomorph - Make conscious trade-off when following [whatwg fetch spec][whatwg-fetch] and [stream spec](https://streams.spec.whatwg.org/) implementation details, document known difference. - Use native promise, but allow substituting it with [insert your favorite promise library]. - Use native stream for body, on both request and response. -- Decode content encoding (gzip/deflate) properly, and convert string output (such as `res.text()` and `res.json()`) to UTF-8 automatically. -- Useful extensions such as timeout, redirect limit, response size limit, [explicit errors](https://github.com/bitinn/node-fetch/blob/master/ERROR-HANDLING.md) for troubleshooting. +- Decode content encoding (gzip/deflate) properly, convert `res.text()` output to UTF-8 optionally. +- Useful extensions such as timeout, redirect limit, response size limit, [explicit errors][ERROR-HANDLING.md] for troubleshooting. ## Difference from client-side fetch -- See [Known Differences](https://github.com/bitinn/node-fetch/blob/master/LIMITS.md) for details. +- See [Known Differences][LIMITS.md] for details. - If you happen to use a missing feature that `window.fetch` offers, feel free to open an issue. - Pull requests are welcomed too! @@ -47,9 +48,12 @@ Next release (`2.x`), currently in alpha $ npm install node-fetch@next --save ``` +*Note: 2.x is expected to be in alpha for quite a while; due to Fetch Spec itself is still evolving and we try to follow its design. Many have been using 2.x for over 6 months, we consider it to be stable and ready for production.* + + ## Usage -Note that documentation below is up-to-date with `2.x` releases, [see `1.x` readme](https://github.com/bitinn/node-fetch/blob/1.x/README.md), [changelog](https://github.com/bitinn/node-fetch/blob/1.x/CHANGELOG.md) and [2.x upgrade guide](https://github.com/bitinn/node-fetch/blob/master/UPGRADE-GUIDE.md) if you want to find out the difference. +Note that documentation below is up-to-date with `2.x` releases, [see `1.x` readme](https://github.com/bitinn/node-fetch/blob/1.x/README.md), [changelog](https://github.com/bitinn/node-fetch/blob/1.x/CHANGELOG.md) and [2.x upgrade guide][UPGRADE-GUIDE.md] if you want to find out the difference. ```javascript import fetch from 'node-fetch'; @@ -398,13 +402,17 @@ Thanks to [github/fetch](https://github.com/github/fetch) for providing a solid [npm-image]: https://img.shields.io/npm/v/node-fetch.svg?style=flat-square +[npm-next-image]: https://img.shields.io/npm/v/node-fetch/next.svg?style=flat-square [npm-url]: https://www.npmjs.com/package/node-fetch [travis-image]: https://img.shields.io/travis/bitinn/node-fetch.svg?style=flat-square [travis-url]: https://travis-ci.org/bitinn/node-fetch [codecov-image]: https://img.shields.io/codecov/c/github/bitinn/node-fetch.svg?style=flat-square [codecov-url]: https://codecov.io/gh/bitinn/node-fetch + [ERROR-HANDLING.md]: https://github.com/bitinn/node-fetch/blob/master/ERROR-HANDLING.md +[LIMITS.md]: https://github.com/bitinn/node-fetch/blob/master/LIMITS.md +[UPGRADE-GUIDE.md]: https://github.com/bitinn/node-fetch/blob/master/UPGRADE-GUIDE.md + [whatwg-fetch]: https://fetch.spec.whatwg.org/ [response-init]: https://fetch.spec.whatwg.org/#responseinit [node-readable]: https://nodejs.org/api/stream.html#stream_readable_streams -[mdn-headers]: https://developer.mozilla.org/en-US/docs/Web/API/Headers From 19b115f9dc85b1fc6b0ea1698f67c9ab4ca2b13f Mon Sep 17 00:00:00 2001 From: Moritz Date: Sat, 27 Jan 2018 20:20:05 +0100 Subject: [PATCH 131/223] Add error event hander for the body stream even if the body isn't accessed (#379) Fixes #378 --- src/body.js | 12 ++++++++++++ test/test.js | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/src/body.js b/src/body.js index 999380858..b944d0354 100644 --- a/src/body.js +++ b/src/body.js @@ -12,6 +12,7 @@ const Stream = require('stream'); const { PassThrough } = require('stream'); const DISTURBED = Symbol('disturbed'); +const ERROR = Symbol('error'); let convert; try { convert = require('encoding').convert; } catch(e) {} @@ -49,8 +50,15 @@ export default function Body(body, { } this.body = body; this[DISTURBED] = false; + this[ERROR] = null; this.size = size; this.timeout = timeout; + + if (this.body instanceof Stream) { + this.body.on('error', err => { + this[ERROR] = new FetchError(`Invalid response body while trying to fetch ${this.url}: ${err.message}`, 'system', err); + }); + } } Body.prototype = { @@ -153,6 +161,10 @@ function consumeBody(body) { this[DISTURBED] = true; + if (this[ERROR]) { + return Body.Promise.reject(this[ERROR]); + } + // body is null if (this.body === null) { return Body.Promise.resolve(Buffer.alloc(0)); diff --git a/test/test.js b/test/test.js index 14e15aab4..4df90def6 100644 --- a/test/test.js +++ b/test/test.js @@ -587,6 +587,40 @@ describe('node-fetch', () => { }); }); + it('should handle errors on the body stream even if it is not used', function(done) { + url = `${base}invalid-content-encoding`; + fetch(url) + .then(res => { + expect(res.status).to.equal(200); + }) + .catch(() => {}) + .then(() => { + // Wait a few ms to see if a uncaught error occurs + setTimeout(() => { + done(); + }, 50); + }); + }); + + it('should collect handled errors on the body stream to reject if the body is used later', function() { + + function delay(value) { + return new Promise((resolve) => { + setTimeout(() => { + resolve(value) + }, 100); + }); + } + + url = `${base}invalid-content-encoding`; + return fetch(url).then(delay).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain'); + return expect(res.text()).to.eventually.be.rejected + .and.be.an.instanceOf(FetchError) + .and.have.property('code', 'Z_DATA_ERROR'); + }); + }); + it('should allow disabling auto decompression', function() { url = `${base}gzip`; opts = { From 5774bf4229cef395288803687639feb267a662cf Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sat, 27 Jan 2018 11:39:47 -0800 Subject: [PATCH 132/223] Update dependencies --- build/rollup-plugin.js | 2 +- package.json | 21 +++++++++++---------- rollup.config.js | 11 +++++------ 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/build/rollup-plugin.js b/build/rollup-plugin.js index 411d677e9..7da31860b 100644 --- a/build/rollup-plugin.js +++ b/build/rollup-plugin.js @@ -4,7 +4,7 @@ export default function tweakDefault() { var lines = source.split('\n'); for (var i = 0; i < lines.length; i++) { var line = lines[i]; - var matches = /^exports\['default'] = (.*);$/.exec(line); + var matches = /^exports(?:\['default']|\.default) = (.*);$/.exec(line); if (matches) { lines[i] = 'module.exports = exports = ' + matches[1] + ';'; break; diff --git a/package.json b/package.json index 7346a5928..8ae128c6b 100644 --- a/package.json +++ b/package.json @@ -36,24 +36,25 @@ }, "homepage": "https://github.com/bitinn/node-fetch", "devDependencies": { - "babel-plugin-istanbul": "^4.0.0", - "babel-preset-env": "^1.1.10", + "babel-core": "^6.26.0", + "babel-plugin-istanbul": "^4.1.5", + "babel-preset-env": "^1.6.1", "babel-register": "^6.16.3", "chai": "^3.5.0", "chai-as-promised": "^7.1.1", "chai-iterator": "^1.1.1", "chai-string": "^1.3.0", - "codecov": "^2.2.0", - "cross-env": "^5.0.1", - "form-data": ">=1.0.0", - "mocha": "^3.1.2", - "nyc": "^11.1.0", + "codecov": "^3.0.0", + "cross-env": "^5.1.3", + "form-data": "^2.3.1", + "mocha": "^5.0.0", + "nyc": "^11.4.1", "parted": "^0.1.1", "promise": "^8.0.1", "resumer": "0.0.0", - "rollup": "^0.45.2", - "rollup-plugin-babel": "^2.6.1", - "url-search-params": "^0.9.0", + "rollup": "^0.55.1", + "rollup-plugin-babel": "^3.0.3", + "url-search-params": "^0.10.0", "whatwg-url": "^5.0.0" }, "dependencies": {} diff --git a/rollup.config.js b/rollup.config.js index c165e4aa4..c66f98052 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -4,18 +4,17 @@ import tweakDefault from './build/rollup-plugin'; process.env.BABEL_ENV = 'rollup'; export default { - entry: 'src/index.js', - exports: 'named', + input: 'src/index.js', + output: [ + { file: 'lib/index.js', format: 'cjs', exports: 'named' }, + { file: 'lib/index.es.js', format: 'es', exports: 'named' } + ], plugins: [ babel({ runtimeHelpers: true }), tweakDefault() ], - targets: [ - { dest: 'lib/index.js', format: 'cjs' }, - { dest: 'lib/index.es.js', format: 'es' } - ], external: function (id) { id = id.split('/').slice(0, id[0] === '@' ? 2 : 1).join('/'); return !!require('./package.json').dependencies[id]; From 8c7c179fef147e8d09c936d8f9c991d90869722f Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sat, 27 Jan 2018 11:49:12 -0800 Subject: [PATCH 133/223] Add some more internal API docs --- src/body.js | 15 +++++++++++++++ src/request.js | 6 ++++++ 2 files changed, 21 insertions(+) diff --git a/src/body.js b/src/body.js index b944d0354..6a56d8079 100644 --- a/src/body.js +++ b/src/body.js @@ -396,6 +396,15 @@ export function extractContentType(instance) { } } +/** + * The Fetch Standard treats this as if "total bytes" is a property on the body. + * For us, we have to explicitly get it with a function. + * + * ref: https://fetch.spec.whatwg.org/#concept-body-total-bytes + * + * @param Body instance Instance of Body + * @return Number? Number of bytes, or null if not possible + */ export function getTotalBytes(instance) { const {body} = instance; @@ -429,6 +438,12 @@ export function getTotalBytes(instance) { } } +/** + * Write a Body to a Node.js WritableStream (e.g. http.Request) object. + * + * @param Body instance Instance of Body + * @return Void + */ export function writeToStream(dest, instance) { const {body} = instance; diff --git a/src/request.js b/src/request.js index da94f6262..8d6b226e2 100644 --- a/src/request.js +++ b/src/request.js @@ -112,6 +112,12 @@ Object.defineProperty(Request.prototype, Symbol.toStringTag, { configurable: true }); +/** + * Convert a Request to Node.js http request options. + * + * @param Request A Request instance + * @return Object The options object to be passed to http.request + */ export function getNodeRequestOptions(request) { const parsedURL = request[PARSED_URL]; const headers = new Headers(request.headers); From bc6f0da3acd8a6e2aa0c93043f6157515322c4e2 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sat, 27 Jan 2018 11:50:29 -0800 Subject: [PATCH 134/223] Remove Blob#close per spec change See https://github.com/w3c/FileAPI/pull/68. --- src/blob.js | 11 +---------- test/test.js | 12 ------------ 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/src/blob.js b/src/blob.js index 36d65bff7..336cd0223 100644 --- a/src/blob.js +++ b/src/blob.js @@ -3,7 +3,6 @@ export const BUFFER = Symbol('buffer'); const TYPE = Symbol('type'); -const CLOSED = Symbol('closed'); export default class Blob { constructor() { @@ -14,7 +13,6 @@ export default class Blob { configurable: true }); - this[CLOSED] = false; this[TYPE] = ''; const blobParts = arguments[0]; @@ -51,14 +49,11 @@ export default class Blob { } } get size() { - return this[CLOSED] ? 0 : this[BUFFER].length; + return this[BUFFER].length; } get type() { return this[TYPE]; } - get isClosed() { - return this[CLOSED]; - } slice() { const size = this.size; @@ -88,12 +83,8 @@ export default class Blob { ); const blob = new Blob([], { type: arguments[2] }); blob[BUFFER] = slicedBuffer; - blob[CLOSED] = this[CLOSED]; return blob; } - close() { - this[CLOSED] = true; - } } Object.defineProperty(Blob.prototype, Symbol.toStringTag, { diff --git a/test/test.js b/test/test.js index 4df90def6..32ab8937c 100644 --- a/test/test.js +++ b/test/test.js @@ -1615,14 +1615,8 @@ describe('node-fetch', () => { }); return res.blob().then(function(result) { expect(result).to.be.an.instanceOf(Blob); - expect(result.isClosed).to.be.false; expect(result.size).to.equal(3); expect(result.type).to.equal('text/plain'); - - result.close(); - expect(result.isClosed).to.be.true; - expect(result.size).to.equal(0); - expect(result.type).to.equal('text/plain'); }); }); @@ -1768,14 +1762,8 @@ describe('node-fetch', () => { expect(req.url).to.equal(url); return req.blob().then(function(result) { expect(result).to.be.an.instanceOf(Blob); - expect(result.isClosed).to.be.false; expect(result.size).to.equal(3); expect(result.type).to.equal(''); - - result.close(); - expect(result.isClosed).to.be.true; - expect(result.size).to.equal(0); - expect(result.type).to.equal(''); }); }); From dccef32e8103d1e867e82d05ab1f99e0e7e941e2 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sat, 3 Feb 2018 11:57:27 -0800 Subject: [PATCH 135/223] Refactor tests --- test/test.js | 404 ++++++++++++++++++++++++++------------------------- 1 file changed, 205 insertions(+), 199 deletions(-) diff --git a/test/test.js b/test/test.js index 32ab8937c..d92521fe0 100644 --- a/test/test.js +++ b/test/test.js @@ -1230,6 +1230,141 @@ describe('node-fetch', () => { }); }); + it('should allow deleting header', function() { + url = `${base}cookie`; + return fetch(url).then(res => { + res.headers.delete('set-cookie'); + expect(res.headers.get('set-cookie')).to.be.null; + }); + }); + + it('should send request with connection keep-alive if agent is provided', function() { + url = `${base}inspect`; + opts = { + agent: new http.Agent({ + keepAlive: true + }) + }; + return fetch(url, opts).then(res => { + return res.json(); + }).then(res => { + expect(res.headers['connection']).to.equal('keep-alive'); + }); + }); + + it('should support fetch with Request instance', function() { + url = `${base}hello`; + const req = new Request(url); + return fetch(req).then(res => { + expect(res.url).to.equal(url); + expect(res.ok).to.be.true; + expect(res.status).to.equal(200); + }); + }); + + it('should support fetch with Node.js URL object', function() { + url = `${base}hello`; + const urlObj = parseURL(url); + const req = new Request(urlObj); + return fetch(req).then(res => { + expect(res.url).to.equal(url); + expect(res.ok).to.be.true; + expect(res.status).to.equal(200); + }); + }); + + it('should support fetch with WHATWG URL object', function() { + url = `${base}hello`; + const urlObj = new URL(url); + const req = new Request(urlObj); + return fetch(req).then(res => { + expect(res.url).to.equal(url); + expect(res.ok).to.be.true; + expect(res.status).to.equal(200); + }); + }); + + it('should support blob round-trip', function() { + url = `${base}hello`; + + let length, type; + + return fetch(url).then(res => res.blob()).then(blob => { + url = `${base}inspect`; + length = blob.size; + type = blob.type; + return fetch(url, { + method: 'POST', + body: blob + }); + }).then(res => res.json()).then(({body, headers}) => { + expect(body).to.equal('world'); + expect(headers['content-type']).to.equal(type); + expect(headers['content-length']).to.equal(String(length)); + }); + }); + + it('should support overwrite Request instance', function() { + url = `${base}inspect`; + const req = new Request(url, { + method: 'POST' + , headers: { + a: '1' + } + }); + return fetch(req, { + method: 'GET' + , headers: { + a: '2' + } + }).then(res => { + return res.json(); + }).then(body => { + expect(body.method).to.equal('GET'); + expect(body.headers.a).to.equal('2'); + }); + }); + + it('should support arrayBuffer(), blob(), text(), json() and buffer() method in Body constructor', function() { + const body = new Body('a=1'); + expect(body).to.have.property('arrayBuffer'); + expect(body).to.have.property('blob'); + expect(body).to.have.property('text'); + expect(body).to.have.property('json'); + expect(body).to.have.property('buffer'); + }); + + it('should create custom FetchError', function funcName() { + const systemError = new Error('system'); + systemError.code = 'ESOMEERROR'; + + const err = new FetchError('test message', 'test-error', systemError); + expect(err).to.be.an.instanceof(Error); + expect(err).to.be.an.instanceof(FetchError); + expect(err.name).to.equal('FetchError'); + expect(err.message).to.equal('test message'); + expect(err.type).to.equal('test-error'); + expect(err.code).to.equal('ESOMEERROR'); + expect(err.errno).to.equal('ESOMEERROR'); + expect(err.stack).to.include('funcName') + .and.to.startWith(`${err.name}: ${err.message}`); + }); + + it('should support https request', function() { + this.timeout(5000); + url = 'https://github.com/'; + opts = { + method: 'HEAD' + }; + return fetch(url, opts).then(res => { + expect(res.status).to.equal(200); + expect(res.ok).to.be.true; + }); + }); + +}); + +describe('Headers', function () { it('should allow iterating through all headers with forEach', function() { const headers = new Headers([ ['b', '2'], @@ -1311,14 +1446,6 @@ describe('node-fetch', () => { .and.to.iterate.over(['1', '2, 3', '4']); }); - it('should allow deleting header', function() { - url = `${base}cookie`; - return fetch(url).then(res => { - res.headers.delete('set-cookie'); - expect(res.headers.get('set-cookie')).to.be.null; - }); - }); - it('should reject illegal header', function() { const headers = new Headers(); expect(() => new Headers({ 'He y': 'ok' })).to.throw(TypeError); @@ -1334,20 +1461,6 @@ describe('node-fetch', () => { new Headers({ 'He-y': 'o k' }); }); - it('should send request with connection keep-alive if agent is provided', function() { - url = `${base}inspect`; - opts = { - agent: new http.Agent({ - keepAlive: true - }) - }; - return fetch(url, opts).then(res => { - return res.json(); - }).then(res => { - expect(res.headers['connection']).to.equal('keep-alive'); - }); - }); - it('should ignore unsupported attributes while reading headers', function() { const FakeHeader = function () {}; // prototypes are currently ignored @@ -1453,121 +1566,10 @@ describe('node-fetch', () => { expect(() => new Headers('b2')).to.throw(TypeError); expect(() => new Headers({ [Symbol.iterator]: 42 })).to.throw(TypeError); }); +}); - it('should support fetch with Request instance', function() { - url = `${base}hello`; - const req = new Request(url); - return fetch(req).then(res => { - expect(res.url).to.equal(url); - expect(res.ok).to.be.true; - expect(res.status).to.equal(200); - }); - }); - - it('should support fetch with Node.js URL object', function() { - url = `${base}hello`; - const urlObj = parseURL(url); - const req = new Request(urlObj); - return fetch(req).then(res => { - expect(res.url).to.equal(url); - expect(res.ok).to.be.true; - expect(res.status).to.equal(200); - }); - }); - - it('should support fetch with WHATWG URL object', function() { - url = `${base}hello`; - const urlObj = new URL(url); - const req = new Request(urlObj); - return fetch(req).then(res => { - expect(res.url).to.equal(url); - expect(res.ok).to.be.true; - expect(res.status).to.equal(200); - }); - }); - - it('should support blob round-trip', function() { - url = `${base}hello`; - - let length, type; - - return fetch(url).then(res => res.blob()).then(blob => { - url = `${base}inspect`; - length = blob.size; - type = blob.type; - return fetch(url, { - method: 'POST', - body: blob - }); - }).then(res => res.json()).then(({body, headers}) => { - expect(body).to.equal('world'); - expect(headers['content-type']).to.equal(type); - expect(headers['content-length']).to.equal(String(length)); - }); - }); - - it('should support wrapping Request instance', function() { - url = `${base}hello`; - - const form = new FormData(); - form.append('a', '1'); - - const r1 = new Request(url, { - method: 'POST' - , follow: 1 - , body: form - }); - const r2 = new Request(r1, { - follow: 2 - }); - - expect(r2.url).to.equal(url); - expect(r2.method).to.equal('POST'); - // note that we didn't clone the body - expect(r2.body).to.equal(form); - expect(r1.follow).to.equal(1); - expect(r2.follow).to.equal(2); - expect(r1.counter).to.equal(0); - expect(r2.counter).to.equal(0); - }); - - it('should support overwrite Request instance', function() { - url = `${base}inspect`; - const req = new Request(url, { - method: 'POST' - , headers: { - a: '1' - } - }); - return fetch(req, { - method: 'GET' - , headers: { - a: '2' - } - }).then(res => { - return res.json(); - }).then(body => { - expect(body.method).to.equal('GET'); - expect(body.headers.a).to.equal('2'); - }); - }); - - it('should throw error with GET/HEAD requests with body', function() { - expect(() => new Request('.', { body: '' })) - .to.throw(TypeError); - expect(() => new Request('.', { body: 'a' })) - .to.throw(TypeError); - expect(() => new Request('.', { body: '', method: 'HEAD' })) - .to.throw(TypeError); - expect(() => new Request('.', { body: 'a', method: 'HEAD' })) - .to.throw(TypeError); - expect(() => new Request('.', { body: 'a', method: 'get' })) - .to.throw(TypeError); - expect(() => new Request('.', { body: 'a', method: 'head' })) - .to.throw(TypeError); - }); - - it('should support empty options in Response constructor', function() { +describe('Response', function () { + it('should support empty options', function() { let body = resumer().queue('a=1').end(); body = body.pipe(new stream.PassThrough()); const res = new Response(body); @@ -1576,7 +1578,7 @@ describe('node-fetch', () => { }); }); - it('should support parsing headers in Response constructor', function() { + it('should support parsing headers', function() { const res = new Response(null, { headers: { a: '1' @@ -1585,28 +1587,28 @@ describe('node-fetch', () => { expect(res.headers.get('a')).to.equal('1'); }); - it('should support text() method in Response constructor', function() { + it('should support text() method', function() { const res = new Response('a=1'); return res.text().then(result => { expect(result).to.equal('a=1'); }); }); - it('should support json() method in Response constructor', function() { + it('should support json() method', function() { const res = new Response('{"a":1}'); return res.json().then(result => { expect(result.a).to.equal(1); }); }); - it('should support buffer() method in Response constructor', function() { + it('should support buffer() method', function() { const res = new Response('a=1'); return res.buffer().then(result => { expect(result.toString()).to.equal('a=1'); }); }); - it('should support blob() method in Response constructor', function() { + it('should support blob() method', function() { const res = new Response('a=1', { method: 'POST', headers: { @@ -1620,7 +1622,7 @@ describe('node-fetch', () => { }); }); - it('should support clone() method in Response constructor', function() { + it('should support clone() method', function() { let body = resumer().queue('a=1').end(); body = body.pipe(new stream.PassThrough()); const res = new Response(body, { @@ -1644,7 +1646,7 @@ describe('node-fetch', () => { }); }); - it('should support stream as body in Response constructor', function() { + it('should support stream as body', function() { let body = resumer().queue('a=1').end(); body = body.pipe(new stream.PassThrough()); const res = new Response(body); @@ -1653,21 +1655,21 @@ describe('node-fetch', () => { }); }); - it('should support string as body in Response constructor', function() { + it('should support string as body', function() { const res = new Response('a=1'); return res.text().then(result => { expect(result).to.equal('a=1'); }); }); - it('should support buffer as body in Response constructor', function() { + it('should support buffer as body', function() { const res = new Response(Buffer.from('a=1')); return res.text().then(result => { expect(result).to.equal('a=1'); }); }); - it('should support blob as body in Response constructor', function() { + it('should support blob as body', function() { const res = new Response(new Blob(['a=1'])); return res.text().then(result => { expect(result).to.equal('a=1'); @@ -1677,22 +1679,64 @@ describe('node-fetch', () => { it('should default to null as body', function() { const res = new Response(); expect(res.body).to.equal(null); - const req = new Request('.'); - expect(req.body).to.equal(null); - const cb = result => expect(result).to.equal(''); - return Promise.all([ - res.text().then(cb), - req.text().then(cb) - ]); + return res.text().then(result => expect(result).to.equal('')); }); it('should default to 200 as status code', function() { const res = new Response(null); expect(res.status).to.equal(200); }); +}); + +describe('Request', function () { + it('should support wrapping Request instance', function() { + url = `${base}hello`; + + const form = new FormData(); + form.append('a', '1'); - it('should support parsing headers in Request constructor', function() { + const r1 = new Request(url, { + method: 'POST' + , follow: 1 + , body: form + }); + const r2 = new Request(r1, { + follow: 2 + }); + + expect(r2.url).to.equal(url); + expect(r2.method).to.equal('POST'); + // note that we didn't clone the body + expect(r2.body).to.equal(form); + expect(r1.follow).to.equal(1); + expect(r2.follow).to.equal(2); + expect(r1.counter).to.equal(0); + expect(r2.counter).to.equal(0); + }); + + it('should throw error with GET/HEAD requests with body', function() { + expect(() => new Request('.', { body: '' })) + .to.throw(TypeError); + expect(() => new Request('.', { body: 'a' })) + .to.throw(TypeError); + expect(() => new Request('.', { body: '', method: 'HEAD' })) + .to.throw(TypeError); + expect(() => new Request('.', { body: 'a', method: 'HEAD' })) + .to.throw(TypeError); + expect(() => new Request('.', { body: 'a', method: 'get' })) + .to.throw(TypeError); + expect(() => new Request('.', { body: 'a', method: 'head' })) + .to.throw(TypeError); + }); + + it('should default to null as body', function() { + const req = new Request('.'); + expect(req.body).to.equal(null); + return req.text().then(result => expect(result).to.equal('')); + }); + + it('should support parsing headers', function() { url = base; const req = new Request(url, { headers: { @@ -1703,7 +1747,7 @@ describe('node-fetch', () => { expect(req.headers.get('a')).to.equal('1'); }); - it('should support arrayBuffer() method in Request constructor', function() { + it('should support arrayBuffer() method', function() { url = base; var req = new Request(url, { method: 'POST', @@ -1717,7 +1761,7 @@ describe('node-fetch', () => { }); }); - it('should support text() method in Request constructor', function() { + it('should support text() method', function() { url = base; const req = new Request(url, { method: 'POST', @@ -1729,7 +1773,7 @@ describe('node-fetch', () => { }); }); - it('should support json() method in Request constructor', function() { + it('should support json() method', function() { url = base; const req = new Request(url, { method: 'POST', @@ -1741,7 +1785,7 @@ describe('node-fetch', () => { }); }); - it('should support buffer() method in Request constructor', function() { + it('should support buffer() method', function() { url = base; const req = new Request(url, { method: 'POST', @@ -1753,7 +1797,7 @@ describe('node-fetch', () => { }); }); - it('should support blob() method in Request constructor', function() { + it('should support blob() method', function() { url = base; var req = new Request(url, { method: 'POST', @@ -1767,13 +1811,13 @@ describe('node-fetch', () => { }); }); - it('should support arbitrary url in Request constructor', function() { + it('should support arbitrary url', function() { url = 'anything'; const req = new Request(url); expect(req.url).to.equal('anything'); }); - it('should support clone() method in Request constructor', function() { + it('should support clone() method', function() { url = base; let body = resumer().queue('a=1').end(); body = body.pipe(new stream.PassThrough()); @@ -1806,44 +1850,6 @@ describe('node-fetch', () => { expect(results[1]).to.equal('a=1'); }); }); - - it('should support arrayBuffer(), blob(), text(), json() and buffer() method in Body constructor', function() { - const body = new Body('a=1'); - expect(body).to.have.property('arrayBuffer'); - expect(body).to.have.property('blob'); - expect(body).to.have.property('text'); - expect(body).to.have.property('json'); - expect(body).to.have.property('buffer'); - }); - - it('should create custom FetchError', function funcName() { - const systemError = new Error('system'); - systemError.code = 'ESOMEERROR'; - - const err = new FetchError('test message', 'test-error', systemError); - expect(err).to.be.an.instanceof(Error); - expect(err).to.be.an.instanceof(FetchError); - expect(err.name).to.equal('FetchError'); - expect(err.message).to.equal('test message'); - expect(err.type).to.equal('test-error'); - expect(err.code).to.equal('ESOMEERROR'); - expect(err.errno).to.equal('ESOMEERROR'); - expect(err.stack).to.include('funcName') - .and.to.startWith(`${err.name}: ${err.message}`); - }); - - it('should support https request', function() { - this.timeout(5000); - url = 'https://github.com/'; - opts = { - method: 'HEAD' - }; - return fetch(url, opts).then(res => { - expect(res.status).to.equal(200); - expect(res.ok).to.be.true; - }); - }); - }); function streamToPromise(stream, dataHandler) { From b1cd2dd438a17e4e4412365bc33ab4d542e34930 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sat, 27 Jan 2018 12:28:56 -0800 Subject: [PATCH 136/223] Better compliance with Web IDL - Make read-only attributes actually read-only - Set @@toStringTag on the prototype only - Make prototype methods/getters enumerable Based on #354. Co-authored-by: Benjamin Seber --- src/blob.js | 15 ++++----- src/body.js | 89 +++++++++++++++++++++++++++++-------------------- src/headers.js | 21 +++++++----- src/index.js | 19 +++++++---- src/request.js | 73 ++++++++++++++++++++++++++++------------ src/response.js | 47 +++++++++++++++++++------- test/test.js | 58 ++++++++++++++++++++++++++++++++ 7 files changed, 230 insertions(+), 92 deletions(-) diff --git a/src/blob.js b/src/blob.js index 336cd0223..b5fa64565 100644 --- a/src/blob.js +++ b/src/blob.js @@ -6,13 +6,6 @@ const TYPE = Symbol('type'); export default class Blob { constructor() { - Object.defineProperty(this, Symbol.toStringTag, { - value: 'Blob', - writable: false, - enumerable: false, - configurable: true - }); - this[TYPE] = ''; const blobParts = arguments[0]; @@ -87,8 +80,14 @@ export default class Blob { } } +Object.defineProperties(Blob.prototype, { + size: { enumerable: true }, + type: { enumerable: true }, + slice: { enumerable: true } +}); + Object.defineProperty(Blob.prototype, Symbol.toStringTag, { - value: 'BlobPrototype', + value: 'Blob', writable: false, enumerable: false, configurable: true diff --git a/src/body.js b/src/body.js index 6a56d8079..5bb045e42 100644 --- a/src/body.js +++ b/src/body.js @@ -11,16 +11,15 @@ import FetchError from './fetch-error.js'; const Stream = require('stream'); const { PassThrough } = require('stream'); -const DISTURBED = Symbol('disturbed'); -const ERROR = Symbol('error'); - let convert; try { convert = require('encoding').convert; } catch(e) {} +const INTERNALS = Symbol('Body internals'); + /** - * Body class + * Body mixin * - * Cannot use ES6 class because Body must be called with .call(). + * Ref: https://fetch.spec.whatwg.org/#body * * @param Stream body Readable stream * @param Object opts Response options @@ -48,22 +47,28 @@ export default function Body(body, { // coerce to string body = String(body); } - this.body = body; - this[DISTURBED] = false; - this[ERROR] = null; + this[INTERNALS] = { + body, + disturbed: false, + error: null + }; this.size = size; this.timeout = timeout; - if (this.body instanceof Stream) { - this.body.on('error', err => { - this[ERROR] = new FetchError(`Invalid response body while trying to fetch ${this.url}: ${err.message}`, 'system', err); + if (body instanceof Stream) { + body.on('error', err => { + this[INTERNALS].error = new FetchError(`Invalid response body while trying to fetch ${this.url}: ${err.message}`, 'system', err); }); } } Body.prototype = { + get body() { + return this[INTERNALS].body; + }, + get bodyUsed() { - return this[DISTURBED]; + return this[INTERNALS].disturbed; }, /** @@ -139,6 +144,16 @@ Body.prototype = { }; +// In browsers, all properties are enumerable. +Object.defineProperties(Body.prototype, { + body: { enumerable: true }, + bodyUsed: { enumerable: true }, + arrayBuffer: { enumerable: true }, + blob: { enumerable: true }, + json: { enumerable: true }, + text: { enumerable: true } +}); + Body.mixIn = function (proto) { for (const name of Object.getOwnPropertyNames(Body.prototype)) { // istanbul ignore else: future proof @@ -150,19 +165,21 @@ Body.mixIn = function (proto) { }; /** - * Decode buffers into utf-8 string + * Consume and convert an entire Body to a Buffer. + * + * Ref: https://fetch.spec.whatwg.org/#concept-body-consume-body * * @return Promise */ -function consumeBody(body) { - if (this[DISTURBED]) { - return Body.Promise.reject(new Error(`body used already for: ${this.url}`)); +function consumeBody() { + if (this[INTERNALS].disturbed) { + return Body.Promise.reject(new TypeError(`body used already for: ${this.url}`)); } - this[DISTURBED] = true; + this[INTERNALS].disturbed = true; - if (this[ERROR]) { - return Body.Promise.reject(this[ERROR]); + if (this[INTERNALS].error) { + return Body.Promise.reject(this[INTERNALS].error); } // body is null @@ -309,21 +326,21 @@ function convertBody(buffer, headers) { * @return String */ function isURLSearchParams(obj) { - // Duck-typing as a necessary condition. - if (typeof obj !== 'object' || - typeof obj.append !== 'function' || - typeof obj.delete !== 'function' || - typeof obj.get !== 'function' || - typeof obj.getAll !== 'function' || - typeof obj.has !== 'function' || - typeof obj.set !== 'function') { - return false; - } - - // Brand-checking and more duck-typing as optional condition. - return obj.constructor.name === 'URLSearchParams' || - Object.prototype.toString.call(obj) === '[object URLSearchParams]' || - typeof obj.sort === 'function'; + // Duck-typing as a necessary condition. + if (typeof obj !== 'object' || + typeof obj.append !== 'function' || + typeof obj.delete !== 'function' || + typeof obj.get !== 'function' || + typeof obj.getAll !== 'function' || + typeof obj.has !== 'function' || + typeof obj.set !== 'function') { + return false; + } + + // Brand-checking and more duck-typing as optional condition. + return obj.constructor.name === 'URLSearchParams' || + Object.prototype.toString.call(obj) === '[object URLSearchParams]' || + typeof obj.sort === 'function'; } /** @@ -350,7 +367,7 @@ export function clone(instance) { body.pipe(p1); body.pipe(p2); // set instance body to teed body and return the other teed body - instance.body = p1; + instance[INTERNALS].body = p1; body = p2; } @@ -362,7 +379,7 @@ export function clone(instance) { * specified in the specification: * https://fetch.spec.whatwg.org/#concept-bodyinit-extract * - * This function assumes that instance.body is present and non-null. + * This function assumes that instance.body is present. * * @param Mixed instance Response or Request instance */ diff --git a/src/headers.js b/src/headers.js index ee4003f15..9b9ba09f1 100644 --- a/src/headers.js +++ b/src/headers.js @@ -84,13 +84,6 @@ export default class Headers { } else { throw new TypeError('Provided initializer must be an object'); } - - Object.defineProperty(this, Symbol.toStringTag, { - value: 'Headers', - writable: false, - enumerable: false, - configurable: true - }); } /** @@ -214,12 +207,24 @@ export default class Headers { Headers.prototype.entries = Headers.prototype[Symbol.iterator]; Object.defineProperty(Headers.prototype, Symbol.toStringTag, { - value: 'HeadersPrototype', + value: 'Headers', writable: false, enumerable: false, configurable: true }); +Object.defineProperties(Headers.prototype, { + get: { enumerable: true }, + forEach: { enumerable: true }, + set: { enumerable: true }, + append: { enumerable: true }, + has: { enumerable: true }, + delete: { enumerable: true }, + keys: { enumerable: true }, + values: { enumerable: true }, + entries: { enumerable: true } +}); + function getHeaderPairs(headers, kind) { const keys = Object.keys(headers[MAP]).sort(); return keys.map( diff --git a/src/index.js b/src/index.js index e6ac30b46..0c9f7b58e 100644 --- a/src/index.js +++ b/src/index.js @@ -84,18 +84,25 @@ export default function fetch(url, opts) { return; } + // Create a new Request object. + const requestOpts = { + headers: new Headers(request.headers), + follow: request.follow, + counter: request.counter + 1, + agent: request.agent, + compress: request.compress, + method: request.method + }; + // per fetch spec, for POST request with 301/302 response, or any request with 303 response, use GET when following redirect if (res.statusCode === 303 || ((res.statusCode === 301 || res.statusCode === 302) && request.method === 'POST')) { - request.method = 'GET'; - request.body = null; - request.headers.delete('content-length'); + requestOpts.method = 'GET'; + requestOpts.headers.delete('content-length'); } - request.counter++; - - resolve(fetch(resolve_url(request.url, res.headers.location), request)); + resolve(fetch(new Request(resolve_url(request.url, res.headers.location), requestOpts))); return; } diff --git a/src/request.js b/src/request.js index 8d6b226e2..191de3dfd 100644 --- a/src/request.js +++ b/src/request.js @@ -10,7 +10,20 @@ import Body, { clone, extractContentType, getTotalBytes } from './body'; const { format: format_url, parse: parse_url } = require('url'); -const PARSED_URL = Symbol('url'); +const INTERNALS = Symbol('Request internals'); + +/** + * Check if a value is an instance of Request. + * + * @param Mixed input + * @return Boolean + */ +function isRequest(input) { + return ( + typeof input === 'object' && + typeof input[INTERNALS] === 'object' + ); +} /** * Request class @@ -24,7 +37,7 @@ export default class Request { let parsedURL; // normalize input - if (!(input instanceof Request)) { + if (!isRequest(input)) { if (input && input.href) { // in order to support Node.js' Url objects; though WHATWG's URL objects // will fall into this branch also (since their `toString()` will return @@ -42,14 +55,14 @@ export default class Request { let method = init.method || input.method || 'GET'; method = method.toUpperCase(); - if ((init.body != null || input instanceof Request && input.body !== null) && + if ((init.body != null || isRequest(input) && input.body !== null) && (method === 'GET' || method === 'HEAD')) { throw new TypeError('Request with GET/HEAD method cannot have body'); } let inputBody = init.body != null ? init.body : - input instanceof Request && input.body !== null ? + isRequest(input) && input.body !== null ? clone(input) : null; @@ -58,19 +71,23 @@ export default class Request { size: init.size || input.size || 0 }); - // fetch spec options - this.method = method; - this.redirect = init.redirect || input.redirect || 'follow'; - this.headers = new Headers(init.headers || input.headers || {}); + const headers = new Headers(init.headers || input.headers || {}); if (init.body != null) { const contentType = extractContentType(this); - if (contentType !== null && !this.headers.has('Content-Type')) { - this.headers.append('Content-Type', contentType); + if (contentType !== null && !headers.has('Content-Type')) { + headers.append('Content-Type', contentType); } } - // server only options + this[INTERNALS] = { + method, + redirect: init.redirect || input.redirect || 'follow', + headers, + parsedURL + }; + + // node-fetch-only options this.follow = init.follow !== undefined ? init.follow : input.follow !== undefined ? input.follow : 20; @@ -79,18 +96,22 @@ export default class Request { input.compress : true; this.counter = init.counter || input.counter || 0; this.agent = init.agent || input.agent; + } - this[PARSED_URL] = parsedURL; - Object.defineProperty(this, Symbol.toStringTag, { - value: 'Request', - writable: false, - enumerable: false, - configurable: true - }); + get method() { + return this[INTERNALS].method; } get url() { - return format_url(this[PARSED_URL]); + return format_url(this[INTERNALS].parsedURL); + } + + get headers() { + return this[INTERNALS].headers; + } + + get redirect() { + return this[INTERNALS].redirect; } /** @@ -106,12 +127,20 @@ export default class Request { Body.mixIn(Request.prototype); Object.defineProperty(Request.prototype, Symbol.toStringTag, { - value: 'RequestPrototype', + value: 'Request', writable: false, enumerable: false, configurable: true }); +Object.defineProperties(Request.prototype, { + method: { enumerable: true }, + url: { enumerable: true }, + headers: { enumerable: true }, + redirect: { enumerable: true }, + clone: { enumerable: true } +}); + /** * Convert a Request to Node.js http request options. * @@ -119,8 +148,8 @@ Object.defineProperty(Request.prototype, Symbol.toStringTag, { * @return Object The options object to be passed to http.request */ export function getNodeRequestOptions(request) { - const parsedURL = request[PARSED_URL]; - const headers = new Headers(request.headers); + const parsedURL = request[INTERNALS].parsedURL; + const headers = new Headers(request[INTERNALS].headers); // fetch step 3 if (!headers.has('Accept')) { diff --git a/src/response.js b/src/response.js index bd5dd2a54..742134274 100644 --- a/src/response.js +++ b/src/response.js @@ -10,6 +10,8 @@ import Body, { clone } from './body'; const { STATUS_CODES } = require('http'); +const INTERNALS = Symbol('Response internals'); + /** * Response class * @@ -21,25 +23,37 @@ export default class Response { constructor(body = null, opts = {}) { Body.call(this, body, opts); - this.url = opts.url; - this.status = opts.status || 200; - this.statusText = opts.statusText || STATUS_CODES[this.status]; + const status = opts.status || 200; - this.headers = new Headers(opts.headers); + this[INTERNALS] = { + url: opts.url, + status, + statusText: opts.statusText || STATUS_CODES[status], + headers: new Headers(opts.headers) + }; + } - Object.defineProperty(this, Symbol.toStringTag, { - value: 'Response', - writable: false, - enumerable: false, - configurable: true - }); + get url() { + return this[INTERNALS].url; + } + + get status() { + return this[INTERNALS].status; } /** * Convenience property representing if the request ended normally */ get ok() { - return this.status >= 200 && this.status < 300; + return this[INTERNALS].status >= 200 && this[INTERNALS].status < 300; + } + + get statusText() { + return this[INTERNALS].statusText; + } + + get headers() { + return this[INTERNALS].headers; } /** @@ -62,8 +76,17 @@ export default class Response { Body.mixIn(Response.prototype); +Object.defineProperties(Response.prototype, { + url: { enumerable: true }, + status: { enumerable: true }, + ok: { enumerable: true }, + statusText: { enumerable: true }, + headers: { enumerable: true }, + clone: { enumerable: true } +}); + Object.defineProperty(Response.prototype, Symbol.toStringTag, { - value: 'ResponsePrototype', + value: 'Response', writable: false, enumerable: false, configurable: true diff --git a/test/test.js b/test/test.js index d92521fe0..cb33d0044 100644 --- a/test/test.js +++ b/test/test.js @@ -1365,6 +1365,21 @@ describe('node-fetch', () => { }); describe('Headers', function () { + it('should have attributes conforming to Web IDL', function () { + const headers = new Headers(); + expect(Object.getOwnPropertyNames(headers)).to.be.empty; + const enumerableProperties = []; + for (const property in headers) { + enumerableProperties.push(property); + } + for (const toCheck of [ + 'append', 'delete', 'entries', 'forEach', 'get', 'has', 'keys', 'set', + 'values' + ]) { + expect(enumerableProperties).to.contain(toCheck); + } + }); + it('should allow iterating through all headers with forEach', function() { const headers = new Headers([ ['b', '2'], @@ -1569,6 +1584,28 @@ describe('Headers', function () { }); describe('Response', function () { + it('should have attributes conforming to Web IDL', function () { + const res = new Response(); + const enumerableProperties = []; + for (const property in res) { + enumerableProperties.push(property); + } + for (const toCheck of [ + 'body', 'bodyUsed', 'arrayBuffer', 'blob', 'json', 'text', + 'url', 'status', 'ok', 'statusText', 'headers', 'clone' + ]) { + expect(enumerableProperties).to.contain(toCheck); + } + for (const toCheck of [ + 'body', 'bodyUsed', 'url', 'status', 'ok', 'statusText', + 'headers' + ]) { + expect(() => { + res[toCheck] = 'abc'; + }).to.throw(); + } + }); + it('should support empty options', function() { let body = resumer().queue('a=1').end(); body = body.pipe(new stream.PassThrough()); @@ -1690,6 +1727,27 @@ describe('Response', function () { }); describe('Request', function () { + it('should have attributes conforming to Web IDL', function () { + const req = new Request('https://github.com/'); + const enumerableProperties = []; + for (const property in req) { + enumerableProperties.push(property); + } + for (const toCheck of [ + 'body', 'bodyUsed', 'arrayBuffer', 'blob', 'json', 'text', + 'method', 'url', 'headers', 'redirect', 'clone' + ]) { + expect(enumerableProperties).to.contain(toCheck); + } + for (const toCheck of [ + 'body', 'bodyUsed', 'method', 'url', 'headers', 'redirect' + ]) { + expect(() => { + req[toCheck] = 'abc'; + }).to.throw(); + } + }); + it('should support wrapping Request instance', function() { url = `${base}hello`; From 44c67b11a36f83d2c9e0cf1671043f2a87c058f9 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sat, 3 Feb 2018 12:33:32 -0800 Subject: [PATCH 137/223] Add docs for v2.x --- CHANGELOG.md | 4 ++-- README.md | 11 +---------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15ed0d9ba..3a0f18316 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ This is a major release. Check [our upgrade guide](https://github.com/bitinn/nod - Major: Node.js 0.10.x and 0.12.x support is dropped - Major: `require('node-fetch/lib/response')` etc. is now unsupported; use `require('node-fetch').Response` or ES6 module imports -- Enhance: start testing on Node.js 4, 6, 8 LTS +- Enhance: start testing on Node.js v4.x, v6.x, v8.x LTS, as well as v9.x stable - Enhance: use Rollup to produce a distributed bundle (less memory overhead and faster startup) - Enhance: make `Object.prototype.toString()` on Headers, Requests, and Responses return correct class strings - Other: rewrite in ES2015 using Babel @@ -26,8 +26,8 @@ This is a major release. Check [our upgrade guide](https://github.com/bitinn/nod ### HTTP requests - Major: overwrite user's `Content-Length` if we can be sure our information is correct (per spec) +- Fix: errors in a response are caught before the body is accessed - Fix: support WHATWG URL objects, created by `whatwg-url` package or `require('url').URL` in Node.js 7+ -- Fix: a regression in older v2 build where `index.es.js` doesn't require https module, causing HTTPS request to fail (alpha.8) ### Response and Request classes diff --git a/README.md b/README.md index c532bc6f1..5f9a091f8 100644 --- a/README.md +++ b/README.md @@ -36,21 +36,12 @@ See Matt Andrews' [isomorphic-fetch](https://github.com/matthew-andrews/isomorph ## Install -Stable release (`1.x`) +Stable release (`2.x`) ```sh $ npm install node-fetch --save ``` -Next release (`2.x`), currently in alpha - -```sh -$ npm install node-fetch@next --save -``` - -*Note: 2.x is expected to be in alpha for quite a while; due to Fetch Spec itself is still evolving and we try to follow its design. Many have been using 2.x for over 6 months, we consider it to be stable and ready for production.* - - ## Usage Note that documentation below is up-to-date with `2.x` releases, [see `1.x` readme](https://github.com/bitinn/node-fetch/blob/1.x/README.md), [changelog](https://github.com/bitinn/node-fetch/blob/1.x/CHANGELOG.md) and [2.x upgrade guide][UPGRADE-GUIDE.md] if you want to find out the difference. From c0950b7b9f0720cc9a0915968d5585d87286fba1 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sat, 3 Feb 2018 12:34:24 -0800 Subject: [PATCH 138/223] v2.0.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8ae128c6b..99f1aaa2b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.0.0-alpha.9", + "version": "2.0.0", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", "browser": "./browser.js", From a8f6d79c39dabd2fa8a4938b2eaae5ae8d3e6f1d Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 4 Mar 2018 12:21:40 -0800 Subject: [PATCH 139/223] Simplify header validity check See: nodejs/node@9f55eac346dea4fba6ace9becc1e01d893f56c5c See: nodejs/node@862389b0aacfd41770c3fa6f692aed11c2a4b3ed --- src/common.js | 111 ------------------------------------------------- src/headers.js | 7 ++-- 2 files changed, 4 insertions(+), 114 deletions(-) delete mode 100644 src/common.js diff --git a/src/common.js b/src/common.js deleted file mode 100644 index b0f9c999a..000000000 --- a/src/common.js +++ /dev/null @@ -1,111 +0,0 @@ -/** - * A set of utilities borrowed from Node.js' _http_common.js - */ - -/** - * Verifies that the given val is a valid HTTP token - * per the rules defined in RFC 7230 - * See https://tools.ietf.org/html/rfc7230#section-3.2.6 - * - * Allowed characters in an HTTP token: - * ^_`a-z 94-122 - * A-Z 65-90 - * - 45 - * 0-9 48-57 - * ! 33 - * #$%&' 35-39 - * *+ 42-43 - * . 46 - * | 124 - * ~ 126 - * - * This implementation of checkIsHttpToken() loops over the string instead of - * using a regular expression since the former is up to 180% faster with v8 4.9 - * depending on the string length (the shorter the string, the larger the - * performance difference) - * - * Additionally, checkIsHttpToken() is currently designed to be inlinable by v8, - * so take care when making changes to the implementation so that the source - * code size does not exceed v8's default max_inlined_source_size setting. - **/ -/* istanbul ignore next */ -function isValidTokenChar(ch) { - if (ch >= 94 && ch <= 122) - return true; - if (ch >= 65 && ch <= 90) - return true; - if (ch === 45) - return true; - if (ch >= 48 && ch <= 57) - return true; - if (ch === 34 || ch === 40 || ch === 41 || ch === 44) - return false; - if (ch >= 33 && ch <= 46) - return true; - if (ch === 124 || ch === 126) - return true; - return false; -} -/* istanbul ignore next */ -function checkIsHttpToken(val) { - if (typeof val !== 'string' || val.length === 0) - return false; - if (!isValidTokenChar(val.charCodeAt(0))) - return false; - const len = val.length; - if (len > 1) { - if (!isValidTokenChar(val.charCodeAt(1))) - return false; - if (len > 2) { - if (!isValidTokenChar(val.charCodeAt(2))) - return false; - if (len > 3) { - if (!isValidTokenChar(val.charCodeAt(3))) - return false; - for (var i = 4; i < len; i++) { - if (!isValidTokenChar(val.charCodeAt(i))) - return false; - } - } - } - } - return true; -} -export { checkIsHttpToken }; - -/** - * True if val contains an invalid field-vchar - * field-value = *( field-content / obs-fold ) - * field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ] - * field-vchar = VCHAR / obs-text - * - * checkInvalidHeaderChar() is currently designed to be inlinable by v8, - * so take care when making changes to the implementation so that the source - * code size does not exceed v8's default max_inlined_source_size setting. - **/ -/* istanbul ignore next */ -function checkInvalidHeaderChar(val) { - val += ''; - if (val.length < 1) - return false; - var c = val.charCodeAt(0); - if ((c <= 31 && c !== 9) || c > 255 || c === 127) - return true; - if (val.length < 2) - return false; - c = val.charCodeAt(1); - if ((c <= 31 && c !== 9) || c > 255 || c === 127) - return true; - if (val.length < 3) - return false; - c = val.charCodeAt(2); - if ((c <= 31 && c !== 9) || c > 255 || c === 127) - return true; - for (var i = 3; i < val.length; ++i) { - c = val.charCodeAt(i); - if ((c <= 31 && c !== 9) || c > 255 || c === 127) - return true; - } - return false; -} -export { checkInvalidHeaderChar }; diff --git a/src/headers.js b/src/headers.js index 9b9ba09f1..9d2f4a29b 100644 --- a/src/headers.js +++ b/src/headers.js @@ -5,11 +5,12 @@ * Headers class offers convenient helpers */ -import { checkInvalidHeaderChar, checkIsHttpToken } from './common.js'; +const invalidTokenRegex = /[^\^_`a-zA-Z\-0-9!#$%&'*+.|~]/; +const invalidHeaderCharRegex = /[^\t\x20-\x7e\x80-\xff]/; function sanitizeName(name) { name += ''; - if (!checkIsHttpToken(name)) { + if (invalidTokenRegex.test(name)) { throw new TypeError(`${name} is not a legal HTTP header name`); } return name.toLowerCase(); @@ -17,7 +18,7 @@ function sanitizeName(name) { function sanitizeValue(value) { value += ''; - if (checkInvalidHeaderChar(value)) { + if (invalidHeaderCharRegex.test(value)) { throw new TypeError(`${value} is not a legal HTTP header value`); } return value; From 1592ca1148e657110fd6f64dc8b29a576cd1da1c Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 4 Mar 2018 12:54:56 -0800 Subject: [PATCH 140/223] Use caseless Headers handling This is unfortunately impossible to test, since the Node.js HTTP library lower-cases all incoming headers. However, this matters for outgoing HTTP requests. See the linked issues from the linked Fetch Standard pull request. See: https://github.com/whatwg/fetch/pull/476 --- src/headers.js | 96 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 61 insertions(+), 35 deletions(-) diff --git a/src/headers.js b/src/headers.js index 9d2f4a29b..23b323506 100644 --- a/src/headers.js +++ b/src/headers.js @@ -8,20 +8,36 @@ const invalidTokenRegex = /[^\^_`a-zA-Z\-0-9!#$%&'*+.|~]/; const invalidHeaderCharRegex = /[^\t\x20-\x7e\x80-\xff]/; -function sanitizeName(name) { - name += ''; +function validateName(name) { + name = `${name}`; if (invalidTokenRegex.test(name)) { throw new TypeError(`${name} is not a legal HTTP header name`); } - return name.toLowerCase(); } -function sanitizeValue(value) { - value += ''; +function validateValue(value) { + value = `${value}`; if (invalidHeaderCharRegex.test(value)) { throw new TypeError(`${value} is not a legal HTTP header value`); } - return value; +} + +/** + * Find the key in the map object given a header name. + * + * Returns undefined if not found. + * + * @param String name Header name + * @return String|Undefined + */ +function find(map, name) { + name = name.toLowerCase(); + for (const key in map) { + if (key.toLowerCase() === name) { + return key; + } + } + return undefined; } const MAP = Symbol('map'); @@ -88,18 +104,20 @@ export default class Headers { } /** - * Return first header value given name + * Return combined header value given name * * @param String name Header name * @return Mixed */ get(name) { - const list = this[MAP][sanitizeName(name)]; - if (!list) { + name = `${name}`; + validateName(name); + const key = find(this[MAP], name); + if (key === undefined) { return null; } - return list.join(', '); + return this[MAP][key].join(', '); } /** @@ -110,12 +128,12 @@ export default class Headers { * @return Void */ forEach(callback, thisArg = undefined) { - let pairs = getHeaderPairs(this); + let pairs = getHeaders(this); let i = 0; while (i < pairs.length) { const [name, value] = pairs[i]; callback.call(thisArg, value, name, this); - pairs = getHeaderPairs(this); + pairs = getHeaders(this); i++; } } @@ -128,7 +146,12 @@ export default class Headers { * @return Void */ set(name, value) { - this[MAP][sanitizeName(name)] = [sanitizeValue(value)]; + name = `${name}`; + value = `${value}`; + validateName(name); + validateValue(value); + const key = find(this[MAP], name); + this[MAP][key !== undefined ? key : name] = [value]; } /** @@ -139,12 +162,16 @@ export default class Headers { * @return Void */ append(name, value) { - if (!this.has(name)) { - this.set(name, value); - return; + name = `${name}`; + value = `${value}`; + validateName(name); + validateValue(value); + const key = find(this[MAP], name); + if (key !== undefined) { + this[MAP][key].push(value); + } else { + this[MAP][name] = [value]; } - - this[MAP][sanitizeName(name)].push(sanitizeValue(value)); } /** @@ -154,7 +181,9 @@ export default class Headers { * @return Boolean */ has(name) { - return !!this[MAP][sanitizeName(name)]; + name = `${name}`; + validateName(name); + return find(this[MAP], name) !== undefined; } /** @@ -164,7 +193,12 @@ export default class Headers { * @return Void */ delete(name) { - delete this[MAP][sanitizeName(name)]; + name = `${name}`; + validateName(name); + const key = find(this[MAP], name); + if (key !== undefined) { + delete this[MAP][key]; + } }; /** @@ -226,12 +260,14 @@ Object.defineProperties(Headers.prototype, { entries: { enumerable: true } }); -function getHeaderPairs(headers, kind) { +function getHeaders(headers, kind = 'key+value') { const keys = Object.keys(headers[MAP]).sort(); return keys.map( kind === 'key' ? - k => [k] : - k => [k, headers.get(k)] + k => k.toLowerCase() : + kind === 'value' ? + k => headers[MAP][k].join(', ') : + k => [k.toLowerCase(), headers[MAP][k].join(', ')] ); } @@ -260,7 +296,7 @@ const HeadersIteratorPrototype = Object.setPrototypeOf({ kind, index } = this[INTERNAL]; - const values = getHeaderPairs(target, kind); + const values = getHeaders(target, kind); const len = values.length; if (index >= len) { return { @@ -269,20 +305,10 @@ const HeadersIteratorPrototype = Object.setPrototypeOf({ }; } - const pair = values[index]; this[INTERNAL].index = index + 1; - let result; - if (kind === 'key') { - result = pair[0]; - } else if (kind === 'value') { - result = pair[1]; - } else { - result = pair; - } - return { - value: result, + value: values[index], done: false }; } From feae6d6ec83de4bef57ffa0a3fe42bcd91d1922b Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 4 Mar 2018 13:12:36 -0800 Subject: [PATCH 141/223] Ignore illegal HTTP headers Fixes: #411 --- src/headers.js | 31 +++++++++++++++++++++++++++++++ src/index.js | 17 ++++------------- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/src/headers.js b/src/headers.js index 23b323506..80077e80d 100644 --- a/src/headers.js +++ b/src/headers.js @@ -322,3 +322,34 @@ Object.defineProperty(HeadersIteratorPrototype, Symbol.toStringTag, { enumerable: false, configurable: true }); + +/** + * Create a Headers object from an object of headers, ignoring those that do + * not conform to HTTP grammar productions. + * + * @param Object obj Object of headers + * @return Headers + */ +export function createHeadersLenient(obj) { + const headers = new Headers(); + for (const name of Object.keys(obj)) { + if (invalidTokenRegex.test(name)) { + continue; + } + if (Array.isArray(obj[name])) { + for (const val of obj[name]) { + if (invalidHeaderCharRegex.test(val)) { + continue; + } + if (headers[MAP][name] === undefined) { + headers[MAP][name] = [val]; + } else { + headers[MAP][name].push(val); + } + } + } else if (!invalidHeaderCharRegex.test(obj[name])) { + headers[MAP][name] = [obj[name]]; + } + } + return headers; +} diff --git a/src/index.js b/src/index.js index 0c9f7b58e..08474c2a5 100644 --- a/src/index.js +++ b/src/index.js @@ -7,7 +7,7 @@ import Body, { writeToStream } from './body'; import Response from './response'; -import Headers from './headers'; +import Headers, { createHeadersLenient } from './headers'; import Request, { getNodeRequestOptions } from './request'; import FetchError from './fetch-error'; @@ -106,19 +106,10 @@ export default function fetch(url, opts) { return; } + const headers = createHeadersLenient(res.headers); // normalize location header for manual redirect mode - const headers = new Headers(); - for (const name of Object.keys(res.headers)) { - if (Array.isArray(res.headers[name])) { - for (const val of res.headers[name]) { - headers.append(name, val); - } - } else { - headers.append(name, res.headers[name]); - } - } - if (request.redirect === 'manual' && headers.has('location')) { - headers.set('location', resolve_url(request.url, headers.get('location'))); + if (request.redirect === 'manual' && headers.has('Location')) { + headers.set('Location', resolve_url(request.url, headers.get('Location'))); } // prepare response From fc539951ca6ce70f42f42516ce098840c191c9a0 Mon Sep 17 00:00:00 2001 From: Gregor Martynus Date: Mon, 5 Mar 2018 00:40:39 +0000 Subject: [PATCH 142/223] Support ArrayBuffer as body (#408) --- package.json | 1 + src/body.js | 17 +++++++++++++++++ test/test.js | 16 ++++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/package.json b/package.json index 99f1aaa2b..6ed380e26 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "resumer": "0.0.0", "rollup": "^0.55.1", "rollup-plugin-babel": "^3.0.3", + "string-to-arraybuffer": "^1.0.0", "url-search-params": "^0.10.0", "whatwg-url": "^5.0.0" }, diff --git a/src/body.js b/src/body.js index 5bb045e42..fcd3b4ae8 100644 --- a/src/body.js +++ b/src/body.js @@ -40,6 +40,8 @@ export default function Body(body, { // body is blob } else if (Buffer.isBuffer(body)) { // body is buffer + } else if (Object.prototype.toString.call(body) === '[object ArrayBuffer]') { + // body is array buffer } else if (body instanceof Stream) { // body is stream } else { @@ -202,6 +204,11 @@ function consumeBody() { return Body.Promise.resolve(this.body); } + // body is buffer + if (Object.prototype.toString.call(this.body) === '[object ArrayBuffer]') { + return Body.Promise.resolve(this.body); + } + // istanbul ignore if: should never happen if (!(this.body instanceof Stream)) { return Body.Promise.resolve(Buffer.alloc(0)); @@ -403,6 +410,9 @@ export function extractContentType(instance) { } else if (Buffer.isBuffer(body)) { // body is buffer return null; + } else if (Object.prototype.toString.call(body) === '[object ArrayBuffer]') { + // body is array buffer + return null; } else if (typeof body.getBoundary === 'function') { // detect form data input from form-data module return `multipart/form-data;boundary=${body.getBoundary()}`; @@ -441,6 +451,9 @@ export function getTotalBytes(instance) { } else if (Buffer.isBuffer(body)) { // body is buffer return body.length; + } else if (Object.prototype.toString.call(body) === '[object ArrayBuffer]') { + // body is array buffer + return body.byteLength; } else if (body && typeof body.getLengthSync === 'function') { // detect form data input from form-data module if (body._lengthRetrievers && body._lengthRetrievers.length == 0 || // 1.x @@ -483,6 +496,10 @@ export function writeToStream(dest, instance) { // body is buffer dest.write(body); dest.end() + } else if (Object.prototype.toString.call(body) === '[object ArrayBuffer]') { + // body is array buffer + dest.write(Buffer.from(body)); + dest.end() } else { // body is stream body.pipe(dest); diff --git a/test/test.js b/test/test.js index cb33d0044..ff89a3be0 100644 --- a/test/test.js +++ b/test/test.js @@ -7,6 +7,7 @@ import chaiString from 'chai-string'; import then from 'promise'; import resumer from 'resumer'; import FormData from 'form-data'; +import stringToArrayBuffer from 'string-to-arraybuffer'; import URLSearchParams_Polyfill from 'url-search-params'; import { URL } from 'whatwg-url'; @@ -771,6 +772,21 @@ describe('node-fetch', () => { }); }); + it('should allow POST request with ArrayBuffer body', function() { + url = `${base}inspect`; + opts = { + method: 'POST' + , body: stringToArrayBuffer('Hello, world!\n') + }; + return fetch(url, opts).then(res => res.json()).then(res => { + expect(res.method).to.equal('POST'); + expect(res.body).to.equal('Hello, world!\n'); + expect(res.headers['transfer-encoding']).to.be.undefined; + expect(res.headers['content-type']).to.be.undefined; + expect(res.headers['content-length']).to.equal('14'); + }); + }); + it('should allow POST request with blob body without type', function() { url = `${base}inspect`; opts = { From 35722f19e5cf203d877ebc5f2d44a327aca6dcaf Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 4 Mar 2018 19:29:59 -0800 Subject: [PATCH 143/223] Significant rework of redirection - Handle Location-less redirect like non-redirect response. - Include bodies when redirecting to non-POST 301/302 and all 307/308 response. Co-authored-by: Gregor Martynus --- src/index.js | 102 +++++++++++++++++++++++++++++-------------------- test/server.js | 3 +- test/test.js | 64 +++++++++++++++++++++++++++---- 3 files changed, 118 insertions(+), 51 deletions(-) diff --git a/src/index.js b/src/index.js index 08474c2a5..f764a9b95 100644 --- a/src/index.js +++ b/src/index.js @@ -5,7 +5,7 @@ * a request API compatible with window.fetch */ -import Body, { writeToStream } from './body'; +import Body, { writeToStream, getTotalBytes } from './body'; import Response from './response'; import Headers, { createHeadersLenient } from './headers'; import Request, { getNodeRequestOptions } from './request'; @@ -67,49 +67,67 @@ export default function fetch(url, opts) { req.on('response', res => { clearTimeout(reqTimeout); - // handle redirect - if (fetch.isRedirect(res.statusCode) && request.redirect !== 'manual') { - if (request.redirect === 'error') { - reject(new FetchError(`redirect mode is set to error: ${request.url}`, 'no-redirect')); - return; - } - - if (request.counter >= request.follow) { - reject(new FetchError(`maximum redirect reached at: ${request.url}`, 'max-redirect')); - return; - } - - if (!res.headers.location) { - reject(new FetchError(`redirect location header missing at: ${request.url}`, 'invalid-redirect')); - return; - } + const headers = createHeadersLenient(res.headers); - // Create a new Request object. - const requestOpts = { - headers: new Headers(request.headers), - follow: request.follow, - counter: request.counter + 1, - agent: request.agent, - compress: request.compress, - method: request.method - }; - - // per fetch spec, for POST request with 301/302 response, or any request with 303 response, use GET when following redirect - if (res.statusCode === 303 - || ((res.statusCode === 301 || res.statusCode === 302) && request.method === 'POST')) - { - requestOpts.method = 'GET'; - requestOpts.headers.delete('content-length'); + // HTTP fetch step 5 + if (fetch.isRedirect(res.statusCode)) { + // HTTP fetch step 5.2 + const location = headers.get('Location'); + + // HTTP fetch step 5.3 + const locationURL = location === null ? null : resolve_url(request.url, location); + + // HTTP fetch step 5.5 + switch (request.redirect) { + case 'error': + reject(new FetchError(`redirect mode is set to error: ${request.url}`, 'no-redirect')); + return; + case 'manual': + // node-fetch-specific step: make manual redirect a bit easier to use by setting the Location header value to the resolved URL. + if (locationURL !== null) { + headers.set('Location', locationURL); + } + break; + case 'follow': + // HTTP-redirect fetch step 2 + if (locationURL === null) { + break; + } + + // HTTP-redirect fetch step 5 + if (request.counter >= request.follow) { + reject(new FetchError(`maximum redirect reached at: ${request.url}`, 'max-redirect')); + return; + } + + // HTTP-redirect fetch step 6 (counter increment) + // Create a new Request object. + const requestOpts = { + headers: new Headers(request.headers), + follow: request.follow, + counter: request.counter + 1, + agent: request.agent, + compress: request.compress, + method: request.method, + body: request.body + }; + + // HTTP-redirect fetch step 9 + if (res.statusCode !== 303 && request.body && getTotalBytes(request) === null) { + reject(new FetchError('Cannot follow redirect with body being a readable stream', 'unsupported-redirect')) + } + + // HTTP-redirect fetch step 11 + if (res.statusCode === 303 || ((res.statusCode === 301 || res.statusCode === 302) && request.method === 'POST')) { + requestOpts.method = 'GET'; + requestOpts.body = undefined; + requestOpts.headers.delete('content-length'); + } + + // HTTP-redirect fetch step 15 + resolve(fetch(new Request(locationURL, requestOpts))); + return; } - - resolve(fetch(new Request(resolve_url(request.url, res.headers.location), requestOpts))); - return; - } - - const headers = createHeadersLenient(res.headers); - // normalize location header for manual redirect mode - if (request.redirect === 'manual' && headers.has('Location')) { - headers.set('Location', resolve_url(request.url, headers.get('Location'))); } // prepare response diff --git a/test/server.js b/test/server.js index 6f5b11620..d5594633b 100644 --- a/test/server.js +++ b/test/server.js @@ -250,9 +250,8 @@ export default class TestServer { res.end(); } - if (p === '/error/redirect') { + if (p === '/redirect/no-location') { res.statusCode = 301; - //res.setHeader('Location', '/inspect'); res.end(); } diff --git a/test/test.js b/test/test.js index ff89a3be0..527c6fe3d 100644 --- a/test/test.js +++ b/test/test.js @@ -274,6 +274,22 @@ describe('node-fetch', () => { }); }); + it('should follow PATCH request redirect code 301 with PATCH', function() { + url = `${base}redirect/301`; + opts = { + method: 'PATCH' + , body: 'a=1' + }; + return fetch(url, opts).then(res => { + expect(res.url).to.equal(`${base}inspect`); + expect(res.status).to.equal(200); + return res.json().then(res => { + expect(res.method).to.equal('PATCH'); + expect(res.body).to.equal('a=1'); + }); + }); + }); + it('should follow POST request redirect code 302 with GET', function() { url = `${base}redirect/302`; opts = { @@ -290,6 +306,22 @@ describe('node-fetch', () => { }); }); + it('should follow PATCH request redirect code 302 with PATCH', function() { + url = `${base}redirect/302`; + opts = { + method: 'PATCH' + , body: 'a=1' + }; + return fetch(url, opts).then(res => { + expect(res.url).to.equal(`${base}inspect`); + expect(res.status).to.equal(200); + return res.json().then(res => { + expect(res.method).to.equal('PATCH'); + expect(res.body).to.equal('a=1'); + }); + }); + }); + it('should follow redirect code 303 with GET', function() { url = `${base}redirect/303`; opts = { @@ -306,6 +338,22 @@ describe('node-fetch', () => { }); }); + it('should follow PATCH request redirect code 307 with PATCH', function() { + url = `${base}redirect/307`; + opts = { + method: 'PATCH' + , body: 'a=1' + }; + return fetch(url, opts).then(res => { + expect(res.url).to.equal(`${base}inspect`); + expect(res.status).to.equal(200); + return res.json().then(result => { + expect(result.method).to.equal('PATCH'); + expect(result.body).to.equal('a=1'); + }); + }); + }); + it('should obey maximum redirect, reject case', function() { url = `${base}redirect/chain`; opts = { @@ -384,15 +432,17 @@ describe('node-fetch', () => { }); }); - it('should reject broken redirect', function() { - url = `${base}error/redirect`; - return expect(fetch(url)).to.eventually.be.rejected - .and.be.an.instanceOf(FetchError) - .and.have.property('type', 'invalid-redirect'); + it('should treat broken redirect as ordinary response (follow)', function() { + url = `${base}redirect/no-location`; + return fetch(url, opts).then(res => { + expect(res.url).to.equal(url); + expect(res.status).to.equal(301); + expect(res.headers.get('location')).to.be.null; + }); }); - it('should not reject broken redirect under manual redirect', function() { - url = `${base}error/redirect`; + it('should treat broken redirect as ordinary response (manual)', function() { + url = `${base}redirect/no-location`; opts = { redirect: 'manual' }; From 119138ef641de841ebdefca68deb4dcfd5626f56 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 4 Mar 2018 19:38:57 -0800 Subject: [PATCH 144/223] Update existing algorithm step numbers --- src/index.js | 6 ++++-- src/request.js | 12 +++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/index.js b/src/index.js index f764a9b95..6a8c18bf0 100644 --- a/src/index.js +++ b/src/index.js @@ -3,6 +3,8 @@ * index.js * * a request API compatible with window.fetch + * + * All spec algorithm step numbers are based on https://fetch.spec.whatwg.org/commit-snapshots/ae716822cb3a61843226cd090eefc6589446c1d2/. */ import Body, { writeToStream, getTotalBytes } from './body'; @@ -141,10 +143,10 @@ export default function fetch(url, opts) { , timeout: request.timeout }; - // HTTP-network fetch step 16.1.2 + // HTTP-network fetch step 12.1.1.3 const codings = headers.get('Content-Encoding'); - // HTTP-network fetch step 16.1.3: handle content codings + // HTTP-network fetch step 12.1.1.4: handle content codings // in following scenarios we ignore compression support // 1. compression support is disabled diff --git a/src/request.js b/src/request.js index 191de3dfd..108a6bf2e 100644 --- a/src/request.js +++ b/src/request.js @@ -3,6 +3,8 @@ * request.js * * Request class contains server only options + * + * All spec algorithm step numbers are based on https://fetch.spec.whatwg.org/commit-snapshots/ae716822cb3a61843226cd090eefc6589446c1d2/. */ import Headers from './headers.js'; @@ -151,7 +153,7 @@ export function getNodeRequestOptions(request) { const parsedURL = request[INTERNALS].parsedURL; const headers = new Headers(request[INTERNALS].headers); - // fetch step 3 + // fetch step 1.3 if (!headers.has('Accept')) { headers.set('Accept', '*/*'); } @@ -165,7 +167,7 @@ export function getNodeRequestOptions(request) { throw new TypeError('Only HTTP(S) protocols are supported'); } - // HTTP-network-or-cache fetch steps 5-9 + // HTTP-network-or-cache fetch steps 2.4-2.7 let contentLengthValue = null; if (request.body == null && /^(POST|PUT)$/i.test(request.method)) { contentLengthValue = '0'; @@ -180,12 +182,12 @@ export function getNodeRequestOptions(request) { headers.set('Content-Length', contentLengthValue); } - // HTTP-network-or-cache fetch step 12 + // HTTP-network-or-cache fetch step 2.11 if (!headers.has('User-Agent')) { headers.set('User-Agent', 'node-fetch/1.0 (+https://github.com/bitinn/node-fetch)'); } - // HTTP-network-or-cache fetch step 16 + // HTTP-network-or-cache fetch step 2.15 if (request.compress) { headers.set('Accept-Encoding', 'gzip,deflate'); } @@ -193,7 +195,7 @@ export function getNodeRequestOptions(request) { headers.set('Connection', 'close'); } - // HTTP-network fetch step 4 + // HTTP-network fetch step 4.2 // chunked encoding is handled by Node.js return Object.assign({}, parsedURL, { From 780598ad72d7205aa5616be2215692785270eda3 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 4 Mar 2018 20:29:12 -0800 Subject: [PATCH 145/223] Harmonize style (#413) --- src/body.js | 6 +- src/index.js | 12 +- src/response.js | 11 +- test/test.js | 461 ++++++++++++++++++++++++------------------------ 4 files changed, 244 insertions(+), 246 deletions(-) diff --git a/src/body.js b/src/body.js index fcd3b4ae8..60d100573 100644 --- a/src/body.js +++ b/src/body.js @@ -319,9 +319,9 @@ function convertBody(buffer, headers) { // turn raw buffers into a single utf-8 buffer return convert( - buffer - , 'UTF-8' - , charset + buffer, + 'UTF-8', + charset ).toString(); } diff --git a/src/index.js b/src/index.js index 6a8c18bf0..d395dade2 100644 --- a/src/index.js +++ b/src/index.js @@ -135,12 +135,12 @@ export default function fetch(url, opts) { // prepare response let body = res.pipe(new PassThrough()); const response_options = { - url: request.url - , status: res.statusCode - , statusText: res.statusMessage - , headers: headers - , size: request.size - , timeout: request.timeout + url: request.url, + status: res.statusCode, + statusText: res.statusMessage, + headers: headers, + size: request.size, + timeout: request.timeout }; // HTTP-network fetch step 12.1.1.3 diff --git a/src/response.js b/src/response.js index 742134274..dd0a2ceba 100644 --- a/src/response.js +++ b/src/response.js @@ -62,13 +62,12 @@ export default class Response { * @return Response */ clone() { - return new Response(clone(this), { - url: this.url - , status: this.status - , statusText: this.statusText - , headers: this.headers - , ok: this.ok + url: this.url, + status: this.status, + statusText: this.statusText, + headers: this.headers, + ok: this.ok }); } diff --git a/test/test.js b/test/test.js index 527c6fe3d..ba9cfa50e 100644 --- a/test/test.js +++ b/test/test.js @@ -48,7 +48,6 @@ const supportToString = ({ const local = new TestServer(); const base = `http://${local.hostname}:${local.port}/`; -let url, opts; before(done => { local.start(done); @@ -60,14 +59,14 @@ after(done => { describe('node-fetch', () => { it('should return a promise', function() { - url = `${base}hello`; + const url = `${base}hello`; const p = fetch(url); expect(p).to.be.an.instanceof(fetch.Promise); expect(p).to.have.property('then'); }); it('should allow custom promise', function() { - url = `${base}hello`; + const url = `${base}hello`; const old = fetch.Promise; fetch.Promise = then; expect(fetch(url)).to.be.an.instanceof(then); @@ -76,7 +75,7 @@ describe('node-fetch', () => { }); it('should throw error when no promise implementation are found', function() { - url = `${base}hello`; + const url = `${base}hello`; const old = fetch.Promise; fetch.Promise = undefined; expect(() => { @@ -99,29 +98,29 @@ describe('node-fetch', () => { }); it('should reject with error if url is protocol relative', function() { - url = '//example.com/'; + const url = '//example.com/'; return expect(fetch(url)).to.eventually.be.rejectedWith(TypeError, 'Only absolute URLs are supported'); }); it('should reject with error if url is relative path', function() { - url = '/some/path'; + const url = '/some/path'; return expect(fetch(url)).to.eventually.be.rejectedWith(TypeError, 'Only absolute URLs are supported'); }); it('should reject with error if protocol is unsupported', function() { - url = 'ftp://example.com/'; + const url = 'ftp://example.com/'; return expect(fetch(url)).to.eventually.be.rejectedWith(TypeError, 'Only HTTP(S) protocols are supported'); }); it('should reject with error on network failure', function() { - url = 'http://localhost:50000/'; + const url = 'http://localhost:50000/'; return expect(fetch(url)).to.eventually.be.rejected .and.be.an.instanceOf(FetchError) .and.include({ type: 'system', code: 'ECONNREFUSED', errno: 'ECONNREFUSED' }); }); it('should resolve into response', function() { - url = `${base}hello`; + const url = `${base}hello`; return fetch(url).then(res => { expect(res).to.be.an.instanceof(Response); expect(res.headers).to.be.an.instanceof(Headers); @@ -136,7 +135,7 @@ describe('node-fetch', () => { }); it('should accept plain text response', function() { - url = `${base}plain`; + const url = `${base}plain`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); return res.text().then(result => { @@ -148,7 +147,7 @@ describe('node-fetch', () => { }); it('should accept html response (like plain text)', function() { - url = `${base}html`; + const url = `${base}html`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/html'); return res.text().then(result => { @@ -160,7 +159,7 @@ describe('node-fetch', () => { }); it('should accept json response', function() { - url = `${base}json`; + const url = `${base}json`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('application/json'); return res.json().then(result => { @@ -172,8 +171,8 @@ describe('node-fetch', () => { }); it('should send request with custom headers', function() { - url = `${base}inspect`; - opts = { + const url = `${base}inspect`; + const opts = { headers: { 'x-custom-header': 'abc' } }; return fetch(url, opts).then(res => { @@ -184,8 +183,8 @@ describe('node-fetch', () => { }); it('should accept headers instance', function() { - url = `${base}inspect`; - opts = { + const url = `${base}inspect`; + const opts = { headers: new Headers({ 'x-custom-header': 'abc' }) }; return fetch(url, opts).then(res => { @@ -196,8 +195,8 @@ describe('node-fetch', () => { }); it('should accept custom host header', function() { - url = `${base}inspect`; - opts = { + const url = `${base}inspect`; + const opts = { headers: { host: 'example.com' } @@ -210,7 +209,7 @@ describe('node-fetch', () => { }); it('should follow redirect code 301', function() { - url = `${base}redirect/301`; + const url = `${base}redirect/301`; return fetch(url).then(res => { expect(res.url).to.equal(`${base}inspect`); expect(res.status).to.equal(200); @@ -219,7 +218,7 @@ describe('node-fetch', () => { }); it('should follow redirect code 302', function() { - url = `${base}redirect/302`; + const url = `${base}redirect/302`; return fetch(url).then(res => { expect(res.url).to.equal(`${base}inspect`); expect(res.status).to.equal(200); @@ -227,7 +226,7 @@ describe('node-fetch', () => { }); it('should follow redirect code 303', function() { - url = `${base}redirect/303`; + const url = `${base}redirect/303`; return fetch(url).then(res => { expect(res.url).to.equal(`${base}inspect`); expect(res.status).to.equal(200); @@ -235,7 +234,7 @@ describe('node-fetch', () => { }); it('should follow redirect code 307', function() { - url = `${base}redirect/307`; + const url = `${base}redirect/307`; return fetch(url).then(res => { expect(res.url).to.equal(`${base}inspect`); expect(res.status).to.equal(200); @@ -243,7 +242,7 @@ describe('node-fetch', () => { }); it('should follow redirect code 308', function() { - url = `${base}redirect/308`; + const url = `${base}redirect/308`; return fetch(url).then(res => { expect(res.url).to.equal(`${base}inspect`); expect(res.status).to.equal(200); @@ -251,7 +250,7 @@ describe('node-fetch', () => { }); it('should follow redirect chain', function() { - url = `${base}redirect/chain`; + const url = `${base}redirect/chain`; return fetch(url).then(res => { expect(res.url).to.equal(`${base}inspect`); expect(res.status).to.equal(200); @@ -259,10 +258,10 @@ describe('node-fetch', () => { }); it('should follow POST request redirect code 301 with GET', function() { - url = `${base}redirect/301`; - opts = { - method: 'POST' - , body: 'a=1' + const url = `${base}redirect/301`; + const opts = { + method: 'POST', + body: 'a=1' }; return fetch(url, opts).then(res => { expect(res.url).to.equal(`${base}inspect`); @@ -275,10 +274,10 @@ describe('node-fetch', () => { }); it('should follow PATCH request redirect code 301 with PATCH', function() { - url = `${base}redirect/301`; - opts = { - method: 'PATCH' - , body: 'a=1' + const url = `${base}redirect/301`; + const opts = { + method: 'PATCH', + body: 'a=1' }; return fetch(url, opts).then(res => { expect(res.url).to.equal(`${base}inspect`); @@ -291,10 +290,10 @@ describe('node-fetch', () => { }); it('should follow POST request redirect code 302 with GET', function() { - url = `${base}redirect/302`; - opts = { - method: 'POST' - , body: 'a=1' + const url = `${base}redirect/302`; + const opts = { + method: 'POST', + body: 'a=1' }; return fetch(url, opts).then(res => { expect(res.url).to.equal(`${base}inspect`); @@ -307,10 +306,10 @@ describe('node-fetch', () => { }); it('should follow PATCH request redirect code 302 with PATCH', function() { - url = `${base}redirect/302`; - opts = { - method: 'PATCH' - , body: 'a=1' + const url = `${base}redirect/302`; + const opts = { + method: 'PATCH', + body: 'a=1' }; return fetch(url, opts).then(res => { expect(res.url).to.equal(`${base}inspect`); @@ -323,10 +322,10 @@ describe('node-fetch', () => { }); it('should follow redirect code 303 with GET', function() { - url = `${base}redirect/303`; - opts = { - method: 'PUT' - , body: 'a=1' + const url = `${base}redirect/303`; + const opts = { + method: 'PUT', + body: 'a=1' }; return fetch(url, opts).then(res => { expect(res.url).to.equal(`${base}inspect`); @@ -339,10 +338,10 @@ describe('node-fetch', () => { }); it('should follow PATCH request redirect code 307 with PATCH', function() { - url = `${base}redirect/307`; - opts = { - method: 'PATCH' - , body: 'a=1' + const url = `${base}redirect/307`; + const opts = { + method: 'PATCH', + body: 'a=1' }; return fetch(url, opts).then(res => { expect(res.url).to.equal(`${base}inspect`); @@ -355,8 +354,8 @@ describe('node-fetch', () => { }); it('should obey maximum redirect, reject case', function() { - url = `${base}redirect/chain`; - opts = { + const url = `${base}redirect/chain`; + const opts = { follow: 1 } return expect(fetch(url, opts)).to.eventually.be.rejected @@ -365,8 +364,8 @@ describe('node-fetch', () => { }); it('should obey redirect chain, resolve case', function() { - url = `${base}redirect/chain`; - opts = { + const url = `${base}redirect/chain`; + const opts = { follow: 2 } return fetch(url, opts).then(res => { @@ -376,8 +375,8 @@ describe('node-fetch', () => { }); it('should allow not following redirect', function() { - url = `${base}redirect/301`; - opts = { + const url = `${base}redirect/301`; + const opts = { follow: 0 } return expect(fetch(url, opts)).to.eventually.be.rejected @@ -386,8 +385,8 @@ describe('node-fetch', () => { }); it('should support redirect mode, manual flag', function() { - url = `${base}redirect/301`; - opts = { + const url = `${base}redirect/301`; + const opts = { redirect: 'manual' }; return fetch(url, opts).then(res => { @@ -398,8 +397,8 @@ describe('node-fetch', () => { }); it('should support redirect mode, error flag', function() { - url = `${base}redirect/301`; - opts = { + const url = `${base}redirect/301`; + const opts = { redirect: 'error' }; return expect(fetch(url, opts)).to.eventually.be.rejected @@ -408,8 +407,8 @@ describe('node-fetch', () => { }); it('should support redirect mode, manual flag when there is no redirect', function() { - url = `${base}hello`; - opts = { + const url = `${base}hello`; + const opts = { redirect: 'manual' }; return fetch(url, opts).then(res => { @@ -420,8 +419,8 @@ describe('node-fetch', () => { }); it('should follow redirect code 301 and keep existing headers', function() { - url = `${base}redirect/301`; - opts = { + const url = `${base}redirect/301`; + const opts = { headers: new Headers({ 'x-custom-header': 'abc' }) }; return fetch(url, opts).then(res => { @@ -433,8 +432,8 @@ describe('node-fetch', () => { }); it('should treat broken redirect as ordinary response (follow)', function() { - url = `${base}redirect/no-location`; - return fetch(url, opts).then(res => { + const url = `${base}redirect/no-location`; + return fetch(url).then(res => { expect(res.url).to.equal(url); expect(res.status).to.equal(301); expect(res.headers.get('location')).to.be.null; @@ -442,8 +441,8 @@ describe('node-fetch', () => { }); it('should treat broken redirect as ordinary response (manual)', function() { - url = `${base}redirect/no-location`; - opts = { + const url = `${base}redirect/no-location`; + const opts = { redirect: 'manual' }; return fetch(url, opts).then(res => { @@ -454,7 +453,7 @@ describe('node-fetch', () => { }); it('should handle client-error response', function() { - url = `${base}error/400`; + const url = `${base}error/400`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); expect(res.status).to.equal(400); @@ -469,7 +468,7 @@ describe('node-fetch', () => { }); it('should handle server-error response', function() { - url = `${base}error/500`; + const url = `${base}error/500`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); expect(res.status).to.equal(500); @@ -484,21 +483,21 @@ describe('node-fetch', () => { }); it('should handle network-error response', function() { - url = `${base}error/reset`; + const url = `${base}error/reset`; return expect(fetch(url)).to.eventually.be.rejected .and.be.an.instanceOf(FetchError) .and.have.property('code', 'ECONNRESET'); }); it('should handle DNS-error response', function() { - url = 'http://domain.invalid'; + const url = 'http://domain.invalid'; return expect(fetch(url)).to.eventually.be.rejected .and.be.an.instanceOf(FetchError) .and.have.property('code', 'ENOTFOUND'); }); it('should reject invalid json response', function() { - url = `${base}error/json`; + const url = `${base}error/json`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('application/json'); return expect(res.json()).to.eventually.be.rejected @@ -508,7 +507,7 @@ describe('node-fetch', () => { }); it('should handle no content response', function() { - url = `${base}no-content`; + const url = `${base}no-content`; return fetch(url).then(res => { expect(res.status).to.equal(204); expect(res.statusText).to.equal('No Content'); @@ -521,7 +520,7 @@ describe('node-fetch', () => { }); it('should reject when trying to parse no content response as json', function() { - url = `${base}no-content`; + const url = `${base}no-content`; return fetch(url).then(res => { expect(res.status).to.equal(204); expect(res.statusText).to.equal('No Content'); @@ -533,7 +532,7 @@ describe('node-fetch', () => { }); it('should handle no content response with gzip encoding', function() { - url = `${base}no-content/gzip`; + const url = `${base}no-content/gzip`; return fetch(url).then(res => { expect(res.status).to.equal(204); expect(res.statusText).to.equal('No Content'); @@ -547,7 +546,7 @@ describe('node-fetch', () => { }); it('should handle not modified response', function() { - url = `${base}not-modified`; + const url = `${base}not-modified`; return fetch(url).then(res => { expect(res.status).to.equal(304); expect(res.statusText).to.equal('Not Modified'); @@ -560,7 +559,7 @@ describe('node-fetch', () => { }); it('should handle not modified response with gzip encoding', function() { - url = `${base}not-modified/gzip`; + const url = `${base}not-modified/gzip`; return fetch(url).then(res => { expect(res.status).to.equal(304); expect(res.statusText).to.equal('Not Modified'); @@ -574,7 +573,7 @@ describe('node-fetch', () => { }); it('should decompress gzip response', function() { - url = `${base}gzip`; + const url = `${base}gzip`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); return res.text().then(result => { @@ -585,7 +584,7 @@ describe('node-fetch', () => { }); it('should decompress slightly invalid gzip response', function() { - url = `${base}gzip-truncated`; + const url = `${base}gzip-truncated`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); return res.text().then(result => { @@ -596,7 +595,7 @@ describe('node-fetch', () => { }); it('should decompress deflate response', function() { - url = `${base}deflate`; + const url = `${base}deflate`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); return res.text().then(result => { @@ -607,7 +606,7 @@ describe('node-fetch', () => { }); it('should decompress deflate raw response from old apache server', function() { - url = `${base}deflate-raw`; + const url = `${base}deflate-raw`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); return res.text().then(result => { @@ -618,7 +617,7 @@ describe('node-fetch', () => { }); it('should skip decompression if unsupported', function() { - url = `${base}sdch`; + const url = `${base}sdch`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); return res.text().then(result => { @@ -629,7 +628,7 @@ describe('node-fetch', () => { }); it('should reject if response compression is invalid', function() { - url = `${base}invalid-content-encoding`; + const url = `${base}invalid-content-encoding`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); return expect(res.text()).to.eventually.be.rejected @@ -639,7 +638,7 @@ describe('node-fetch', () => { }); it('should handle errors on the body stream even if it is not used', function(done) { - url = `${base}invalid-content-encoding`; + const url = `${base}invalid-content-encoding`; fetch(url) .then(res => { expect(res.status).to.equal(200); @@ -663,7 +662,7 @@ describe('node-fetch', () => { }); } - url = `${base}invalid-content-encoding`; + const url = `${base}invalid-content-encoding`; return fetch(url).then(delay).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); return expect(res.text()).to.eventually.be.rejected @@ -673,8 +672,8 @@ describe('node-fetch', () => { }); it('should allow disabling auto decompression', function() { - url = `${base}gzip`; - opts = { + const url = `${base}gzip`; + const opts = { compress: false }; return fetch(url, opts).then(res => { @@ -688,8 +687,8 @@ describe('node-fetch', () => { it('should allow custom timeout', function() { this.timeout(500); - url = `${base}timeout`; - opts = { + const url = `${base}timeout`; + const opts = { timeout: 100 }; return expect(fetch(url, opts)).to.eventually.be.rejected @@ -699,8 +698,8 @@ describe('node-fetch', () => { it('should allow custom timeout on response body', function() { this.timeout(500); - url = `${base}slow`; - opts = { + const url = `${base}slow`; + const opts = { timeout: 100 }; return fetch(url, opts).then(res => { @@ -736,15 +735,15 @@ describe('node-fetch', () => { }); it('should set default User-Agent', function () { - url = `${base}inspect`; + const url = `${base}inspect`; fetch(url).then(res => res.json()).then(res => { expect(res.headers['user-agent']).to.startWith('node-fetch/'); }); }); it('should allow setting User-Agent', function () { - url = `${base}inspect`; - opts = { + const url = `${base}inspect`; + const opts = { headers: { 'user-agent': 'faked' } @@ -755,15 +754,15 @@ describe('node-fetch', () => { }); it('should set default Accept header', function () { - url = `${base}inspect`; + const url = `${base}inspect`; fetch(url).then(res => res.json()).then(res => { expect(res.headers.accept).to.equal('*/*'); }); }); it('should allow setting Accept header', function () { - url = `${base}inspect`; - opts = { + const url = `${base}inspect`; + const opts = { headers: { 'accept': 'application/json' } @@ -774,8 +773,8 @@ describe('node-fetch', () => { }); it('should allow POST request', function() { - url = `${base}inspect`; - opts = { + const url = `${base}inspect`; + const opts = { method: 'POST' }; return fetch(url, opts).then(res => { @@ -789,10 +788,10 @@ describe('node-fetch', () => { }); it('should allow POST request with string body', function() { - url = `${base}inspect`; - opts = { - method: 'POST' - , body: 'a=1' + const url = `${base}inspect`; + const opts = { + method: 'POST', + body: 'a=1' }; return fetch(url, opts).then(res => { return res.json(); @@ -806,10 +805,10 @@ describe('node-fetch', () => { }); it('should allow POST request with buffer body', function() { - url = `${base}inspect`; - opts = { - method: 'POST' - , body: Buffer.from('a=1', 'utf-8') + const url = `${base}inspect`; + const opts = { + method: 'POST', + body: Buffer.from('a=1', 'utf-8') }; return fetch(url, opts).then(res => { return res.json(); @@ -823,10 +822,10 @@ describe('node-fetch', () => { }); it('should allow POST request with ArrayBuffer body', function() { - url = `${base}inspect`; - opts = { - method: 'POST' - , body: stringToArrayBuffer('Hello, world!\n') + const url = `${base}inspect`; + const opts = { + method: 'POST', + body: stringToArrayBuffer('Hello, world!\n') }; return fetch(url, opts).then(res => res.json()).then(res => { expect(res.method).to.equal('POST'); @@ -838,10 +837,10 @@ describe('node-fetch', () => { }); it('should allow POST request with blob body without type', function() { - url = `${base}inspect`; - opts = { - method: 'POST' - , body: new Blob(['a=1']) + const url = `${base}inspect`; + const opts = { + method: 'POST', + body: new Blob(['a=1']) }; return fetch(url, opts).then(res => { return res.json(); @@ -855,8 +854,8 @@ describe('node-fetch', () => { }); it('should allow POST request with blob body with type', function() { - url = `${base}inspect`; - opts = { + const url = `${base}inspect`; + const opts = { method: 'POST', body: new Blob(['a=1'], { type: 'text/plain;charset=UTF-8' @@ -877,10 +876,10 @@ describe('node-fetch', () => { let body = resumer().queue('a=1').end(); body = body.pipe(new stream.PassThrough()); - url = `${base}inspect`; - opts = { - method: 'POST' - , body + const url = `${base}inspect`; + const opts = { + method: 'POST', + body }; return fetch(url, opts).then(res => { return res.json(); @@ -897,10 +896,10 @@ describe('node-fetch', () => { const form = new FormData(); form.append('a','1'); - url = `${base}multipart`; - opts = { - method: 'POST' - , body: form + const url = `${base}multipart`; + const opts = { + method: 'POST', + body: form }; return fetch(url, opts).then(res => { return res.json(); @@ -916,10 +915,10 @@ describe('node-fetch', () => { const form = new FormData(); form.append('my_field', fs.createReadStream(path.join(__dirname, 'dummy.txt'))); - url = `${base}multipart`; - opts = { - method: 'POST' - , body: form + const url = `${base}multipart`; + const opts = { + method: 'POST', + body: form }; return fetch(url, opts).then(res => { @@ -939,11 +938,11 @@ describe('node-fetch', () => { const headers = form.getHeaders(); headers['b'] = '2'; - url = `${base}multipart`; - opts = { - method: 'POST' - , body: form - , headers + const url = `${base}multipart`; + const opts = { + method: 'POST', + body: form, + headers }; return fetch(url, opts).then(res => { return res.json(); @@ -957,11 +956,11 @@ describe('node-fetch', () => { }); it('should allow POST request with object body', function() { - url = `${base}inspect`; + const url = `${base}inspect`; // note that fetch simply calls tostring on an object - opts = { - method: 'POST' - , body: { a:1 } + const opts = { + method: 'POST', + body: { a: 1 } }; return fetch(url, opts).then(res => { return res.json(); @@ -978,8 +977,8 @@ describe('node-fetch', () => { const params = new URLSearchParams(); params.append('a','1'); - url = `${base}inspect`; - opts = { + const url = `${base}inspect`; + const opts = { method: 'POST', body: params, }; @@ -998,8 +997,8 @@ describe('node-fetch', () => { const params = new CustomSearchParams(); params.append('a','1'); - url = `${base}inspect`; - opts = { + const url = `${base}inspect`; + const opts = { method: 'POST', body: params, }; @@ -1020,8 +1019,8 @@ describe('node-fetch', () => { const params = new CustomPolyfilledSearchParams(); params.append('a','1'); - url = `${base}inspect`; - opts = { + const url = `${base}inspect`; + const opts = { method: 'POST', body: params, }; @@ -1036,9 +1035,9 @@ describe('node-fetch', () => { }); it('should overwrite Content-Length if possible', function() { - url = `${base}inspect`; + const url = `${base}inspect`; // note that fetch simply calls tostring on an object - opts = { + const opts = { method: 'POST', headers: { 'Content-Length': '1000' @@ -1057,10 +1056,10 @@ describe('node-fetch', () => { }); it('should allow PUT request', function() { - url = `${base}inspect`; - opts = { - method: 'PUT' - , body: 'a=1' + const url = `${base}inspect`; + const opts = { + method: 'PUT', + body: 'a=1' }; return fetch(url, opts).then(res => { return res.json(); @@ -1071,8 +1070,8 @@ describe('node-fetch', () => { }); it('should allow DELETE request', function() { - url = `${base}inspect`; - opts = { + const url = `${base}inspect`; + const opts = { method: 'DELETE' }; return fetch(url, opts).then(res => { @@ -1083,10 +1082,10 @@ describe('node-fetch', () => { }); it('should allow DELETE request with string body', function() { - url = `${base}inspect`; - opts = { - method: 'DELETE' - , body: 'a=1' + const url = `${base}inspect`; + const opts = { + method: 'DELETE', + body: 'a=1' }; return fetch(url, opts).then(res => { return res.json(); @@ -1099,10 +1098,10 @@ describe('node-fetch', () => { }); it('should allow PATCH request', function() { - url = `${base}inspect`; - opts = { - method: 'PATCH' - , body: 'a=1' + const url = `${base}inspect`; + const opts = { + method: 'PATCH', + body: 'a=1' }; return fetch(url, opts).then(res => { return res.json(); @@ -1113,8 +1112,8 @@ describe('node-fetch', () => { }); it('should allow HEAD request', function() { - url = `${base}hello`; - opts = { + const url = `${base}hello`; + const opts = { method: 'HEAD' }; return fetch(url, opts).then(res => { @@ -1129,8 +1128,8 @@ describe('node-fetch', () => { }); it('should allow HEAD request with content-encoding header', function() { - url = `${base}error/404`; - opts = { + const url = `${base}error/404`; + const opts = { method: 'HEAD' }; return fetch(url, opts).then(res => { @@ -1143,8 +1142,8 @@ describe('node-fetch', () => { }); it('should allow OPTIONS request', function() { - url = `${base}options`; - opts = { + const url = `${base}options`; + const opts = { method: 'OPTIONS' }; return fetch(url, opts).then(res => { @@ -1156,7 +1155,7 @@ describe('node-fetch', () => { }); it('should reject decoding body twice', function() { - url = `${base}plain`; + const url = `${base}plain`; return fetch(url).then(res => { expect(res.headers.get('content-type')).to.equal('text/plain'); return res.text().then(result => { @@ -1167,8 +1166,8 @@ describe('node-fetch', () => { }); it('should support maximum response size, multiple chunk', function() { - url = `${base}size/chunk`; - opts = { + const url = `${base}size/chunk`; + const opts = { size: 5 }; return fetch(url, opts).then(res => { @@ -1181,8 +1180,8 @@ describe('node-fetch', () => { }); it('should support maximum response size, single chunk', function() { - url = `${base}size/long`; - opts = { + const url = `${base}size/long`; + const opts = { size: 5 }; return fetch(url, opts).then(res => { @@ -1195,7 +1194,7 @@ describe('node-fetch', () => { }); it('should allow piping response body as stream', function() { - url = `${base}hello`; + const url = `${base}hello`; return fetch(url).then(res => { expect(res.body).to.be.an.instanceof(stream.Transform); return streamToPromise(res.body, chunk => { @@ -1208,7 +1207,7 @@ describe('node-fetch', () => { }); it('should allow cloning a response, and use both as stream', function() { - url = `${base}hello`; + const url = `${base}hello`; return fetch(url).then(res => { const r1 = res.clone(); expect(res.body).to.be.an.instanceof(stream.Transform); @@ -1228,7 +1227,7 @@ describe('node-fetch', () => { }); it('should allow cloning a json response and log it as text response', function() { - url = `${base}json`; + const url = `${base}json`; return fetch(url).then(res => { const r1 = res.clone(); return Promise.all([res.json(), r1.text()]).then(results => { @@ -1239,7 +1238,7 @@ describe('node-fetch', () => { }); it('should allow cloning a json response, and then log it as text response', function() { - url = `${base}json`; + const url = `${base}json`; return fetch(url).then(res => { const r1 = res.clone(); return res.json().then(result => { @@ -1252,7 +1251,7 @@ describe('node-fetch', () => { }); it('should allow cloning a json response, first log as text response, then return json object', function() { - url = `${base}json`; + const url = `${base}json`; return fetch(url).then(res => { const r1 = res.clone(); return r1.text().then(result => { @@ -1265,7 +1264,7 @@ describe('node-fetch', () => { }); it('should not allow cloning a response after its been used', function() { - url = `${base}hello`; + const url = `${base}hello`; return fetch(url).then(res => res.text().then(result => { expect(() => { @@ -1276,7 +1275,7 @@ describe('node-fetch', () => { }); it('should allow get all responses of a header', function() { - url = `${base}cookie`; + const url = `${base}cookie`; return fetch(url).then(res => { const expected = 'a=1, b=1'; expect(res.headers.get('set-cookie')).to.equal(expected); @@ -1285,7 +1284,7 @@ describe('node-fetch', () => { }); it('should return all headers using raw()', function() { - url = `${base}cookie`; + const url = `${base}cookie`; return fetch(url).then(res => { const expected = [ 'a=1', @@ -1297,7 +1296,7 @@ describe('node-fetch', () => { }); it('should allow deleting header', function() { - url = `${base}cookie`; + const url = `${base}cookie`; return fetch(url).then(res => { res.headers.delete('set-cookie'); expect(res.headers.get('set-cookie')).to.be.null; @@ -1305,8 +1304,8 @@ describe('node-fetch', () => { }); it('should send request with connection keep-alive if agent is provided', function() { - url = `${base}inspect`; - opts = { + const url = `${base}inspect`; + const opts = { agent: new http.Agent({ keepAlive: true }) @@ -1319,7 +1318,7 @@ describe('node-fetch', () => { }); it('should support fetch with Request instance', function() { - url = `${base}hello`; + const url = `${base}hello`; const req = new Request(url); return fetch(req).then(res => { expect(res.url).to.equal(url); @@ -1329,7 +1328,7 @@ describe('node-fetch', () => { }); it('should support fetch with Node.js URL object', function() { - url = `${base}hello`; + const url = `${base}hello`; const urlObj = parseURL(url); const req = new Request(urlObj); return fetch(req).then(res => { @@ -1340,7 +1339,7 @@ describe('node-fetch', () => { }); it('should support fetch with WHATWG URL object', function() { - url = `${base}hello`; + const url = `${base}hello`; const urlObj = new URL(url); const req = new Request(urlObj); return fetch(req).then(res => { @@ -1351,12 +1350,12 @@ describe('node-fetch', () => { }); it('should support blob round-trip', function() { - url = `${base}hello`; + const url = `${base}hello`; let length, type; return fetch(url).then(res => res.blob()).then(blob => { - url = `${base}inspect`; + const url = `${base}inspect`; length = blob.size; type = blob.type; return fetch(url, { @@ -1371,16 +1370,16 @@ describe('node-fetch', () => { }); it('should support overwrite Request instance', function() { - url = `${base}inspect`; + const url = `${base}inspect`; const req = new Request(url, { - method: 'POST' - , headers: { + method: 'POST', + headers: { a: '1' } }); return fetch(req, { - method: 'GET' - , headers: { + method: 'GET', + headers: { a: '2' } }).then(res => { @@ -1418,8 +1417,8 @@ describe('node-fetch', () => { it('should support https request', function() { this.timeout(5000); - url = 'https://github.com/'; - opts = { + const url = 'https://github.com/'; + const opts = { method: 'HEAD' }; return fetch(url, opts).then(res => { @@ -1461,9 +1460,9 @@ describe('Headers', function () { }); expect(result).to.deep.equal([ - ["a", "1"] - , ["b", "2, 3"] - , ["c", "4"] + ["a", "1"], + ["b", "2, 3"], + ["c", "4"] ]); }); @@ -1731,10 +1730,10 @@ describe('Response', function () { const res = new Response(body, { headers: { a: '1' - } - , url: base - , status: 346 - , statusText: 'production' + }, + url: base, + status: 346, + statusText: 'production' }); const cl = res.clone(); expect(cl.headers.get('a')).to.equal('1'); @@ -1815,15 +1814,15 @@ describe('Request', function () { }); it('should support wrapping Request instance', function() { - url = `${base}hello`; + const url = `${base}hello`; const form = new FormData(); form.append('a', '1'); const r1 = new Request(url, { - method: 'POST' - , follow: 1 - , body: form + method: 'POST', + follow: 1, + body: form }); const r2 = new Request(r1, { follow: 2 @@ -1861,7 +1860,7 @@ describe('Request', function () { }); it('should support parsing headers', function() { - url = base; + const url = base; const req = new Request(url, { headers: { a: '1' @@ -1872,7 +1871,7 @@ describe('Request', function () { }); it('should support arrayBuffer() method', function() { - url = base; + const url = base; var req = new Request(url, { method: 'POST', body: 'a=1' @@ -1886,7 +1885,7 @@ describe('Request', function () { }); it('should support text() method', function() { - url = base; + const url = base; const req = new Request(url, { method: 'POST', body: 'a=1' @@ -1898,7 +1897,7 @@ describe('Request', function () { }); it('should support json() method', function() { - url = base; + const url = base; const req = new Request(url, { method: 'POST', body: '{"a":1}' @@ -1910,7 +1909,7 @@ describe('Request', function () { }); it('should support buffer() method', function() { - url = base; + const url = base; const req = new Request(url, { method: 'POST', body: 'a=1' @@ -1922,7 +1921,7 @@ describe('Request', function () { }); it('should support blob() method', function() { - url = base; + const url = base; var req = new Request(url, { method: 'POST', body: Buffer.from('a=1') @@ -1936,26 +1935,26 @@ describe('Request', function () { }); it('should support arbitrary url', function() { - url = 'anything'; + const url = 'anything'; const req = new Request(url); expect(req.url).to.equal('anything'); }); it('should support clone() method', function() { - url = base; + const url = base; let body = resumer().queue('a=1').end(); body = body.pipe(new stream.PassThrough()); const agent = new http.Agent(); const req = new Request(url, { - body - , method: 'POST' - , redirect: 'manual' - , headers: { + body, + method: 'POST', + redirect: 'manual', + headers: { b: '2' - } - , follow: 3 - , compress: false - , agent + }, + follow: 3, + compress: false, + agent }); const cl = req.clone(); expect(cl.url).to.equal(url); @@ -1997,7 +1996,7 @@ describe('external encoding', () => { }); it('should only use UTF-8 decoding with text()', function() { - url = `${base}encoding/euc-jp`; + const url = `${base}encoding/euc-jp`; return fetch(url).then(res => { expect(res.status).to.equal(200); return res.text().then(result => { @@ -2007,7 +2006,7 @@ describe('external encoding', () => { }); it('should support encoding decode, xml dtd detect', function() { - url = `${base}encoding/euc-jp`; + const url = `${base}encoding/euc-jp`; return fetch(url).then(res => { expect(res.status).to.equal(200); return res.textConverted().then(result => { @@ -2017,7 +2016,7 @@ describe('external encoding', () => { }); it('should support encoding decode, content-type detect', function() { - url = `${base}encoding/shift-jis`; + const url = `${base}encoding/shift-jis`; return fetch(url).then(res => { expect(res.status).to.equal(200); return res.textConverted().then(result => { @@ -2027,7 +2026,7 @@ describe('external encoding', () => { }); it('should support encoding decode, html5 detect', function() { - url = `${base}encoding/gbk`; + const url = `${base}encoding/gbk`; return fetch(url).then(res => { expect(res.status).to.equal(200); return res.textConverted().then(result => { @@ -2037,7 +2036,7 @@ describe('external encoding', () => { }); it('should support encoding decode, html4 detect', function() { - url = `${base}encoding/gb2312`; + const url = `${base}encoding/gb2312`; return fetch(url).then(res => { expect(res.status).to.equal(200); return res.textConverted().then(result => { @@ -2047,7 +2046,7 @@ describe('external encoding', () => { }); it('should default to utf8 encoding', function() { - url = `${base}encoding/utf8`; + const url = `${base}encoding/utf8`; return fetch(url).then(res => { expect(res.status).to.equal(200); expect(res.headers.get('content-type')).to.be.null; @@ -2058,7 +2057,7 @@ describe('external encoding', () => { }); it('should support uncommon content-type order, charset in front', function() { - url = `${base}encoding/order1`; + const url = `${base}encoding/order1`; return fetch(url).then(res => { expect(res.status).to.equal(200); return res.textConverted().then(result => { @@ -2068,7 +2067,7 @@ describe('external encoding', () => { }); it('should support uncommon content-type order, end with qs', function() { - url = `${base}encoding/order2`; + const url = `${base}encoding/order2`; return fetch(url).then(res => { expect(res.status).to.equal(200); return res.textConverted().then(result => { @@ -2078,7 +2077,7 @@ describe('external encoding', () => { }); it('should support chunked encoding, html4 detect', function() { - url = `${base}encoding/chunked`; + const url = `${base}encoding/chunked`; return fetch(url).then(res => { expect(res.status).to.equal(200); const padding = 'a'.repeat(10); @@ -2089,7 +2088,7 @@ describe('external encoding', () => { }); it('should only do encoding detection up to 1024 bytes', function() { - url = `${base}encoding/invalid`; + const url = `${base}encoding/invalid`; return fetch(url).then(res => { expect(res.status).to.equal(200); const padding = 'a'.repeat(1200); @@ -2106,7 +2105,7 @@ describe('external encoding', () => { }); it('should throw a FetchError if res.textConverted() is called without `encoding` in require cache', () => { - url = `${base}hello`; + const url = `${base}hello`; return fetch(url).then((res) => { return expect(res.textConverted()).to.eventually.be.rejected .and.have.property('message').which.includes('encoding') From ccaeae096f654a1df3d81c3a4e250aa859659d39 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 4 Mar 2018 20:41:14 -0800 Subject: [PATCH 146/223] v2.1.0 --- CHANGELOG.md | 8 ++++++++ package.json | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a0f18316..3df595800 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ Changelog # 2.x release +## v2.1.0 + +- Enhance: allow using ArrayBuffer as the `body` of a `fetch()` or `Request` +- Fix: store HTTP headers of a `Headers` object internally with the given case, for compatibility with older servers that incorrectly treated header names in a case-sensitive manner +- Fix: silently ignore invalid HTTP headers +- Fix: handle HTTP redirect responses without a `Location` header just like non-redirect responses +- Fix: include bodies when following a redirection when appropriate + ## v2.0.0 This is a major release. Check [our upgrade guide](https://github.com/bitinn/node-fetch/blob/master/UPGRADE-GUIDE.md) for an overview on some key differences between v1 and v2. diff --git a/package.json b/package.json index 6ed380e26..6434d768a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.0.0", + "version": "2.1.0", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", "browser": "./browser.js", From 6a9828e0b540cf3e7c89758b6ea5df9d8c29496b Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 4 Mar 2018 20:43:54 -0800 Subject: [PATCH 147/223] Update CHANGELOG --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3df595800..61b267963 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ Changelog # 2.x release +## v2.1.1 + +Fix packaging errors in v2.1.0. + ## v2.1.0 - Enhance: allow using ArrayBuffer as the `body` of a `fetch()` or `Request` From fb86ab34a6e6f00bfae5fd8cbebda9af4a0f412f Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 4 Mar 2018 20:44:09 -0800 Subject: [PATCH 148/223] v2.1.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6434d768a..528eec76b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.1.0", + "version": "2.1.1", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", "browser": "./browser.js", From 4c4f2f29e52f194870bd9b31c6f72f13bfcd9d54 Mon Sep 17 00:00:00 2001 From: William MacDonald Date: Tue, 20 Mar 2018 09:57:03 -0700 Subject: [PATCH 149/223] Wrap ArrayBuffer with Buffer internally to fix Body methods (#426) --- src/body.js | 2 +- test/test.js | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/body.js b/src/body.js index 60d100573..3dc4d98f1 100644 --- a/src/body.js +++ b/src/body.js @@ -206,7 +206,7 @@ function consumeBody() { // body is buffer if (Object.prototype.toString.call(this.body) === '[object ArrayBuffer]') { - return Body.Promise.resolve(this.body); + return Body.Promise.resolve(Buffer.from(this.body)); } // istanbul ignore if: should never happen diff --git a/test/test.js b/test/test.js index ba9cfa50e..be8539dbe 100644 --- a/test/test.js +++ b/test/test.js @@ -1771,6 +1771,13 @@ describe('Response', function () { }); }); + it('should support ArrayBuffer as body', function() { + const res = new Response(stringToArrayBuffer('a=1')); + return res.text().then(result => { + expect(result).to.equal('a=1'); + }); + }); + it('should support blob as body', function() { const res = new Response(new Blob(['a=1'])); return res.text().then(result => { @@ -1973,6 +1980,16 @@ describe('Request', function () { expect(results[1]).to.equal('a=1'); }); }); + + it('should support ArrayBuffer as body', function() { + const req = new Request('', { + method: 'POST', + body: stringToArrayBuffer('a=1') + }); + return req.text().then(result => { + expect(result).to.equal('a=1'); + }); + }); }); function streamToPromise(stream, dataHandler) { From c012c4116baaa71226b09cbc2b8c4e7715a00425 Mon Sep 17 00:00:00 2001 From: Jason Rogers Date: Thu, 22 Mar 2018 21:38:03 -0400 Subject: [PATCH 150/223] Reject when stream accumulation fails (#415) Fixes: #414 --- src/body.js | 8 +++++++- test/test.js | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/body.js b/src/body.js index 3dc4d98f1..5bc6b4feb 100644 --- a/src/body.js +++ b/src/body.js @@ -257,7 +257,13 @@ function consumeBody() { } clearTimeout(resTimeout); - resolve(Buffer.concat(accum)); + + try { + resolve(Buffer.concat(accum)); + } catch (err) { + // handle streams that have accumulated too much data (issue #414) + reject(new FetchError(`Could not create Buffer from response body for ${this.url}: ${err.message}`, 'system', err)); + } }); }); } diff --git a/test/test.js b/test/test.js index be8539dbe..555446d8a 100644 --- a/test/test.js +++ b/test/test.js @@ -1427,6 +1427,25 @@ describe('node-fetch', () => { }); }); + // issue #414 + it('should reject if attempt to accumulate body stream throws', function () { + let body = resumer().queue('a=1').end(); + body = body.pipe(new stream.PassThrough()); + const res = new Response(body); + const bufferConcat = Buffer.concat; + const restoreBufferConcat = () => Buffer.concat = bufferConcat; + Buffer.concat = () => { throw new Error('embedded error'); }; + + const textPromise = res.text(); + // Ensure that `Buffer.concat` is always restored: + textPromise.then(restoreBufferConcat, restoreBufferConcat); + + return expect(textPromise).to.eventually.be.rejected + .and.be.an.instanceOf(FetchError) + .and.include({ type: 'system' }) + .and.have.property('message').that.includes('Could not create Buffer') + .and.that.includes('embedded error'); + }); }); describe('Headers', function () { From 8aac53679d04ade28068bdec63488e35908717dd Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Thu, 22 Mar 2018 22:01:45 -0700 Subject: [PATCH 151/223] Fix custom Host header with arbitrary case (#430) Regression since 1592ca1148e657110fd6f64dc8b29a576cd1da1c. Fixes: #416 Fixes: #425 --- src/headers.js | 19 +++++++++++++++++++ src/index.js | 5 ----- src/request.js | 4 ++-- test/test.js | 14 ++++++++++++++ 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/headers.js b/src/headers.js index 80077e80d..6b75371e8 100644 --- a/src/headers.js +++ b/src/headers.js @@ -323,6 +323,25 @@ Object.defineProperty(HeadersIteratorPrototype, Symbol.toStringTag, { configurable: true }); +/** + * Export the Headers object in a form that Node.js can consume. + * + * @param Headers headers + * @return Object + */ +export function exportNodeCompatibleHeaders(headers) { + const obj = Object.assign({ __proto__: null }, headers[MAP]); + + // http.request() only supports string as Host header. This hack makes + // specifying custom Host header possible. + const hostHeaderKey = find(headers[MAP], 'Host'); + if (hostHeaderKey !== undefined) { + obj[hostHeaderKey] = obj[hostHeaderKey][0]; + } + + return obj; +} + /** * Create a Headers object from an object of headers, ignoring those that do * not conform to HTTP grammar productions. diff --git a/src/index.js b/src/index.js index d395dade2..c6d856efa 100644 --- a/src/index.js +++ b/src/index.js @@ -43,11 +43,6 @@ export default function fetch(url, opts) { const send = (options.protocol === 'https:' ? https : http).request; - // http.request only support string as host header, this hack make custom host header possible - if (options.headers.host) { - options.headers.host = options.headers.host[0]; - } - // send request const req = send(options); let reqTimeout; diff --git a/src/request.js b/src/request.js index 108a6bf2e..fb067838f 100644 --- a/src/request.js +++ b/src/request.js @@ -7,7 +7,7 @@ * All spec algorithm step numbers are based on https://fetch.spec.whatwg.org/commit-snapshots/ae716822cb3a61843226cd090eefc6589446c1d2/. */ -import Headers from './headers.js'; +import Headers, { exportNodeCompatibleHeaders } from './headers.js'; import Body, { clone, extractContentType, getTotalBytes } from './body'; const { format: format_url, parse: parse_url } = require('url'); @@ -200,7 +200,7 @@ export function getNodeRequestOptions(request) { return Object.assign({}, parsedURL, { method: request.method, - headers: headers.raw(), + headers: exportNodeCompatibleHeaders(headers), agent: request.agent }); } diff --git a/test/test.js b/test/test.js index 555446d8a..cca2e61dd 100644 --- a/test/test.js +++ b/test/test.js @@ -208,6 +208,20 @@ describe('node-fetch', () => { }); }); + it('should accept custom HoSt header', function() { + const url = `${base}inspect`; + const opts = { + headers: { + HoSt: 'example.com' + } + }; + return fetch(url, opts).then(res => { + return res.json(); + }).then(res => { + expect(res.headers['host']).to.equal('example.com'); + }); + }); + it('should follow redirect code 301', function() { const url = `${base}redirect/301`; return fetch(url).then(res => { From a1cbcb5706a230dfcc185f9ac5c7e6091029a83c Mon Sep 17 00:00:00 2001 From: Mark Herhold Date: Sat, 24 Mar 2018 14:29:00 -0400 Subject: [PATCH 152/223] Support TypeScript import in browser (#433) --- browser.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/browser.js b/browser.js index 7e24a0735..c6849f92b 100644 --- a/browser.js +++ b/browser.js @@ -1,4 +1,8 @@ module.exports = exports = window.fetch; + +// Needed for TypeScript. +exports.default = window.fetch; + exports.Headers = window.Headers; exports.Request = window.Request; exports.Response = window.Response; From d522036bee580abdca0fc6ab7afcd38f573e4f96 Mon Sep 17 00:00:00 2001 From: Mark Herhold Date: Sat, 24 Mar 2018 23:22:34 -0400 Subject: [PATCH 153/223] Bind fetch to window in the browser (#434) --- browser.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/browser.js b/browser.js index c6849f92b..ee023e0c5 100644 --- a/browser.js +++ b/browser.js @@ -1,7 +1,7 @@ module.exports = exports = window.fetch; -// Needed for TypeScript. -exports.default = window.fetch; +// Needed for TypeScript and Webpack. +exports.default = window.fetch.bind(window); exports.Headers = window.Headers; exports.Request = window.Request; From aed2e69a39bfe2759aa2062e2b3effeb6ce05dbc Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sat, 24 Mar 2018 20:48:33 -0700 Subject: [PATCH 154/223] Make sure to finalize the request properly (#432) Unfortunately, I could not write a test case that allows testing the bug in #428. Credits to Roman Zaharenkov for discovering this long-standing bug and proposing a first version of the fix. Co-authored-by: Roman Zaharenkov Fixes: #428 --- src/index.js | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/index.js b/src/index.js index c6d856efa..1d6d85b01 100644 --- a/src/index.js +++ b/src/index.js @@ -47,18 +47,23 @@ export default function fetch(url, opts) { const req = send(options); let reqTimeout; + function finalize() { + req.abort(); + clearTimeout(reqTimeout); + } + if (request.timeout) { req.once('socket', socket => { reqTimeout = setTimeout(() => { - req.abort(); reject(new FetchError(`network timeout at: ${request.url}`, 'request-timeout')); + finalize(); }, request.timeout); }); } req.on('error', err => { - clearTimeout(reqTimeout); reject(new FetchError(`request to ${request.url} failed, reason: ${err.message}`, 'system', err)); + finalize(); }); req.on('response', res => { @@ -78,6 +83,7 @@ export default function fetch(url, opts) { switch (request.redirect) { case 'error': reject(new FetchError(`redirect mode is set to error: ${request.url}`, 'no-redirect')); + finalize(); return; case 'manual': // node-fetch-specific step: make manual redirect a bit easier to use by setting the Location header value to the resolved URL. @@ -94,6 +100,7 @@ export default function fetch(url, opts) { // HTTP-redirect fetch step 5 if (request.counter >= request.follow) { reject(new FetchError(`maximum redirect reached at: ${request.url}`, 'max-redirect')); + finalize(); return; } @@ -111,7 +118,9 @@ export default function fetch(url, opts) { // HTTP-redirect fetch step 9 if (res.statusCode !== 303 && request.body && getTotalBytes(request) === null) { - reject(new FetchError('Cannot follow redirect with body being a readable stream', 'unsupported-redirect')) + reject(new FetchError('Cannot follow redirect with body being a readable stream', 'unsupported-redirect')); + finalize(); + return; } // HTTP-redirect fetch step 11 @@ -123,6 +132,7 @@ export default function fetch(url, opts) { // HTTP-redirect fetch step 15 resolve(fetch(new Request(locationURL, requestOpts))); + finalize(); return; } } From 13b230b30b46f3cca3a74267695767a2372f75e9 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 25 Mar 2018 11:51:37 -0700 Subject: [PATCH 155/223] Add a test for redirect with stream body See 35722f19e5cf203d877ebc5f2d44a327aca6dcaf. --- test/test.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/test.js b/test/test.js index cca2e61dd..05a4c4a26 100644 --- a/test/test.js +++ b/test/test.js @@ -367,6 +367,17 @@ describe('node-fetch', () => { }); }); + it('should not follow non-GET redirect if body is a readable stream', function() { + const url = `${base}redirect/307`; + const opts = { + method: 'PATCH', + body: resumer().queue('a=1').end() + }; + return expect(fetch(url, opts)).to.eventually.be.rejected + .and.be.an.instanceOf(FetchError) + .and.have.property('type', 'unsupported-redirect'); + }); + it('should obey maximum redirect, reject case', function() { const url = `${base}redirect/chain`; const opts = { From f6683aa49cf4f74f10ad8868836168cb8da28253 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 25 Mar 2018 12:29:28 -0700 Subject: [PATCH 156/223] Add a test case for invalid headers See feae6d6ec83de4bef57ffa0a3fe42bcd91d1922b. --- test/server.js | 14 ++++++++++++++ test/test.js | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/test/server.js b/test/server.js index d5594633b..6641b1657 100644 --- a/test/server.js +++ b/test/server.js @@ -117,6 +117,20 @@ export default class TestServer { res.end('fake gzip string'); } + if (p === '/invalid-header') { + res.setHeader('Content-Type', 'text/plain'); + res.writeHead(200); + // HACK: add a few invalid headers to the generated header string before + // it is actually sent to the socket. + res._header = res._header.replace(/\r\n$/, [ + 'Invalid-Header : abc\r\n', + 'Invalid-Header-Value: \x07k\r\n', + 'Set-Cookie: \x07k\r\n', + 'Set-Cookie: \x07kk\r\n', + ].join('') + '\r\n'); + res.end('hello world\n'); + } + if (p === '/timeout') { setTimeout(function() { res.statusCode = 200; diff --git a/test/test.js b/test/test.js index 05a4c4a26..573e31fde 100644 --- a/test/test.js +++ b/test/test.js @@ -477,6 +477,20 @@ describe('node-fetch', () => { }); }); + it('should ignore invalid headers', function() { + const url = `${base}invalid-header`; + return fetch(url).then(res => { + expect(res.headers.get('Invalid-Header')).to.be.null; + expect(res.headers.get('Invalid-Header-Value')).to.be.null; + expect(res.headers.get('Set-Cookie')).to.be.null; + expect(Array.from(res.headers.keys()).length).to.equal(4); + expect(res.headers.has('Connection')).to.be.true; + expect(res.headers.has('Content-Type')).to.be.true; + expect(res.headers.has('Date')).to.be.true; + expect(res.headers.has('Transfer-Encoding')).to.be.true; + }); + }); + it('should handle client-error response', function() { const url = `${base}error/400`; return fetch(url).then(res => { From 1d51752ea35729802a43bb1cff173c09e1dd9eb2 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 25 Mar 2018 13:54:40 -0700 Subject: [PATCH 157/223] Update CHANGELOG [ci skip] --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61b267963..6bc6196de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ Changelog # 2.x release +## v2.1.2 + +- Fix: allow `Body` methods to work on ArrayBuffer`-backed `Body` objects +- Fix: reject promise returned by `Body` methods when the accumulated `Buffer` exceeds the maximum size +- Fix: support custom `Host` headers with any casing +- Fix: support importing `fetch()` from TypeScript in `browser.js` +- Fix: handle the redirect response body properly + ## v2.1.1 Fix packaging errors in v2.1.0. From b80c2f56a098fffbb3a5556b2034a3a35c37b2a7 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 25 Mar 2018 13:55:02 -0700 Subject: [PATCH 158/223] v2.1.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 528eec76b..b95e54ee1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.1.1", + "version": "2.1.2", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", "browser": "./browser.js", From bae5fdd30692be3f68a4d2f715b9bbc5d120543c Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 25 Mar 2018 14:09:00 -0700 Subject: [PATCH 159/223] Fix up CHANGELOG typo [ci skip] --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bc6196de..2c0207104 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ Changelog ## v2.1.2 -- Fix: allow `Body` methods to work on ArrayBuffer`-backed `Body` objects +- Fix: allow `Body` methods to work on `ArrayBuffer`-backed `Body` objects - Fix: reject promise returned by `Body` methods when the accumulated `Buffer` exceeds the maximum size - Fix: support custom `Host` headers with any casing - Fix: support importing `fetch()` from TypeScript in `browser.js` From 6b42bd68cbd85adf9c4a7e3ec444d45b8f0ffca1 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 25 Mar 2018 14:56:37 -0700 Subject: [PATCH 160/223] Consolidate Travis CI matrix for encoding (#436) See: #431 --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index d16cb5353..c233e09f2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,13 +6,13 @@ node_js: - "node" env: - FORMDATA_VERSION=1.0.0 - - FORMDATA_VERSION=1.0.0 ENCODING=yes - FORMDATA_VERSION=2.1.0 - - FORMDATA_VERSION=2.1.0 ENCODING=yes before_script: - 'if [ "$FORMDATA_VERSION" ]; then npm install form-data@^$FORMDATA_VERSION; fi' - - 'if [ "$ENCODING" = "yes" ]; then npm install encoding; fi' -script: npm run coverage +script: + - npm run coverage + - npm install encoding + - npm run coverage cache: directories: - node_modules From 989c8434a945736b644ca5b05d280ad36f269e8b Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 25 Mar 2018 18:59:21 -0700 Subject: [PATCH 161/223] Uninstall encoding before first Travis test run encoding can be included in the Travis CI cache, and thus needs to be manually uninstalled first. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index c233e09f2..0f8a1e5f2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ env: before_script: - 'if [ "$FORMDATA_VERSION" ]; then npm install form-data@^$FORMDATA_VERSION; fi' script: + - npm uninstall encoding - npm run coverage - npm install encoding - npm run coverage From fa6548ed316a3c512147a3bb4f4d6e986d738be6 Mon Sep 17 00:00:00 2001 From: Deepak Date: Sun, 8 Apr 2018 07:57:47 +0530 Subject: [PATCH 162/223] fix: add the autoClose: true to download file example (#441) close #375 --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5f9a091f8..2d7d68c04 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,9 @@ fetch('http://domain.invalid/') fetch('https://assets-cdn.github.com/images/modules/logos_page/Octocat.png') .then(res => { - const dest = fs.createWriteStream('./octocat.png'); + const dest = fs.createWriteStream('./octocat.png', { + autoClose: true, + }); res.body.pipe(dest); }); From 5bc23d81cf7ebb97b0a7310b2c88c13cdaa706dc Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Mon, 28 May 2018 06:18:17 +0300 Subject: [PATCH 163/223] Added support for `ArrayBufferView` (#457) --- src/body.js | 34 +++++++++++++++++++++++----------- test/test.js | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 11 deletions(-) diff --git a/src/body.js b/src/body.js index 5bc6b4feb..def889ae5 100644 --- a/src/body.js +++ b/src/body.js @@ -39,9 +39,11 @@ export default function Body(body, { } else if (body instanceof Blob) { // body is blob } else if (Buffer.isBuffer(body)) { - // body is buffer - } else if (Object.prototype.toString.call(body) === '[object ArrayBuffer]') { - // body is array buffer + // body is ArrayBuffer + } else if (body instanceof ArrayBuffer) { + // body is ArrayBufferView + } else if (ArrayBuffer.isView(body)) { + // body is array buffer view } else if (body instanceof Stream) { // body is stream } else { @@ -204,8 +206,8 @@ function consumeBody() { return Body.Promise.resolve(this.body); } - // body is buffer - if (Object.prototype.toString.call(this.body) === '[object ArrayBuffer]') { + // body is ArrayBuffer + if (this.body instanceof ArrayBuffer) { return Body.Promise.resolve(Buffer.from(this.body)); } @@ -416,8 +418,11 @@ export function extractContentType(instance) { } else if (Buffer.isBuffer(body)) { // body is buffer return null; - } else if (Object.prototype.toString.call(body) === '[object ArrayBuffer]') { - // body is array buffer + } else if (body instanceof ArrayBuffer) { + // body is ArrayBuffer + return null; + } else if (ArrayBuffer.isView(body)) { + // body is ArrayBufferView return null; } else if (typeof body.getBoundary === 'function') { // detect form data input from form-data module @@ -457,8 +462,11 @@ export function getTotalBytes(instance) { } else if (Buffer.isBuffer(body)) { // body is buffer return body.length; - } else if (Object.prototype.toString.call(body) === '[object ArrayBuffer]') { - // body is array buffer + } else if (body instanceof ArrayBuffer) { + // body is ArrayBuffer + return body.byteLength; + } else if (ArrayBuffer.isView(body)) { + // body is ArrayBufferView return body.byteLength; } else if (body && typeof body.getLengthSync === 'function') { // detect form data input from form-data module @@ -502,10 +510,14 @@ export function writeToStream(dest, instance) { // body is buffer dest.write(body); dest.end() - } else if (Object.prototype.toString.call(body) === '[object ArrayBuffer]') { - // body is array buffer + } else if (body instanceof ArrayBuffer) { + // body is ArrayBuffer dest.write(Buffer.from(body)); dest.end() + } else if (ArrayBuffer.isView(body)) { + // body is ArrayBufferView + dest.write(Buffer.from(body.buffer, body.byteOffset, body.byteLength)); + dest.end() } else { // body is stream body.pipe(dest); diff --git a/test/test.js b/test/test.js index 573e31fde..2e2699cb1 100644 --- a/test/test.js +++ b/test/test.js @@ -875,6 +875,52 @@ describe('node-fetch', () => { }); }); + it('should allow POST request with ArrayBufferView (Uint8Array) body', function() { + const url = `${base}inspect`; + const opts = { + method: 'POST', + body: new Uint8Array(stringToArrayBuffer('Hello, world!\n')) + }; + return fetch(url, opts).then(res => res.json()).then(res => { + expect(res.method).to.equal('POST'); + expect(res.body).to.equal('Hello, world!\n'); + expect(res.headers['transfer-encoding']).to.be.undefined; + expect(res.headers['content-type']).to.be.undefined; + expect(res.headers['content-length']).to.equal('14'); + }); + }); + + it('should allow POST request with ArrayBufferView (DataView) body', function() { + const url = `${base}inspect`; + const opts = { + method: 'POST', + body: new DataView(stringToArrayBuffer('Hello, world!\n')) + }; + return fetch(url, opts).then(res => res.json()).then(res => { + expect(res.method).to.equal('POST'); + expect(res.body).to.equal('Hello, world!\n'); + expect(res.headers['transfer-encoding']).to.be.undefined; + expect(res.headers['content-type']).to.be.undefined; + expect(res.headers['content-length']).to.equal('14'); + }); + }); + + // TODO: Node.js v4 doesn't support necessary Buffer API, so we skip this test, drop this check once Node.js v4 support is not needed + (Buffer.from.length === 3 ? it : it.skip)('should allow POST request with ArrayBufferView (Uint8Array, offset, length) body', function() { + const url = `${base}inspect`; + const opts = { + method: 'POST', + body: new Uint8Array(stringToArrayBuffer('Hello, world!\n'), 7, 6) + }; + return fetch(url, opts).then(res => res.json()).then(res => { + expect(res.method).to.equal('POST'); + expect(res.body).to.equal('world!'); + expect(res.headers['transfer-encoding']).to.be.undefined; + expect(res.headers['content-type']).to.be.undefined; + expect(res.headers['content-length']).to.equal('6'); + }); + }); + it('should allow POST request with blob body without type', function() { const url = `${base}inspect`; const opts = { From 2b7e1ab27fb3ee424df73a70170f39bafa650b0d Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 27 May 2018 21:26:22 -0700 Subject: [PATCH 164/223] Refine README example (#463) This reverts commit fa6548ed316a3c512147a3bb4f4d6e986d738be6 (#441). The autoClose option has been true by default since at least Node.js v6.0.0. There is no need to set it once more. Instead, make the example more realistic by handling stream outcomes using a promise. See #375. --- README.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2d7d68c04..6805ec3ae 100644 --- a/README.md +++ b/README.md @@ -80,10 +80,19 @@ fetch('http://domain.invalid/') fetch('https://assets-cdn.github.com/images/modules/logos_page/Octocat.png') .then(res => { - const dest = fs.createWriteStream('./octocat.png', { - autoClose: true, + return new Promise((resolve, reject) => { + const dest = fs.createWriteStream('./octocat.png'); + res.body.pipe(dest); + res.body.on('error', err => { + reject(err); + }); + dest.on('finish', () => { + resolve(); + }); + dest.on('error', err => { + reject(err); + }); }); - res.body.pipe(dest); }); // buffer From b2c5f543ce934dc55cd039095cb1bfe79eb6b6ef Mon Sep 17 00:00:00 2001 From: "Bernhard K. Weisshuhn" Date: Tue, 19 Jun 2018 10:50:09 +0200 Subject: [PATCH 165/223] test supplying a lookup function through an agent --- test/test.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/test.js b/test/test.js index 2e2699cb1..e3a7113ad 100644 --- a/test/test.js +++ b/test/test.js @@ -17,6 +17,7 @@ const fs = require('fs'); const path = require('path'); const stream = require('stream'); const { parse: parseURL, URLSearchParams } = require('url'); +const { lookup } = require('dns'); let convert; try { convert = require('encoding').convert; } catch(e) { } @@ -1531,6 +1532,19 @@ describe('node-fetch', () => { .and.have.property('message').that.includes('Could not create Buffer') .and.that.includes('embedded error'); }); + + it("supports supplying a lookup function to the agent", function() { + const url = `${base}redirect/301`; + let called = 0; + function lookupSpy(hostname, options, callback) { + called++; + return lookup(hostname, options, callback); + } + const agent = http.Agent({ lookup: lookupSpy }); + return fetch(url, { agent }).then(() => { + expect(called).to.equal(2); + }); + }); }); describe('Headers', function () { From a33d6b0de15217003353285aeac5e812db271e8d Mon Sep 17 00:00:00 2001 From: "Bernhard K. Weisshuhn" Date: Tue, 19 Jun 2018 18:39:48 +0200 Subject: [PATCH 166/223] mention agent options `family` and `lookup` in readme for reference --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6805ec3ae..2580b34c6 100644 --- a/README.md +++ b/README.md @@ -216,7 +216,7 @@ The default values are shown after each option key. timeout: 0, // req/res timeout in ms, it resets on redirect. 0 to disable (OS limit applies) compress: true, // support gzip/deflate content encoding. false to disable size: 0, // maximum response body size in bytes. 0 to disable - agent: null // http(s).Agent instance, allows custom proxy, certificate etc. + agent: null // http(s).Agent instance, allows custom proxy, certificate, lookup, family etc. } ``` From 287bc3bdcf75c160b1ddf645009b9c9165d920a7 Mon Sep 17 00:00:00 2001 From: "Bernhard K. Weisshuhn" Date: Tue, 19 Jun 2018 19:08:34 +0200 Subject: [PATCH 167/223] test agent option `family` being passed to `lookup` --- test/test.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/test.js b/test/test.js index e3a7113ad..65a035768 100644 --- a/test/test.js +++ b/test/test.js @@ -1545,6 +1545,22 @@ describe('node-fetch', () => { expect(called).to.equal(2); }); }); + + it("supports supplying a famliy option to the agent", function() { + const url = `${base}redirect/301`; + const families = []; + const family = Symbol('family'); + function lookupSpy(hostname, options, callback) { + families.push(options.family) + return lookup(hostname, {}, callback); + } + const agent = http.Agent({ lookup: lookupSpy, family }); + return fetch(url, { agent }).then(() => { + expect(families).to.have.length(2); + expect(families[0]).to.equal(family); + expect(families[1]).to.equal(family); + }); + }); }); describe('Headers', function () { From b091ab5917c691748ca81df707dbd8a280c9d5d3 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sat, 21 Jul 2018 22:13:01 -0700 Subject: [PATCH 168/223] Fix up ArrayBufferView support (#464) Also add more test coverage. Fixes: #482 Closes: #484 --- src/body.js | 19 +++++++----- test/test.js | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 7 deletions(-) diff --git a/src/body.js b/src/body.js index def889ae5..4698c4c5a 100644 --- a/src/body.js +++ b/src/body.js @@ -39,11 +39,11 @@ export default function Body(body, { } else if (body instanceof Blob) { // body is blob } else if (Buffer.isBuffer(body)) { + // body is Buffer + } else if (Object.prototype.toString.call(body) === '[object ArrayBuffer]') { // body is ArrayBuffer - } else if (body instanceof ArrayBuffer) { - // body is ArrayBufferView } else if (ArrayBuffer.isView(body)) { - // body is array buffer view + // body is ArrayBufferView } else if (body instanceof Stream) { // body is stream } else { @@ -207,10 +207,15 @@ function consumeBody() { } // body is ArrayBuffer - if (this.body instanceof ArrayBuffer) { + if (Object.prototype.toString.call(this.body) === '[object ArrayBuffer]') { return Body.Promise.resolve(Buffer.from(this.body)); } + // body is ArrayBufferView + if (ArrayBuffer.isView(this.body)) { + return Body.Promise.resolve(Buffer.from(this.body.buffer, this.body.byteOffset, this.body.byteLength)); + } + // istanbul ignore if: should never happen if (!(this.body instanceof Stream)) { return Body.Promise.resolve(Buffer.alloc(0)); @@ -418,7 +423,7 @@ export function extractContentType(instance) { } else if (Buffer.isBuffer(body)) { // body is buffer return null; - } else if (body instanceof ArrayBuffer) { + } else if (Object.prototype.toString.call(body) === '[object ArrayBuffer]') { // body is ArrayBuffer return null; } else if (ArrayBuffer.isView(body)) { @@ -462,7 +467,7 @@ export function getTotalBytes(instance) { } else if (Buffer.isBuffer(body)) { // body is buffer return body.length; - } else if (body instanceof ArrayBuffer) { + } else if (Object.prototype.toString.call(body) === '[object ArrayBuffer]') { // body is ArrayBuffer return body.byteLength; } else if (ArrayBuffer.isView(body)) { @@ -510,7 +515,7 @@ export function writeToStream(dest, instance) { // body is buffer dest.write(body); dest.end() - } else if (body instanceof ArrayBuffer) { + } else if (Object.prototype.toString.call(body) === '[object ArrayBuffer]') { // body is ArrayBuffer dest.write(Buffer.from(body)); dest.end() diff --git a/test/test.js b/test/test.js index 65a035768..e8342cb56 100644 --- a/test/test.js +++ b/test/test.js @@ -18,6 +18,12 @@ const path = require('path'); const stream = require('stream'); const { parse: parseURL, URLSearchParams } = require('url'); const { lookup } = require('dns'); +const vm = require('vm'); + +const { + ArrayBuffer: VMArrayBuffer, + Uint8Array: VMUint8Array +} = vm.runInNewContext('this'); let convert; try { convert = require('encoding').convert; } catch(e) { } @@ -876,6 +882,27 @@ describe('node-fetch', () => { }); }); + it('should allow POST request with ArrayBuffer body from a VM context', function() { + // TODO: Node.js v4 doesn't support ArrayBuffer from other contexts, so we skip this test, drop this check once Node.js v4 support is not needed + try { + Buffer.from(new VMArrayBuffer()); + } catch (err) { + this.skip(); + } + const url = `${base}inspect`; + const opts = { + method: 'POST', + body: new VMUint8Array(Buffer.from('Hello, world!\n')).buffer + }; + return fetch(url, opts).then(res => res.json()).then(res => { + expect(res.method).to.equal('POST'); + expect(res.body).to.equal('Hello, world!\n'); + expect(res.headers['transfer-encoding']).to.be.undefined; + expect(res.headers['content-type']).to.be.undefined; + expect(res.headers['content-length']).to.equal('14'); + }); + }); + it('should allow POST request with ArrayBufferView (Uint8Array) body', function() { const url = `${base}inspect`; const opts = { @@ -906,6 +933,27 @@ describe('node-fetch', () => { }); }); + it('should allow POST request with ArrayBufferView (Uint8Array) body from a VM context', function() { + // TODO: Node.js v4 doesn't support ArrayBufferView from other contexts, so we skip this test, drop this check once Node.js v4 support is not needed + try { + Buffer.from(new VMArrayBuffer()); + } catch (err) { + this.skip(); + } + const url = `${base}inspect`; + const opts = { + method: 'POST', + body: new VMUint8Array(Buffer.from('Hello, world!\n')) + }; + return fetch(url, opts).then(res => res.json()).then(res => { + expect(res.method).to.equal('POST'); + expect(res.body).to.equal('Hello, world!\n'); + expect(res.headers['transfer-encoding']).to.be.undefined; + expect(res.headers['content-type']).to.be.undefined; + expect(res.headers['content-length']).to.equal('14'); + }); + }); + // TODO: Node.js v4 doesn't support necessary Buffer API, so we skip this test, drop this check once Node.js v4 support is not needed (Buffer.from.length === 3 ? it : it.skip)('should allow POST request with ArrayBufferView (Uint8Array, offset, length) body', function() { const url = `${base}inspect`; @@ -1919,6 +1967,20 @@ describe('Response', function () { }); }); + it('should support Uint8Array as body', function() { + const res = new Response(new Uint8Array(stringToArrayBuffer('a=1'))); + return res.text().then(result => { + expect(result).to.equal('a=1'); + }); + }); + + it('should support DataView as body', function() { + const res = new Response(new DataView(stringToArrayBuffer('a=1'))); + return res.text().then(result => { + expect(result).to.equal('a=1'); + }); + }); + it('should default to null as body', function() { const res = new Response(); expect(res.body).to.equal(null); @@ -2124,6 +2186,26 @@ describe('Request', function () { expect(result).to.equal('a=1'); }); }); + + it('should support Uint8Array as body', function() { + const req = new Request('', { + method: 'POST', + body: new Uint8Array(stringToArrayBuffer('a=1')) + }); + return req.text().then(result => { + expect(result).to.equal('a=1'); + }); + }); + + it('should support DataView as body', function() { + const req = new Request('', { + method: 'POST', + body: new DataView(stringToArrayBuffer('a=1')) + }); + return req.text().then(result => { + expect(result).to.equal('a=1'); + }); + }); }); function streamToPromise(stream, dataHandler) { From ed9e886cf3e981e881ec4a2e270393ccb51572fb Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 22 Jul 2018 09:21:46 -0700 Subject: [PATCH 169/223] Support web workers (#487) Fixes: #485 --- browser.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/browser.js b/browser.js index ee023e0c5..3ed04dede 100644 --- a/browser.js +++ b/browser.js @@ -1,8 +1,10 @@ -module.exports = exports = window.fetch; +"use strict"; + +module.exports = exports = self.fetch; // Needed for TypeScript and Webpack. -exports.default = window.fetch.bind(window); +exports.default = self.fetch.bind(self); -exports.Headers = window.Headers; -exports.Request = window.Request; -exports.Response = window.Response; +exports.Headers = self.Headers; +exports.Request = self.Request; +exports.Response = self.Response; From de66b388ac30371a82024d6deba931bdebc6a404 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 22 Jul 2018 09:22:10 -0700 Subject: [PATCH 170/223] Update dependencies (#488) --- package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index b95e54ee1..1e57efde5 100644 --- a/package.json +++ b/package.json @@ -48,14 +48,14 @@ "cross-env": "^5.1.3", "form-data": "^2.3.1", "mocha": "^5.0.0", - "nyc": "^11.4.1", + "nyc": "11.9.0", "parted": "^0.1.1", "promise": "^8.0.1", "resumer": "0.0.0", - "rollup": "^0.55.1", + "rollup": "^0.63.4", "rollup-plugin-babel": "^3.0.3", "string-to-arraybuffer": "^1.0.0", - "url-search-params": "^0.10.0", + "url-search-params": "^1.0.2", "whatwg-url": "^5.0.0" }, "dependencies": {} From ecd82580aad659bf8b38f7b1176ecd4f89c87f22 Mon Sep 17 00:00:00 2001 From: Steven Date: Sun, 22 Jul 2018 16:34:32 -0400 Subject: [PATCH 171/223] Add badge to display install size (#455) --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 2580b34c6..c4d1a22cc 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ node-fetch [![npm next version][npm-next-image]][npm-url] [![build status][travis-image]][travis-url] [![coverage status][codecov-image]][codecov-url] +[![install size][install-size-image]][install-size-url] A light-weight module that brings `window.fetch` to Node.js @@ -410,6 +411,8 @@ Thanks to [github/fetch](https://github.com/github/fetch) for providing a solid [travis-url]: https://travis-ci.org/bitinn/node-fetch [codecov-image]: https://img.shields.io/codecov/c/github/bitinn/node-fetch.svg?style=flat-square [codecov-url]: https://codecov.io/gh/bitinn/node-fetch +[install-size-image]: https://packagephobia.now.sh/badge?p=node-fetch +[install-size-url]: https://packagephobia.now.sh/result?p=node-fetch [ERROR-HANDLING.md]: https://github.com/bitinn/node-fetch/blob/master/ERROR-HANDLING.md [LIMITS.md]: https://github.com/bitinn/node-fetch/blob/master/LIMITS.md From 6d7daa27c79a6627f9d1bd4577a064739e863ef6 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 22 Jul 2018 14:05:40 -0700 Subject: [PATCH 172/223] Migrate from mocha --compilers (#491) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1e57efde5..b445ab673 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "scripts": { "build": "cross-env BABEL_ENV=rollup rollup -c", "prepare": "npm run build", - "test": "cross-env BABEL_ENV=test mocha --compilers js:babel-register test/test.js", + "test": "cross-env BABEL_ENV=test mocha --require babel-register test/test.js", "report": "cross-env BABEL_ENV=coverage nyc --reporter lcov --reporter text mocha -R spec test/test.js", "coverage": "cross-env BABEL_ENV=coverage nyc --reporter json --reporter text mocha -R spec test/test.js && codecov -f coverage/coverage-final.json" }, From c01a5d22d411bb8508766f4ed7f922db99e38b3d Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 22 Jul 2018 09:53:01 -0700 Subject: [PATCH 173/223] Move .default assignment to plugins --- build/rollup-plugin.js | 5 +++-- src/index.js | 3 --- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/build/rollup-plugin.js b/build/rollup-plugin.js index 7da31860b..4f44eb8e9 100644 --- a/build/rollup-plugin.js +++ b/build/rollup-plugin.js @@ -4,9 +4,10 @@ export default function tweakDefault() { var lines = source.split('\n'); for (var i = 0; i < lines.length; i++) { var line = lines[i]; - var matches = /^exports(?:\['default']|\.default) = (.*);$/.exec(line); + var matches = /^(exports(?:\['default']|\.default)) = (.*);$/.exec(line); if (matches) { - lines[i] = 'module.exports = exports = ' + matches[1] + ';'; + lines[i] = 'module.exports = exports = ' + matches[2] + ';\n' + + matches[1] + ' = exports;'; break; } } diff --git a/src/index.js b/src/index.js index 1d6d85b01..9ffc5ba29 100644 --- a/src/index.js +++ b/src/index.js @@ -215,9 +215,6 @@ export default function fetch(url, opts) { */ fetch.isRedirect = code => code === 301 || code === 302 || code === 303 || code === 307 || code === 308; -// Needed for TypeScript. -fetch.default = fetch; - // expose Promise fetch.Promise = global.Promise; export { From 6868e4aa59a1bcc9ef263c9c2cfa8beca1e6779e Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 22 Jul 2018 09:51:06 -0700 Subject: [PATCH 174/223] Add __esModule property to exports object Fixes: #442 --- build/babel-plugin.js | 13 +++++++++++++ build/rollup-plugin.js | 1 + 2 files changed, 14 insertions(+) diff --git a/build/babel-plugin.js b/build/babel-plugin.js index 08efac90b..8cddae954 100644 --- a/build/babel-plugin.js +++ b/build/babel-plugin.js @@ -29,6 +29,19 @@ module.exports = ({ types: t }) => ({ ) ) ), + t.expressionStatement( + t.callExpression( + t.memberExpression( + t.identifier('Object'), t.identifier('defineProperty')), + [ + t.identifier('exports'), + t.stringLiteral('__esModule'), + t.objectExpression([ + t.objectProperty(t.identifier('value'), t.booleanLiteral(true)) + ]) + ] + ) + ), t.expressionStatement( t.assignmentExpression('=', expr.node.left, t.identifier('exports') diff --git a/build/rollup-plugin.js b/build/rollup-plugin.js index 4f44eb8e9..36ebdc804 100644 --- a/build/rollup-plugin.js +++ b/build/rollup-plugin.js @@ -7,6 +7,7 @@ export default function tweakDefault() { var matches = /^(exports(?:\['default']|\.default)) = (.*);$/.exec(line); if (matches) { lines[i] = 'module.exports = exports = ' + matches[2] + ';\n' + + 'Object.defineProperty(exports, "__esModule", { value: true });\n' + matches[1] + ' = exports;'; break; } From 1d4ab5a0de0df991f92f1f30efd5f4815bbcdb71 Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 22 Jul 2018 09:58:03 -0700 Subject: [PATCH 175/223] Switch to .mjs for ES module output for Node.js compat This also reverts commit 60cf26c2f3baf566c15632b723664b47f5b1f2db. --- package.json | 6 ++++-- rollup.config.js | 7 ++++++- src/body.js | 4 +--- src/index.js | 12 ++++++------ src/request.js | 3 +-- src/response.js | 3 +-- 6 files changed, 19 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index b445ab673..3f7fdbaa2 100644 --- a/package.json +++ b/package.json @@ -2,11 +2,12 @@ "name": "node-fetch", "version": "2.1.2", "description": "A light-weight module that brings window.fetch to node.js", - "main": "lib/index.js", + "main": "lib/index", "browser": "./browser.js", - "module": "lib/index.es.js", + "module": "lib/index.mjs", "files": [ "lib/index.js", + "lib/index.mjs", "lib/index.es.js", "browser.js" ], @@ -47,6 +48,7 @@ "codecov": "^3.0.0", "cross-env": "^5.1.3", "form-data": "^2.3.1", + "is-builtin-module": "^1.0.0", "mocha": "^5.0.0", "nyc": "11.9.0", "parted": "^0.1.1", diff --git a/rollup.config.js b/rollup.config.js index c66f98052..a201ee455 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -1,3 +1,4 @@ +import isBuiltin from 'is-builtin-module'; import babel from 'rollup-plugin-babel'; import tweakDefault from './build/rollup-plugin'; @@ -7,7 +8,8 @@ export default { input: 'src/index.js', output: [ { file: 'lib/index.js', format: 'cjs', exports: 'named' }, - { file: 'lib/index.es.js', format: 'es', exports: 'named' } + { file: 'lib/index.es.js', format: 'es', exports: 'named', intro: 'process.emitWarning("The .es.js file is deprecated. Use .mjs instead.");' }, + { file: 'lib/index.mjs', format: 'es', exports: 'named' }, ], plugins: [ babel({ @@ -16,6 +18,9 @@ export default { tweakDefault() ], external: function (id) { + if (isBuiltin(id)) { + return true; + } id = id.split('/').slice(0, id[0] === '@' ? 2 : 1).join('/'); return !!require('./package.json').dependencies[id]; } diff --git a/src/body.js b/src/body.js index 4698c4c5a..c6dd4b63d 100644 --- a/src/body.js +++ b/src/body.js @@ -5,12 +5,10 @@ * Body interface provides common methods for Request and Response */ +import Stream, { PassThrough } from 'stream'; import Blob, { BUFFER } from './blob.js'; import FetchError from './fetch-error.js'; -const Stream = require('stream'); -const { PassThrough } = require('stream'); - let convert; try { convert = require('encoding').convert; } catch(e) {} diff --git a/src/index.js b/src/index.js index 9ffc5ba29..fa485e42a 100644 --- a/src/index.js +++ b/src/index.js @@ -7,18 +7,18 @@ * All spec algorithm step numbers are based on https://fetch.spec.whatwg.org/commit-snapshots/ae716822cb3a61843226cd090eefc6589446c1d2/. */ +import { resolve as resolve_url } from 'url'; +import http from 'http'; +import https from 'https'; +import zlib from 'zlib'; +import { PassThrough } from 'stream'; + import Body, { writeToStream, getTotalBytes } from './body'; import Response from './response'; import Headers, { createHeadersLenient } from './headers'; import Request, { getNodeRequestOptions } from './request'; import FetchError from './fetch-error'; -const http = require('http'); -const https = require('https'); -const { PassThrough } = require('stream'); -const { resolve: resolve_url } = require('url'); -const zlib = require('zlib'); - /** * Fetch function * diff --git a/src/request.js b/src/request.js index fb067838f..748ba555c 100644 --- a/src/request.js +++ b/src/request.js @@ -7,11 +7,10 @@ * All spec algorithm step numbers are based on https://fetch.spec.whatwg.org/commit-snapshots/ae716822cb3a61843226cd090eefc6589446c1d2/. */ +import { format as format_url, parse as parse_url } from 'url'; import Headers, { exportNodeCompatibleHeaders } from './headers.js'; import Body, { clone, extractContentType, getTotalBytes } from './body'; -const { format: format_url, parse: parse_url } = require('url'); - const INTERNALS = Symbol('Request internals'); /** diff --git a/src/response.js b/src/response.js index dd0a2ceba..506d876fa 100644 --- a/src/response.js +++ b/src/response.js @@ -5,11 +5,10 @@ * Response class provides content decoding */ +import { STATUS_CODES } from 'http'; import Headers from './headers.js'; import Body, { clone } from './body'; -const { STATUS_CODES } = require('http'); - const INTERNALS = Symbol('Response internals'); /** From 09ef40e8a883344a38f8ebddc58ff0ac129a415c Mon Sep 17 00:00:00 2001 From: Timothy Gu Date: Sun, 22 Jul 2018 14:31:00 -0700 Subject: [PATCH 176/223] 2.2.0 --- CHANGELOG.md | 9 +++++++++ package.json | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c0207104..f321e0ee9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ Changelog # 2.x release +## v2.2.0 + +- Enhance: Support all `ArrayBuffer` view types +- Enhance: Support Web Workers +- Enhance: Support Node.js' `--experimental-modules` mode; deprecate `.es.js` file +- Fix: Add `__esModule` property to the exports object +- Other: Better example in README for writing response to a file +- Other: More tests for Agent + ## v2.1.2 - Fix: allow `Body` methods to work on `ArrayBuffer`-backed `Body` objects diff --git a/package.json b/package.json index 3f7fdbaa2..8aa0d64be 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.1.2", + "version": "2.2.0", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index", "browser": "./browser.js", From 745a27c38953129ce81f162b23c3d24f4a4d18f3 Mon Sep 17 00:00:00 2001 From: Jared Kantrowitz Date: Sat, 1 Sep 2018 07:23:02 -0400 Subject: [PATCH 177/223] README update (#504) * v2.x readme overhaul with additions discussed in #448 added "comments" (TODO link references) for changes suggested but not yet implemented for future discussion/prs clarified "native stream" to be "native Node streams" adjusted all uses of http to https to encourage secure protocol usage adjusted whatwg to proper case, WHATWG made code block tags consistent as `js` instead of `javascript` uppercased all method option values (post vs POST) added spec-compliant node to the `response.ok` api section * fix left over cruft, inconsistent hierarchy --- README.md | 346 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 207 insertions(+), 139 deletions(-) diff --git a/README.md b/README.md index c4d1a22cc..738805a8f 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,46 @@ - node-fetch ========== -[![npm stable version][npm-image]][npm-url] -[![npm next version][npm-next-image]][npm-url] +[![npm version][npm-image]][npm-url] [![build status][travis-image]][travis-url] [![coverage status][codecov-image]][codecov-url] [![install size][install-size-image]][install-size-url] A light-weight module that brings `window.fetch` to Node.js + + +- [Motivation](#motivation) +- [Features](#features) +- [Difference from client-side fetch](#difference-from-client-side-fetch) +- [Installation](#installation) +- [Loading and configuring the module](#loading-and-configuring-the-module) +- [Common Usage](#common-usage) + - [Plain text or HTML](#plain-text-or-html) + - [JSON](#json) + - [Simple Post](#simple-post) + - [Post with JSON](#post-with-json) + - [Post with form parameters](#post-with-form-parameters) + - [Handling exceptions](#handling-exceptions) + - [Handling client and server errors](#handling-client-and-server-errors) +- [Advanced Usage](#advanced-usage) + - [Streams](#streams) + - [Buffer](#buffer) + - [Accessing Headers and other Meta data](#accessing-headers-and-other-meta-data) + - [Post data using a file stream](#post-data-using-a-file-stream) + - [Post with form-data (detect multipart)](#post-with-form-data-detect-multipart) +- [API](#api) + - [fetch(url[, options])](#fetchurl-options) + - [Options](#options) + - [Class: Request](#class-request) + - [Class: Response](#class-response) + - [Class: Headers](#class-headers) + - [Interface: Body](#interface-body) + - [Class: FetchError](#class-fetcherror) +- [License](#license) +- [Acknowledgement](#acknowledgement) + + ## Motivation @@ -17,171 +48,204 @@ Instead of implementing `XMLHttpRequest` in Node.js to run browser-specific [Fet See Matt Andrews' [isomorphic-fetch](https://github.com/matthew-andrews/isomorphic-fetch) or Leonardo Quixada's [cross-fetch](https://github.com/lquixada/cross-fetch) for isomorphic usage (exports `node-fetch` for server-side, `whatwg-fetch` for client-side). - ## Features - Stay consistent with `window.fetch` API. -- Make conscious trade-off when following [whatwg fetch spec][whatwg-fetch] and [stream spec](https://streams.spec.whatwg.org/) implementation details, document known difference. +- Make conscious trade-off when following [WHATWG fetch spec][whatwg-fetch] and [stream spec](https://streams.spec.whatwg.org/) implementation details, document known differences. - Use native promise, but allow substituting it with [insert your favorite promise library]. -- Use native stream for body, on both request and response. -- Decode content encoding (gzip/deflate) properly, convert `res.text()` output to UTF-8 optionally. -- Useful extensions such as timeout, redirect limit, response size limit, [explicit errors][ERROR-HANDLING.md] for troubleshooting. - +- Use native Node streams for body, on both request and response. +- Decode content encoding (gzip/deflate) properly, and convert string output (such as `res.text()` and `res.json()`) to UTF-8 automatically. +- Useful extensions such as timeout, redirect limit, response size limit, [explicit errors](ERROR-HANDLING.md) for troubleshooting. ## Difference from client-side fetch -- See [Known Differences][LIMITS.md] for details. +- See [Known Differences](LIMITS.md) for details. - If you happen to use a missing feature that `window.fetch` offers, feel free to open an issue. - Pull requests are welcomed too! +## Installation -## Install - -Stable release (`2.x`) +Current stable release (`2.x`) ```sh $ npm install node-fetch --save ``` -## Usage - -Note that documentation below is up-to-date with `2.x` releases, [see `1.x` readme](https://github.com/bitinn/node-fetch/blob/1.x/README.md), [changelog](https://github.com/bitinn/node-fetch/blob/1.x/CHANGELOG.md) and [2.x upgrade guide][UPGRADE-GUIDE.md] if you want to find out the difference. +## Loading and configuring the module +We suggest you load the module via `require`, pending the stabalizing of es modules in node: +```js +const fetch = require('node-fetch'); +``` -```javascript -import fetch from 'node-fetch'; -// or -// const fetch = require('node-fetch'); +If you are using a Promise library other than native, set it through fetch.Promise: +```js +const Bluebird = require('bluebird'); -// if you are using your own Promise library, set it through fetch.Promise. Eg. +fetch.Promise = Bluebird; +``` -// import Bluebird from 'bluebird'; -// fetch.Promise = Bluebird; +## Common Usage -// plain text or html +NOTE: The documentation below is up-to-date with `2.x` releases, [see `1.x` readme](https://github.com/bitinn/node-fetch/blob/1.x/README.md), [changelog](https://github.com/bitinn/node-fetch/blob/1.x/CHANGELOG.md) and [2.x upgrade guide](UPGRADE-GUIDE.md) for the differences. +#### Plain text or HTML +```js fetch('https://github.com/') - .then(res => res.text()) - .then(body => console.log(body)); + .then(res => res.text()) + .then(body => console.log(body)); +``` + +#### JSON -// json +```js fetch('https://api.github.com/users/github') - .then(res => res.json()) - .then(json => console.log(json)); + .then(res => res.json()) + .then(json => console.log(json)); +``` -// catching network error -// 3xx-5xx responses are NOT network errors, and should be handled in then() -// you only need one catch() at the end of your promise chain +#### Simple Post +```js +fetch('https://httpbin.org/post', { method: 'POST', body: 'a=1' }) + .then(res => res.json()) // expecting a json response + .then(json => console.log(json)); +``` -fetch('http://domain.invalid/') - .catch(err => console.error(err)); +#### Post with JSON -// stream -// the node.js way is to use stream when possible +```js +const body = { a: 1 }; + +fetch('https://httpbin.org/post', { + method: 'post', + body: JSON.stringify(body), + headers: { 'Content-Type': 'application/json' }, + }) + .then(res => res.json()) + .then(json => console.log(json)); +``` -fetch('https://assets-cdn.github.com/images/modules/logos_page/Octocat.png') - .then(res => { - return new Promise((resolve, reject) => { - const dest = fs.createWriteStream('./octocat.png'); - res.body.pipe(dest); - res.body.on('error', err => { - reject(err); - }); - dest.on('finish', () => { - resolve(); - }); - dest.on('error', err => { - reject(err); - }); - }); - }); - -// buffer -// if you prefer to cache binary data in full, use buffer() -// note that buffer() is a node-fetch only API - -import fileType from 'file-type'; +#### Post with form parameters +`URLSearchParams` is available in Node.js as of v7.5.0. See [official documentation](https://nodejs.org/api/url.html#url_class_urlsearchparams) for more usage methods. -fetch('https://assets-cdn.github.com/images/modules/logos_page/Octocat.png') - .then(res => res.buffer()) - .then(buffer => fileType(buffer)) - .then(type => { /* ... */ }); +NOTE: The `Content-Type` header is only set automatically to `x-www-form-urlencoded` when an instance of `URLSearchParams` is given as such: -// meta +```js +const { URLSearchParams } = require('url'); -fetch('https://github.com/') - .then(res => { - console.log(res.ok); - console.log(res.status); - console.log(res.statusText); - console.log(res.headers.raw()); - console.log(res.headers.get('content-type')); - }); +const params = new URLSearchParams(); +params.append('a', 1); -// post +fetch('https://httpbin.org/post', { method: 'POST', body: params }) + .then(res => res.json()) + .then(json => console.log(json)); +``` -fetch('http://httpbin.org/post', { method: 'POST', body: 'a=1' }) - .then(res => res.json()) - .then(json => console.log(json)); +#### Handling exceptions +NOTE: 3xx-5xx responses are *NOT* exceptions, and should be handled in `then()`, see the next section. -// post with stream from file +Adding a catch to the fetch promise chain will catch *all* exceptions, such as errors originating from node core libraries, like network errors, and operational errors which are instances of FetchError. See the [error handling document](ERROR-HANDLING.md) for more details. -import { createReadStream } from 'fs'; +```js +fetch('https://domain.invalid/') + .catch(err => console.error(err)); +``` -const stream = createReadStream('input.txt'); -fetch('http://httpbin.org/post', { method: 'POST', body: stream }) - .then(res => res.json()) - .then(json => console.log(json)); +#### Handling client and server errors +It is common to create a helper function to check that the response contains no client (4xx) or server (5xx) error responses: -// post with JSON +```js +function checkStatus(res) { + if (res.ok) { // res.status >= 200 && res.status < 300 + return res; + } else { + throw MyCustomError(res.statusText); + } +} -var body = { a: 1 }; -fetch('http://httpbin.org/post', { - method: 'POST', - body: JSON.stringify(body), - headers: { 'Content-Type': 'application/json' }, -}) - .then(res => res.json()) - .then(json => console.log(json)); +fetch('https://httpbin.org/status/400') + .then(checkStatus) + .then(res => console.log('will not get here...')) +``` -// post form parameters (x-www-form-urlencoded) +## Advanced Usage -import { URLSearchParams } from 'url'; +#### Streams +The "Node.js way" is to use streams when possible: -const params = new URLSearchParams(); -params.append('a', 1); -fetch('http://httpbin.org/post', { method: 'POST', body: params }) - .then(res => res.json()) - .then(json => console.log(json)); +```js +fetch('https://assets-cdn.github.com/images/modules/logos_page/Octocat.png') + .then(res => { + const dest = fs.createWriteStream('./octocat.png'); + res.body.pipe(dest); + }); +``` -// post with form-data (detect multipart) +[TODO]: # (Somewhere i think we also should mention arrayBuffer also if you want to be cross-fetch compatible.) -import FormData from 'form-data'; +#### Buffer +If you prefer to cache binary data in full, use buffer(). (NOTE: buffer() is a `node-fetch` only API) + +```js +const fileType = require('file-type'); + +fetch('https://assets-cdn.github.com/images/modules/logos_page/Octocat.png') + .then(res => res.buffer()) + .then(buffer => fileType(buffer)) + .then(type => { /* ... */ }); +``` + +#### Accessing Headers and other Meta data +```js +fetch('https://github.com/') + .then(res => { + console.log(res.ok); + console.log(res.status); + console.log(res.statusText); + console.log(res.headers.raw()); + console.log(res.headers.get('content-type')); + }); +``` + +#### Post data using a file stream + +```js +const { createReadStream } = require('fs'); + +const stream = createReadStream('input.txt'); + +fetch('https://httpbin.org/post', { method: 'POST', body: stream }) + .then(res => res.json()) + .then(json => console.log(json)); +``` + +#### Post with form-data (detect multipart) + +```js +const FormData = require('form-data'); const form = new FormData(); form.append('a', 1); -fetch('http://httpbin.org/post', { method: 'POST', body: form }) - .then(res => res.json()) - .then(json => console.log(json)); -// post with form-data (custom headers) -// note that getHeaders() is non-standard API +fetch('https://httpbin.org/post', { method: 'POST', body: form }) + .then(res => res.json()) + .then(json => console.log(json)); -import FormData from 'form-data'; +// OR, using custom headers +// NOTE: getHeaders() is non-standard API const form = new FormData(); form.append('a', 1); -fetch('http://httpbin.org/post', { method: 'POST', body: form, headers: form.getHeaders() }) - .then(res => res.json()) - .then(json => console.log(json)); -// node 7+ with async function +const options = { + method: 'POST', + body: form, + headers: form.getHeaders() +} -(async function () { - const res = await fetch('https://api.github.com/users/github'); - const json = await res.json(); - console.log(json); -})(); +fetch('https://httpbin.org/post', options) + .then(res => res.json()) + .then(json => console.log(json)); ``` See [test cases](https://github.com/bitinn/node-fetch/blob/master/test/test.js) for more examples. @@ -197,27 +261,29 @@ See [test cases](https://github.com/bitinn/node-fetch/blob/master/test/test.js) Perform an HTTP(S) fetch. -`url` should be an absolute url, such as `http://example.com/`. A path-relative URL (`/file/under/root`) or protocol-relative URL (`//can-be-http-or-https.com/`) will result in a rejected promise. +`url` should be an absolute url, such as `https://example.com/`. A path-relative URL (`/file/under/root`) or protocol-relative URL (`//can-be-http-or-https.com/`) will result in a rejected promise. + +[TODO]: # (It might be a good idea to reformat the options section into a table layout, like the headers section, instead of current code block.) -#### Options +### Options The default values are shown after each option key. ```js { - // These properties are part of the Fetch Standard - method: 'GET', - headers: {}, // request headers. format is the identical to that accepted by the Headers constructor (see below) - body: null, // request body. can be null, a string, a Buffer, a Blob, or a Node.js Readable stream - redirect: 'follow', // set to `manual` to extract redirect headers, `error` to reject redirect - - // The following properties are node-fetch extensions - follow: 20, // maximum redirect count. 0 to not follow redirect - timeout: 0, // req/res timeout in ms, it resets on redirect. 0 to disable (OS limit applies) - compress: true, // support gzip/deflate content encoding. false to disable - size: 0, // maximum response body size in bytes. 0 to disable - agent: null // http(s).Agent instance, allows custom proxy, certificate, lookup, family etc. + // These properties are part of the Fetch Standard + method: 'GET', + headers: {}, // request headers. format is the identical to that accepted by the Headers constructor (see below) + body: null, // request body. can be null, a string, a Buffer, a Blob, or a Node.js Readable stream + redirect: 'follow', // set to `manual` to extract redirect headers, `error` to reject redirect + + // The following properties are node-fetch extensions + follow: 20, // maximum redirect count. 0 to not follow redirect + timeout: 0, // req/res timeout in ms, it resets on redirect. 0 to disable (OS limit applies) + compress: true, // support gzip/deflate content encoding. false to disable + size: 0, // maximum response body size in bytes. 0 to disable + agent: null // http(s).Agent instance, allows custom proxy, certificate etc. } ``` @@ -225,6 +291,8 @@ The default values are shown after each option key. If no values are set, the following request headers will be sent automatically: +[TODO]: # ("we always said content-length will be "automatically calculated, if possible" in the default header section, but we never explain what's the condition for it to be calculated, and that chunked transfer-encoding will be used when they are not calculated or supplied." - "Maybe also add Transfer-Encoding: chunked? That header is added by Node.js automatically if the input is a stream, I believe.") + Header | Value ----------------- | -------------------------------------------------------- `Accept-Encoding` | `gzip,deflate` _(when `options.compress === true`)_ @@ -296,6 +364,8 @@ Because Node.js does not implement service workers (for which this class was des #### response.ok +*(spec-compliant)* + Convenience property representing if the request ended normally. Will evaluate to true if the response status was greater than or equal to 200 but smaller than 300. @@ -379,6 +449,8 @@ Consume the body and return a promise that will resolve to one of these formats. Consume the body and return a promise that will resolve to a Buffer. +[TODO]: # (textConverted API should mention an optional dependency on encoding, which users need to install by themselves, and this is done purely for backward compatibility with 1.x release.) + #### body.textConverted() *(node-fetch extension)* @@ -394,18 +466,15 @@ Identical to `body.text()`, except instead of always converting to UTF-8, encodi An operational error in the fetching process. See [ERROR-HANDLING.md][] for more info. -## License - -MIT - - ## Acknowledgement Thanks to [github/fetch](https://github.com/github/fetch) for providing a solid implementation reference. +## License + +MIT [npm-image]: https://img.shields.io/npm/v/node-fetch.svg?style=flat-square -[npm-next-image]: https://img.shields.io/npm/v/node-fetch/next.svg?style=flat-square [npm-url]: https://www.npmjs.com/package/node-fetch [travis-image]: https://img.shields.io/travis/bitinn/node-fetch.svg?style=flat-square [travis-url]: https://travis-ci.org/bitinn/node-fetch @@ -413,11 +482,10 @@ Thanks to [github/fetch](https://github.com/github/fetch) for providing a solid [codecov-url]: https://codecov.io/gh/bitinn/node-fetch [install-size-image]: https://packagephobia.now.sh/badge?p=node-fetch [install-size-url]: https://packagephobia.now.sh/result?p=node-fetch - -[ERROR-HANDLING.md]: https://github.com/bitinn/node-fetch/blob/master/ERROR-HANDLING.md -[LIMITS.md]: https://github.com/bitinn/node-fetch/blob/master/LIMITS.md -[UPGRADE-GUIDE.md]: https://github.com/bitinn/node-fetch/blob/master/UPGRADE-GUIDE.md - [whatwg-fetch]: https://fetch.spec.whatwg.org/ [response-init]: https://fetch.spec.whatwg.org/#responseinit [node-readable]: https://nodejs.org/api/stream.html#stream_readable_streams +[mdn-headers]: https://developer.mozilla.org/en-US/docs/Web/API/Headers +[LIMITS.md]: https://github.com/bitinn/node-fetch/blob/master/LIMITS.md +[ERROR-HANDLING.md]: https://github.com/bitinn/node-fetch/blob/master/ERROR-HANDLING.md +[UPGRADE-GUIDE.md]: https://github.com/bitinn/node-fetch/blob/master/UPGRADE-GUIDE.md \ No newline at end of file From 8cc909f5ee725c76dc8d2b0351a04e7e50dfc4ae Mon Sep 17 00:00:00 2001 From: David Frank Date: Wed, 24 Oct 2018 14:44:16 +0800 Subject: [PATCH 178/223] update readme to add credits and call for collaborators (#540) * also pin chai-string to ~1.3.0 as chai-string 1.5.0 introduce a breaking change that breaks our node v4 CI. --- README.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 738805a8f..af7c8dd37 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ node-fetch A light-weight module that brings `window.fetch` to Node.js +(We are looking for [v2 maintainers and collaborators](https://github.com/bitinn/node-fetch/issues/252)) + - [Motivation](#motivation) @@ -470,6 +472,8 @@ An operational error in the fetching process. See [ERROR-HANDLING.md][] for more Thanks to [github/fetch](https://github.com/github/fetch) for providing a solid implementation reference. +`node-fetch` v1 was maintained by [@bitinn](https://github.com/bitinn), v2 is currently maintained by [@TimothyGu](https://github.com/timothygu), v2 readme is written by [@jkantr](https://github.com/jkantr). + ## License MIT diff --git a/package.json b/package.json index 8aa0d64be..db0d97bbe 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "chai": "^3.5.0", "chai-as-promised": "^7.1.1", "chai-iterator": "^1.1.1", - "chai-string": "^1.3.0", + "chai-string": "~1.3.0", "codecov": "^3.0.0", "cross-env": "^5.1.3", "form-data": "^2.3.1", From 1daae67e9e88f7c0a9bd56e38e5d5efe365fd411 Mon Sep 17 00:00:00 2001 From: David Frank Date: Mon, 5 Nov 2018 17:42:51 +0800 Subject: [PATCH 179/223] Fix import style to workaround node < 10 and webpack issues. (#544) * fix import rule for stream PassThrough * avoid named export for compatibility below node 10 * compress flag should not overwrite accept encoding header * doc update * 2.2.1 --- CHANGELOG.md | 6 ++++++ LIMITS.md | 2 ++ README.md | 27 +++++++++++---------------- package.json | 2 +- src/body.js | 6 +++++- src/index.js | 8 ++++++-- src/request.js | 10 ++++++++-- src/response.js | 6 +++++- test/test.js | 19 ++++++++++++++++--- 9 files changed, 60 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f321e0ee9..7971c3c9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ Changelog # 2.x release +## v2.2.1 + +- Fix: `compress` flag shouldn't overwrite existing `Accept-Encoding` header. +- Fix: multiple `import` rules, where `PassThrough` etc. doesn't have a named export when using node <10 and `--exerimental-modules` flag. +- Other: Better README. + ## v2.2.0 - Enhance: Support all `ArrayBuffer` view types diff --git a/LIMITS.md b/LIMITS.md index bdcf66a55..9c4b8c0c8 100644 --- a/LIMITS.md +++ b/LIMITS.md @@ -26,5 +26,7 @@ Known differences - If you are using `res.clone()` and writing an isomorphic app, note that stream on Node.js have a smaller internal buffer size (16Kb, aka `highWaterMark`) from client-side browsers (>1Mb, not consistent across browsers). +- Because node.js stream doesn't expose a [*disturbed*](https://fetch.spec.whatwg.org/#concept-readablestream-disturbed) property like Stream spec, using a consumed stream for `new Response(body)` will not set `bodyUsed` flag correctly. + [readable-stream]: https://nodejs.org/api/stream.html#stream_readable_streams [ERROR-HANDLING.md]: https://github.com/bitinn/node-fetch/blob/master/ERROR-HANDLING.md diff --git a/README.md b/README.md index af7c8dd37..3b20230f3 100644 --- a/README.md +++ b/README.md @@ -183,8 +183,6 @@ fetch('https://assets-cdn.github.com/images/modules/logos_page/Octocat.png') }); ``` -[TODO]: # (Somewhere i think we also should mention arrayBuffer also if you want to be cross-fetch compatible.) - #### Buffer If you prefer to cache binary data in full, use buffer(). (NOTE: buffer() is a `node-fetch` only API) @@ -265,8 +263,6 @@ Perform an HTTP(S) fetch. `url` should be an absolute url, such as `https://example.com/`. A path-relative URL (`/file/under/root`) or protocol-relative URL (`//can-be-http-or-https.com/`) will result in a rejected promise. -[TODO]: # (It might be a good idea to reformat the options section into a table layout, like the headers section, instead of current code block.) - ### Options @@ -285,7 +281,7 @@ The default values are shown after each option key. timeout: 0, // req/res timeout in ms, it resets on redirect. 0 to disable (OS limit applies) compress: true, // support gzip/deflate content encoding. false to disable size: 0, // maximum response body size in bytes. 0 to disable - agent: null // http(s).Agent instance, allows custom proxy, certificate etc. + agent: null // http(s).Agent instance, allows custom proxy, certificate, dns lookup etc. } ``` @@ -293,15 +289,14 @@ The default values are shown after each option key. If no values are set, the following request headers will be sent automatically: -[TODO]: # ("we always said content-length will be "automatically calculated, if possible" in the default header section, but we never explain what's the condition for it to be calculated, and that chunked transfer-encoding will be used when they are not calculated or supplied." - "Maybe also add Transfer-Encoding: chunked? That header is added by Node.js automatically if the input is a stream, I believe.") - -Header | Value ------------------ | -------------------------------------------------------- -`Accept-Encoding` | `gzip,deflate` _(when `options.compress === true`)_ -`Accept` | `*/*` -`Connection` | `close` _(when no `options.agent` is present)_ -`Content-Length` | _(automatically calculated, if possible)_ -`User-Agent` | `node-fetch/1.0 (+https://github.com/bitinn/node-fetch)` +Header | Value +------------------- | -------------------------------------------------------- +`Accept-Encoding` | `gzip,deflate` _(when `options.compress === true`)_ +`Accept` | `*/*` +`Connection` | `close` _(when no `options.agent` is present)_ +`Content-Length` | _(automatically calculated, if possible)_ +`Transfer-Encoding` | `chunked` _(when `req.body` is a stream)_ +`User-Agent` | `node-fetch/1.0 (+https://github.com/bitinn/node-fetch)` ### Class: Request @@ -451,8 +446,6 @@ Consume the body and return a promise that will resolve to one of these formats. Consume the body and return a promise that will resolve to a Buffer. -[TODO]: # (textConverted API should mention an optional dependency on encoding, which users need to install by themselves, and this is done purely for backward compatibility with 1.x release.) - #### body.textConverted() *(node-fetch extension)* @@ -461,6 +454,8 @@ Consume the body and return a promise that will resolve to a Buffer. Identical to `body.text()`, except instead of always converting to UTF-8, encoding sniffing will be performed and text converted to UTF-8, if possible. +(This API requires an optional dependency on npm package [encoding](https://www.npmjs.com/package/encoding), which you need to install manually. `webpack` users may see [a warning message](https://github.com/bitinn/node-fetch/issues/412#issuecomment-379007792) due to this optional dependency.) + ### Class: FetchError diff --git a/package.json b/package.json index db0d97bbe..577d46721 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.2.0", + "version": "2.2.1", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index", "browser": "./browser.js", diff --git a/src/body.js b/src/body.js index c6dd4b63d..d39149f47 100644 --- a/src/body.js +++ b/src/body.js @@ -5,7 +5,8 @@ * Body interface provides common methods for Request and Response */ -import Stream, { PassThrough } from 'stream'; +import Stream from 'stream'; + import Blob, { BUFFER } from './blob.js'; import FetchError from './fetch-error.js'; @@ -14,6 +15,9 @@ try { convert = require('encoding').convert; } catch(e) {} const INTERNALS = Symbol('Body internals'); +// fix an issue where "PassThrough" isn't a named export for node <10 +const PassThrough = Stream.PassThrough; + /** * Body mixin * diff --git a/src/index.js b/src/index.js index fa485e42a..dca44aaf7 100644 --- a/src/index.js +++ b/src/index.js @@ -7,11 +7,11 @@ * All spec algorithm step numbers are based on https://fetch.spec.whatwg.org/commit-snapshots/ae716822cb3a61843226cd090eefc6589446c1d2/. */ -import { resolve as resolve_url } from 'url'; +import Url from 'url'; import http from 'http'; import https from 'https'; import zlib from 'zlib'; -import { PassThrough } from 'stream'; +import Stream from 'stream'; import Body, { writeToStream, getTotalBytes } from './body'; import Response from './response'; @@ -19,6 +19,10 @@ import Headers, { createHeadersLenient } from './headers'; import Request, { getNodeRequestOptions } from './request'; import FetchError from './fetch-error'; +// fix an issue where "PassThrough", "resolve" aren't a named export for node <10 +const PassThrough = Stream.PassThrough; +const resolve_url = Url.resolve; + /** * Fetch function * diff --git a/src/request.js b/src/request.js index 748ba555c..c4d295923 100644 --- a/src/request.js +++ b/src/request.js @@ -7,12 +7,17 @@ * All spec algorithm step numbers are based on https://fetch.spec.whatwg.org/commit-snapshots/ae716822cb3a61843226cd090eefc6589446c1d2/. */ -import { format as format_url, parse as parse_url } from 'url'; +import Url from 'url'; + import Headers, { exportNodeCompatibleHeaders } from './headers.js'; import Body, { clone, extractContentType, getTotalBytes } from './body'; const INTERNALS = Symbol('Request internals'); +// fix an issue where "format", "parse" aren't a named export for node <10 +const parse_url = Url.parse; +const format_url = Url.format; + /** * Check if a value is an instance of Request. * @@ -187,9 +192,10 @@ export function getNodeRequestOptions(request) { } // HTTP-network-or-cache fetch step 2.15 - if (request.compress) { + if (request.compress && !headers.has('Accept-Encoding')) { headers.set('Accept-Encoding', 'gzip,deflate'); } + if (!headers.has('Connection') && !request.agent) { headers.set('Connection', 'close'); } diff --git a/src/response.js b/src/response.js index 506d876fa..ce946f6ba 100644 --- a/src/response.js +++ b/src/response.js @@ -5,12 +5,16 @@ * Response class provides content decoding */ -import { STATUS_CODES } from 'http'; +import http from 'http'; + import Headers from './headers.js'; import Body, { clone } from './body'; const INTERNALS = Symbol('Response internals'); +// fix an issue where "STATUS_CODES" aren't a named export for node <10 +const STATUS_CODES = http.STATUS_CODES; + /** * Response class * diff --git a/test/test.js b/test/test.js index e8342cb56..ef579400e 100644 --- a/test/test.js +++ b/test/test.js @@ -731,6 +731,19 @@ describe('node-fetch', () => { }); }); + it('should not overwrite existing accept-encoding header when auto decompression is true', function() { + const url = `${base}inspect`; + const opts = { + compress: true, + headers: { + 'Accept-Encoding': 'gzip' + } + }; + return fetch(url, opts).then(res => res.json()).then(res => { + expect(res.headers['accept-encoding']).to.equal('gzip'); + }); + }); + it('should allow custom timeout', function() { this.timeout(500); const url = `${base}timeout`; @@ -782,7 +795,7 @@ describe('node-fetch', () => { it('should set default User-Agent', function () { const url = `${base}inspect`; - fetch(url).then(res => res.json()).then(res => { + return fetch(url).then(res => res.json()).then(res => { expect(res.headers['user-agent']).to.startWith('node-fetch/'); }); }); @@ -794,7 +807,7 @@ describe('node-fetch', () => { 'user-agent': 'faked' } }; - fetch(url, opts).then(res => res.json()).then(res => { + return fetch(url, opts).then(res => res.json()).then(res => { expect(res.headers['user-agent']).to.equal('faked'); }); }); @@ -813,7 +826,7 @@ describe('node-fetch', () => { 'accept': 'application/json' } }; - fetch(url, opts).then(res => res.json()).then(res => { + return fetch(url, opts).then(res => res.json()).then(res => { expect(res.headers.accept).to.equal('application/json'); }); }); From ecd3d52c55b26d2d350e2c9e0347675a02b3d2cc Mon Sep 17 00:00:00 2001 From: Joseph Nields Date: Mon, 12 Nov 2018 20:40:11 -0800 Subject: [PATCH 180/223] Add support for AbortSignal to cancel requests (#539) Thx @jnields @FrogTheFrog @TimothyGu for their work! --- ERROR-HANDLING.md | 12 +- README.md | 48 +++++++- package.json | 2 + src/abort-error.js | 25 ++++ src/body.js | 16 ++- src/index.js | 47 +++++++- src/request.js | 40 ++++++- test/server.js | 14 +++ test/test.js | 284 ++++++++++++++++++++++++++++++++++++++++++++- 9 files changed, 469 insertions(+), 19 deletions(-) create mode 100644 src/abort-error.js diff --git a/ERROR-HANDLING.md b/ERROR-HANDLING.md index 7ff8f5464..89d5691c1 100644 --- a/ERROR-HANDLING.md +++ b/ERROR-HANDLING.md @@ -6,7 +6,17 @@ Because `window.fetch` isn't designed to be transparent about the cause of reque The basics: -- All [operational errors][joyent-guide] are rejected with a [FetchError](https://github.com/bitinn/node-fetch/blob/master/README.md#class-fetcherror). You can handle them all through the promise `catch` clause. +- A cancelled request is rejected with an [`AbortError`](https://github.com/bitinn/node-fetch/blob/master/README.md#class-aborterror). You can check if the reason for rejection was that the request was aborted by checking the `Error`'s `name` is `AbortError`. + +```js +fetch(url, { signal }).catch(err => { + if (err.name === 'AbortError') { + // request was aborted + } +}) +``` + +- All [operational errors][joyent-guide] *other than aborted requests* are rejected with a [FetchError](https://github.com/bitinn/node-fetch/blob/master/README.md#class-fetcherror). You can handle them all through the promise `catch` clause. - All errors come with an `err.message` detailing the cause of errors. diff --git a/README.md b/README.md index 3b20230f3..15fd3f771 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ fetch('https://httpbin.org/post', { method: 'POST', body: 'a=1' }) ```js const body = { a: 1 }; -fetch('https://httpbin.org/post', { +fetch('https://httpbin.org/post', { method: 'post', body: JSON.stringify(body), headers: { 'Content-Type': 'application/json' }, @@ -275,16 +275,51 @@ The default values are shown after each option key. headers: {}, // request headers. format is the identical to that accepted by the Headers constructor (see below) body: null, // request body. can be null, a string, a Buffer, a Blob, or a Node.js Readable stream redirect: 'follow', // set to `manual` to extract redirect headers, `error` to reject redirect + signal: null, // pass an instance of AbortSignal to optionally abort requests // The following properties are node-fetch extensions follow: 20, // maximum redirect count. 0 to not follow redirect - timeout: 0, // req/res timeout in ms, it resets on redirect. 0 to disable (OS limit applies) + timeout: 0, // req/res timeout in ms, it resets on redirect. 0 to disable (OS limit applies). Signal is recommended instead. compress: true, // support gzip/deflate content encoding. false to disable size: 0, // maximum response body size in bytes. 0 to disable agent: null // http(s).Agent instance, allows custom proxy, certificate, dns lookup etc. } ``` +#### Request cancellation with AbortController: + +> NOTE: You may only cancel streamed requests on Node >= v8.0.0 + +You may cancel requests with `AbortController`. A suggested implementation is [`abort-controller`](https://www.npmjs.com/package/abort-controller). + +An example of timing out a request after 150ms could be achieved as follows: + +```js +import AbortContoller from 'abort-controller'; + +const controller = new AbortController(); +const timeout = setTimeout( + () => { controller.abort(); }, + 150, +); + +fetch(url, { signal: controller.signal }) + .then(res => res.json()) + .then( + data => { + useData(data) + }, + err => { + if (err.name === 'AbortError') { + // request was aborted + } + }, + ) + .finally(() => { + clearTimeout(timeout); + }); +``` + ##### Default Headers If no values are set, the following request headers will be sent automatically: @@ -463,6 +498,13 @@ Identical to `body.text()`, except instead of always converting to UTF-8, encodi An operational error in the fetching process. See [ERROR-HANDLING.md][] for more info. + +### Class: AbortError + +*(node-fetch extension)* + +An Error thrown when the request is aborted in response to an `AbortSignal`'s `abort` event. It has a `name` property of `AbortError`. See [ERROR-HANDLING.MD][] for more info. + ## Acknowledgement Thanks to [github/fetch](https://github.com/github/fetch) for providing a solid implementation reference. @@ -487,4 +529,4 @@ MIT [mdn-headers]: https://developer.mozilla.org/en-US/docs/Web/API/Headers [LIMITS.md]: https://github.com/bitinn/node-fetch/blob/master/LIMITS.md [ERROR-HANDLING.md]: https://github.com/bitinn/node-fetch/blob/master/ERROR-HANDLING.md -[UPGRADE-GUIDE.md]: https://github.com/bitinn/node-fetch/blob/master/UPGRADE-GUIDE.md \ No newline at end of file +[UPGRADE-GUIDE.md]: https://github.com/bitinn/node-fetch/blob/master/UPGRADE-GUIDE.md diff --git a/package.json b/package.json index 577d46721..a491f2acd 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,8 @@ }, "homepage": "https://github.com/bitinn/node-fetch", "devDependencies": { + "abort-controller": "^1.0.2", + "abortcontroller-polyfill": "^1.1.9", "babel-core": "^6.26.0", "babel-plugin-istanbul": "^4.1.5", "babel-preset-env": "^1.6.1", diff --git a/src/abort-error.js b/src/abort-error.js new file mode 100644 index 000000000..cbb13caba --- /dev/null +++ b/src/abort-error.js @@ -0,0 +1,25 @@ +/** + * abort-error.js + * + * AbortError interface for cancelled requests + */ + +/** + * Create AbortError instance + * + * @param String message Error message for human + * @return AbortError + */ +export default function AbortError(message) { + Error.call(this, message); + + this.type = 'aborted'; + this.message = message; + + // hide custom error implementation details from end-users + Error.captureStackTrace(this, this.constructor); +} + +AbortError.prototype = Object.create(Error.prototype); +AbortError.prototype.constructor = AbortError; +AbortError.prototype.name = 'AbortError'; diff --git a/src/body.js b/src/body.js index d39149f47..6efe52d6d 100644 --- a/src/body.js +++ b/src/body.js @@ -63,7 +63,10 @@ export default function Body(body, { if (body instanceof Stream) { body.on('error', err => { - this[INTERNALS].error = new FetchError(`Invalid response body while trying to fetch ${this.url}: ${err.message}`, 'system', err); + const error = err.name === 'AbortError' + ? err + : new FetchError(`Invalid response body while trying to fetch ${this.url}: ${err.message}`, 'system', err); + this[INTERNALS].error = error; }); } } @@ -240,9 +243,16 @@ function consumeBody() { }, this.timeout); } - // handle stream error, such as incorrect content-encoding + // handle stream errors this.body.on('error', err => { - reject(new FetchError(`Invalid response body while trying to fetch ${this.url}: ${err.message}`, 'system', err)); + if (err.name === 'AbortError') { + // if the request was aborted, reject with this Error + abort = true; + reject(err); + } else { + // other errors, such as incorrect content-encoding + reject(new FetchError(`Invalid response body while trying to fetch ${this.url}: ${err.message}`, 'system', err)); + } }); this.body.on('data', chunk => { diff --git a/src/index.js b/src/index.js index dca44aaf7..3c25c75df 100644 --- a/src/index.js +++ b/src/index.js @@ -18,6 +18,7 @@ import Response from './response'; import Headers, { createHeadersLenient } from './headers'; import Request, { getNodeRequestOptions } from './request'; import FetchError from './fetch-error'; +import AbortError from './abort-error'; // fix an issue where "PassThrough", "resolve" aren't a named export for node <10 const PassThrough = Stream.PassThrough; @@ -46,13 +47,40 @@ export default function fetch(url, opts) { const options = getNodeRequestOptions(request); const send = (options.protocol === 'https:' ? https : http).request; + const { signal } = request; + let response = null; + + const abort = () => { + let error = new AbortError('The user aborted a request.'); + reject(error); + if (request.body && request.body instanceof Stream.Readable) { + request.body.destroy(error); + } + if (!response || !response.body) return; + response.body.emit('error', error); + } + + if (signal && signal.aborted) { + abort(); + return; + } + + const abortAndFinalize = () => { + abort(); + finalize(); + } // send request const req = send(options); let reqTimeout; + if (signal) { + signal.addEventListener('abort', abortAndFinalize); + } + function finalize() { req.abort(); + if (signal) signal.removeEventListener('abort', abortAndFinalize); clearTimeout(reqTimeout); } @@ -117,7 +145,8 @@ export default function fetch(url, opts) { agent: request.agent, compress: request.compress, method: request.method, - body: request.body + body: request.body, + signal: request.signal, }; // HTTP-redirect fetch step 9 @@ -142,7 +171,11 @@ export default function fetch(url, opts) { } // prepare response + res.once('end', () => { + if (signal) signal.removeEventListener('abort', abortAndFinalize); + }); let body = res.pipe(new PassThrough()); + const response_options = { url: request.url, status: res.statusCode, @@ -164,7 +197,8 @@ export default function fetch(url, opts) { // 4. no content response (204) // 5. content not modified response (304) if (!request.compress || request.method === 'HEAD' || codings === null || res.statusCode === 204 || res.statusCode === 304) { - resolve(new Response(body, response_options)); + response = new Response(body, response_options); + resolve(response); return; } @@ -181,7 +215,8 @@ export default function fetch(url, opts) { // for gzip if (codings == 'gzip' || codings == 'x-gzip') { body = body.pipe(zlib.createGunzip(zlibOptions)); - resolve(new Response(body, response_options)); + response = new Response(body, response_options); + resolve(response); return; } @@ -197,13 +232,15 @@ export default function fetch(url, opts) { } else { body = body.pipe(zlib.createInflateRaw()); } - resolve(new Response(body, response_options)); + response = new Response(body, response_options); + resolve(response); }); return; } // otherwise, use response as-is - resolve(new Response(body, response_options)); + response = new Response(body, response_options); + resolve(response); }); writeToStream(req, request); diff --git a/src/request.js b/src/request.js index c4d295923..5023aa502 100644 --- a/src/request.js +++ b/src/request.js @@ -8,7 +8,7 @@ */ import Url from 'url'; - +import Stream from 'stream'; import Headers, { exportNodeCompatibleHeaders } from './headers.js'; import Body, { clone, extractContentType, getTotalBytes } from './body'; @@ -18,6 +18,8 @@ const INTERNALS = Symbol('Request internals'); const parse_url = Url.parse; const format_url = Url.format; +const streamDestructionSupported = 'destroy' in Stream.Readable.prototype; + /** * Check if a value is an instance of Request. * @@ -31,6 +33,15 @@ function isRequest(input) { ); } +function isAbortSignal(signal) { + const proto = ( + signal + && typeof signal === 'object' + && Object.getPrototypeOf(signal) + ); + return !!(proto && proto.constructor.name === 'AbortSignal'); +} + /** * Request class * @@ -86,11 +97,21 @@ export default class Request { } } + let signal = isRequest(input) + ? input.signal + : null; + if ('signal' in init) signal = init.signal + + if (signal != null && !isAbortSignal(signal)) { + throw new TypeError('Expected signal to be an instanceof AbortSignal'); + } + this[INTERNALS] = { method, redirect: init.redirect || input.redirect || 'follow', headers, - parsedURL + parsedURL, + signal, }; // node-fetch-only options @@ -120,6 +141,10 @@ export default class Request { return this[INTERNALS].redirect; } + get signal() { + return this[INTERNALS].signal; + } + /** * Clone this request * @@ -144,7 +169,8 @@ Object.defineProperties(Request.prototype, { url: { enumerable: true }, headers: { enumerable: true }, redirect: { enumerable: true }, - clone: { enumerable: true } + clone: { enumerable: true }, + signal: { enumerable: true }, }); /** @@ -171,6 +197,14 @@ export function getNodeRequestOptions(request) { throw new TypeError('Only HTTP(S) protocols are supported'); } + if ( + request.signal + && request.body instanceof Stream.Readable + && !streamDestructionSupported + ) { + throw new Error('Cancellation of streamed requests with AbortSignal is not supported in node < 8'); + } + // HTTP-network-or-cache fetch steps 2.4-2.7 let contentLengthValue = null; if (request.body == null && /^(POST|PUT)$/i.test(request.method)) { diff --git a/test/server.js b/test/server.js index 6641b1657..4028f0cc4 100644 --- a/test/server.js +++ b/test/server.js @@ -269,6 +269,20 @@ export default class TestServer { res.end(); } + if (p === '/redirect/slow') { + res.statusCode = 301; + res.setHeader('Location', '/redirect/301'); + setTimeout(function() { + res.end(); + }, 1000); + } + + if (p === '/redirect/slow-stream') { + res.statusCode = 301; + res.setHeader('Location', '/slow'); + res.end(); + } + if (p === '/error/400') { res.statusCode = 400; res.setHeader('Content-Type', 'text/plain'); diff --git a/test/test.js b/test/test.js index ef579400e..e9b5db241 100644 --- a/test/test.js +++ b/test/test.js @@ -10,6 +10,8 @@ import FormData from 'form-data'; import stringToArrayBuffer from 'string-to-arraybuffer'; import URLSearchParams_Polyfill from 'url-search-params'; import { URL } from 'whatwg-url'; +import { AbortController } from 'abortcontroller-polyfill/dist/abortcontroller'; +import AbortController2 from 'abort-controller'; const { spawn } = require('child_process'); const http = require('http'); @@ -53,6 +55,8 @@ const supportToString = ({ [Symbol.toStringTag]: 'z' }).toString() === '[object z]'; +const supportStreamDestroy = 'destroy' in stream.Readable.prototype; + const local = new TestServer(); const base = `http://${local.hostname}:${local.port}/`; @@ -793,6 +797,247 @@ describe('node-fetch', () => { }); }); + it('should support request cancellation with signal', function () { + this.timeout(500); + const controller = new AbortController(); + const controller2 = new AbortController2(); + + const fetches = [ + fetch(`${base}timeout`, { signal: controller.signal }), + fetch(`${base}timeout`, { signal: controller2.signal }), + fetch( + `${base}timeout`, + { + method: 'POST', + signal: controller.signal, + headers: { + 'Content-Type': 'application/json', + body: JSON.stringify({ hello: 'world' }) + } + } + ) + ]; + setTimeout(() => { + controller.abort(); + controller2.abort(); + }, 100); + + return Promise.all(fetches.map(fetched => expect(fetched) + .to.eventually.be.rejected + .and.be.an.instanceOf(Error) + .and.include({ + type: 'aborted', + name: 'AbortError', + }) + )); + }); + + it('should reject immediately if signal has already been aborted', function () { + const url = `${base}timeout`; + const controller = new AbortController(); + const opts = { + signal: controller.signal + }; + controller.abort(); + const fetched = fetch(url, opts); + return expect(fetched).to.eventually.be.rejected + .and.be.an.instanceOf(Error) + .and.include({ + type: 'aborted', + name: 'AbortError', + }); + }); + + it('should clear internal timeout when request is cancelled with an AbortSignal', function(done) { + this.timeout(2000); + const script = ` + var AbortController = require('abortcontroller-polyfill/dist/cjs-ponyfill').AbortController; + var controller = new AbortController(); + require('./')( + '${base}timeout', + { signal: controller.signal, timeout: 10000 } + ); + setTimeout(function () { controller.abort(); }, 100); + ` + spawn('node', ['-e', script]) + .on('exit', () => { + done(); + }); + }); + + it('should remove internal AbortSignal event listener after request is aborted', function () { + const controller = new AbortController(); + const { signal } = controller; + const promise = fetch( + `${base}timeout`, + { signal } + ); + const result = expect(promise).to.eventually.be.rejected + .and.be.an.instanceof(Error) + .and.have.property('name', 'AbortError') + .then(() => { + expect(signal.listeners.abort.length).to.equal(0); + }); + controller.abort(); + return result; + }); + + it('should allow redirects to be aborted', function() { + const abortController = new AbortController(); + const request = new Request(`${base}redirect/slow`, { + signal: abortController.signal + }); + setTimeout(() => { + abortController.abort(); + }, 50); + return expect(fetch(request)).to.be.eventually.rejected + .and.be.an.instanceOf(Error) + .and.have.property('name', 'AbortError'); + }); + + it('should allow redirected response body to be aborted', function() { + const abortController = new AbortController(); + const request = new Request(`${base}redirect/slow-stream`, { + signal: abortController.signal + }); + return expect(fetch(request).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain'); + const result = res.text(); + abortController.abort(); + return result; + })).to.be.eventually.rejected + .and.be.an.instanceOf(Error) + .and.have.property('name', 'AbortError'); + }); + + it('should remove internal AbortSignal event listener after request and response complete without aborting', () => { + const controller = new AbortController(); + const { signal } = controller; + const fetchHtml = fetch(`${base}html`, { signal }) + .then(res => res.text()); + const fetchResponseError = fetch(`${base}error/reset`, { signal }); + const fetchRedirect = fetch(`${base}redirect/301`, { signal }).then(res => res.json()); + return Promise.all([ + expect(fetchHtml).to.eventually.be.fulfilled.and.equal(''), + expect(fetchResponseError).to.be.eventually.rejected, + expect(fetchRedirect).to.eventually.be.fulfilled, + ]).then(() => { + expect(signal.listeners.abort.length).to.equal(0) + }); + }); + + it('should reject response body with AbortError when aborted before stream has been read completely', () => { + const controller = new AbortController(); + return expect(fetch( + `${base}slow`, + { signal: controller.signal } + )) + .to.eventually.be.fulfilled + .then((res) => { + const promise = res.text(); + controller.abort(); + return expect(promise) + .to.eventually.be.rejected + .and.be.an.instanceof(Error) + .and.have.property('name', 'AbortError'); + }); + }); + + it('should reject response body methods immediately with AbortError when aborted before stream is disturbed', () => { + const controller = new AbortController(); + return expect(fetch( + `${base}slow`, + { signal: controller.signal } + )) + .to.eventually.be.fulfilled + .then((res) => { + controller.abort(); + return expect(res.text()) + .to.eventually.be.rejected + .and.be.an.instanceof(Error) + .and.have.property('name', 'AbortError'); + }); + }); + + it('should emit error event to response body with an AbortError when aborted before underlying stream is closed', (done) => { + const controller = new AbortController(); + expect(fetch( + `${base}slow`, + { signal: controller.signal } + )) + .to.eventually.be.fulfilled + .then((res) => { + res.body.on('error', (err) => { + expect(err) + .to.be.an.instanceof(Error) + .and.have.property('name', 'AbortError'); + done(); + }); + controller.abort(); + }); + }); + + (supportStreamDestroy ? it : it.skip)('should cancel request body of type Stream with AbortError when aborted', () => { + const controller = new AbortController(); + const body = new stream.Readable({ objectMode: true }); + body._read = () => {}; + const promise = fetch( + `${base}slow`, + { signal: controller.signal, body, method: 'POST' } + ); + + const result = Promise.all([ + new Promise((resolve, reject) => { + body.on('error', (error) => { + try { + expect(error).to.be.an.instanceof(Error).and.have.property('name', 'AbortError') + resolve(); + } catch (err) { + reject(err); + } + }); + }), + expect(promise).to.eventually.be.rejected + .and.be.an.instanceof(Error) + .and.have.property('name', 'AbortError') + ]); + + controller.abort(); + + return result; + }); + + (supportStreamDestroy ? it.skip : it)('should immediately reject when attempting to cancel streamed Requests in node < 8', () => { + const controller = new AbortController(); + const body = new stream.Readable({ objectMode: true }); + body._read = () => {}; + const promise = fetch( + `${base}slow`, + { signal: controller.signal, body, method: 'POST' } + ); + + return expect(promise).to.eventually.be.rejected + .and.be.an.instanceof(Error) + .and.have.property('message').includes('not supported'); + }); + + it('should throw a TypeError if a signal is not of type AbortSignal', () => { + return Promise.all([ + expect(fetch(`${base}inspect`, { signal: {} })) + .to.be.eventually.rejected + .and.be.an.instanceof(TypeError) + .and.have.property('message').includes('AbortSignal'), + expect(fetch(`${base}inspect`, { signal: '' })) + .to.be.eventually.rejected + .and.be.an.instanceof(TypeError) + .and.have.property('message').includes('AbortSignal'), + expect(fetch(`${base}inspect`, { signal: Object.create(null) })) + .to.be.eventually.rejected + .and.be.an.instanceof(TypeError) + .and.have.property('message').includes('AbortSignal'), + ]); + }); + it('should set default User-Agent', function () { const url = `${base}inspect`; return fetch(url).then(res => res.json()).then(res => { @@ -2016,12 +2261,12 @@ describe('Request', function () { } for (const toCheck of [ 'body', 'bodyUsed', 'arrayBuffer', 'blob', 'json', 'text', - 'method', 'url', 'headers', 'redirect', 'clone' + 'method', 'url', 'headers', 'redirect', 'clone', 'signal', ]) { expect(enumerableProperties).to.contain(toCheck); } for (const toCheck of [ - 'body', 'bodyUsed', 'method', 'url', 'headers', 'redirect' + 'body', 'bodyUsed', 'method', 'url', 'headers', 'redirect', 'signal', ]) { expect(() => { req[toCheck] = 'abc'; @@ -2034,11 +2279,13 @@ describe('Request', function () { const form = new FormData(); form.append('a', '1'); + const { signal } = new AbortController(); const r1 = new Request(url, { method: 'POST', follow: 1, - body: form + body: form, + signal, }); const r2 = new Request(r1, { follow: 2 @@ -2046,6 +2293,7 @@ describe('Request', function () { expect(r2.url).to.equal(url); expect(r2.method).to.equal('POST'); + expect(r2.signal).to.equal(signal); // note that we didn't clone the body expect(r2.body).to.equal(form); expect(r1.follow).to.equal(1); @@ -2054,6 +2302,31 @@ describe('Request', function () { expect(r2.counter).to.equal(0); }); + it('should override signal on derived Request instances', function() { + const parentAbortController = new AbortController(); + const derivedAbortController = new AbortController(); + const parentRequest = new Request(`test`, { + signal: parentAbortController.signal + }); + const derivedRequest = new Request(parentRequest, { + signal: derivedAbortController.signal + }); + expect(parentRequest.signal).to.equal(parentAbortController.signal); + expect(derivedRequest.signal).to.equal(derivedAbortController.signal); + }); + + it('should allow removing signal on derived Request instances', function() { + const parentAbortController = new AbortController(); + const parentRequest = new Request(`test`, { + signal: parentAbortController.signal + }); + const derivedRequest = new Request(parentRequest, { + signal: null + }); + expect(parentRequest.signal).to.equal(parentAbortController.signal); + expect(derivedRequest.signal).to.equal(null); + }); + it('should throw error with GET/HEAD requests with body', function() { expect(() => new Request('.', { body: '' })) .to.throw(TypeError); @@ -2161,6 +2434,7 @@ describe('Request', function () { let body = resumer().queue('a=1').end(); body = body.pipe(new stream.PassThrough()); const agent = new http.Agent(); + const { signal } = new AbortController(); const req = new Request(url, { body, method: 'POST', @@ -2170,7 +2444,8 @@ describe('Request', function () { }, follow: 3, compress: false, - agent + agent, + signal, }); const cl = req.clone(); expect(cl.url).to.equal(url); @@ -2182,6 +2457,7 @@ describe('Request', function () { expect(cl.method).to.equal('POST'); expect(cl.counter).to.equal(0); expect(cl.agent).to.equal(agent); + expect(cl.signal).to.equal(signal); // clone body shouldn't be the same body expect(cl.body).to.not.equal(body); return Promise.all([cl.text(), req.text()]).then(results => { From d1ca2dfbb97247d57a8be934edcd7f2dabed73fe Mon Sep 17 00:00:00 2001 From: David Frank Date: Tue, 13 Nov 2018 12:43:27 +0800 Subject: [PATCH 181/223] Workaround lack of global context in react-native (#545) --- CHANGELOG.md | 4 ++++ browser.js | 23 ++++++++++++++++++----- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7971c3c9a..4452b8dde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ Changelog # 2.x release +## master + +- Fix: update `browser.js` to support react-native, where `self` isn't available. + ## v2.2.1 - Fix: `compress` flag shouldn't overwrite existing `Accept-Encoding` header. diff --git a/browser.js b/browser.js index 3ed04dede..0ad5de004 100644 --- a/browser.js +++ b/browser.js @@ -1,10 +1,23 @@ "use strict"; -module.exports = exports = self.fetch; +// ref: https://github.com/tc39/proposal-global +var getGlobal = function () { + // the only reliable means to get the global object is + // `Function('return this')()` + // However, this causes CSP violations in Chrome apps. + if (typeof self !== 'undefined') { return self; } + if (typeof window !== 'undefined') { return window; } + if (typeof global !== 'undefined') { return global; } + throw new Error('unable to locate global object'); +} + +var global = getGlobal(); + +module.exports = exports = global.fetch; // Needed for TypeScript and Webpack. -exports.default = self.fetch.bind(self); +exports.default = global.fetch.bind(global); -exports.Headers = self.Headers; -exports.Request = self.Request; -exports.Response = self.Response; +exports.Headers = global.Headers; +exports.Request = global.Request; +exports.Response = global.Response; \ No newline at end of file From 5367fe6a978e01745e4264384a91140dc99a4bf8 Mon Sep 17 00:00:00 2001 From: David Frank Date: Tue, 13 Nov 2018 14:35:09 +0800 Subject: [PATCH 182/223] v2.3.0 (#548) * doc update * handle corrupted location header during redirect --- CHANGELOG.md | 6 +++-- README.md | 69 ++++++++++++++++++++++++++-------------------------- package.json | 2 +- src/index.js | 8 +++++- 4 files changed, 47 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4452b8dde..85b2e2ad0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,9 +5,11 @@ Changelog # 2.x release -## master +## v2.3.0 -- Fix: update `browser.js` to support react-native, where `self` isn't available. +- New: `AbortSignal` support, with README example. +- Enhance: handle invalid `Location` header during redirect by rejecting them explicitly with `FetchError`. +- Fix: update `browser.js` to support react-native environment, where `self` isn't available globally. ## v2.2.1 diff --git a/README.md b/README.md index 15fd3f771..b8b3926c8 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ A light-weight module that brings `window.fetch` to Node.js - [Accessing Headers and other Meta data](#accessing-headers-and-other-meta-data) - [Post data using a file stream](#post-data-using-a-file-stream) - [Post with form-data (detect multipart)](#post-with-form-data-detect-multipart) + - [Request cancellation with AbortSignal](#request-cancellation-with-abortsignal) - [API](#api) - [fetch(url[, options])](#fetchurl-options) - [Options](#options) @@ -248,6 +249,40 @@ fetch('https://httpbin.org/post', options) .then(json => console.log(json)); ``` +#### Request cancellation with AbortSignal + +> NOTE: You may only cancel streamed requests on Node >= v8.0.0 + +You may cancel requests with `AbortController`. A suggested implementation is [`abort-controller`](https://www.npmjs.com/package/abort-controller). + +An example of timing out a request after 150ms could be achieved as follows: + +```js +import AbortContoller from 'abort-controller'; + +const controller = new AbortController(); +const timeout = setTimeout( + () => { controller.abort(); }, + 150, +); + +fetch(url, { signal: controller.signal }) + .then(res => res.json()) + .then( + data => { + useData(data) + }, + err => { + if (err.name === 'AbortError') { + // request was aborted + } + }, + ) + .finally(() => { + clearTimeout(timeout); + }); +``` + See [test cases](https://github.com/bitinn/node-fetch/blob/master/test/test.js) for more examples. @@ -286,40 +321,6 @@ The default values are shown after each option key. } ``` -#### Request cancellation with AbortController: - -> NOTE: You may only cancel streamed requests on Node >= v8.0.0 - -You may cancel requests with `AbortController`. A suggested implementation is [`abort-controller`](https://www.npmjs.com/package/abort-controller). - -An example of timing out a request after 150ms could be achieved as follows: - -```js -import AbortContoller from 'abort-controller'; - -const controller = new AbortController(); -const timeout = setTimeout( - () => { controller.abort(); }, - 150, -); - -fetch(url, { signal: controller.signal }) - .then(res => res.json()) - .then( - data => { - useData(data) - }, - err => { - if (err.name === 'AbortError') { - // request was aborted - } - }, - ) - .finally(() => { - clearTimeout(timeout); - }); -``` - ##### Default Headers If no values are set, the following request headers will be sent automatically: diff --git a/package.json b/package.json index a491f2acd..3d591b0e1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.2.1", + "version": "2.3.0", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index", "browser": "./browser.js", diff --git a/src/index.js b/src/index.js index 3c25c75df..e22799840 100644 --- a/src/index.js +++ b/src/index.js @@ -120,7 +120,13 @@ export default function fetch(url, opts) { case 'manual': // node-fetch-specific step: make manual redirect a bit easier to use by setting the Location header value to the resolved URL. if (locationURL !== null) { - headers.set('Location', locationURL); + // handle corrupted header + try { + headers.set('Location', locationURL); + } catch (err) { + // istanbul ignore next: nodejs server prevent invalid response headers, we can't test this through normal request + reject(err); + } } break; case 'follow': From 2d0fc689c63c67dddb21adabf4e68f8e861b1d53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jimmy=20W=C3=A4rting?= Date: Tue, 13 Nov 2018 17:36:44 +0100 Subject: [PATCH 183/223] Clone URLSearchParams to avoid mutation (#547) * And make sure Request/Response set Content-Type per Fetch Spec * And make sure users can read the body as string via text() --- src/body.js | 10 ++++++---- src/request.js | 6 +++--- src/response.js | 12 ++++++++++-- test/test.js | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 51 insertions(+), 9 deletions(-) diff --git a/src/body.js b/src/body.js index 6efe52d6d..d9ef9218f 100644 --- a/src/body.js +++ b/src/body.js @@ -37,6 +37,7 @@ export default function Body(body, { } else if (typeof body === 'string') { // body is string } else if (isURLSearchParams(body)) { + body = Buffer.from(body.toString()); // body is a URLSearchParams } else if (body instanceof Blob) { // body is blob @@ -415,9 +416,7 @@ export function clone(instance) { * * @param Mixed instance Response or Request instance */ -export function extractContentType(instance) { - const {body} = instance; - +export function extractContentType(body) { // istanbul ignore if: Currently, because of a guard in Request, body // can never be null. Included here for completeness. if (body === null) { @@ -444,10 +443,13 @@ export function extractContentType(instance) { } else if (typeof body.getBoundary === 'function') { // detect form data input from form-data module return `multipart/form-data;boundary=${body.getBoundary()}`; - } else { + } else if (body instanceof Stream) { // body is stream // can't really do much about this return null; + } else { + // Body constructor defaults other things to string + return 'text/plain;charset=UTF-8'; } } diff --git a/src/request.js b/src/request.js index 5023aa502..99aef257f 100644 --- a/src/request.js +++ b/src/request.js @@ -90,9 +90,9 @@ export default class Request { const headers = new Headers(init.headers || input.headers || {}); - if (init.body != null) { - const contentType = extractContentType(this); - if (contentType !== null && !headers.has('Content-Type')) { + if (inputBody != null && !headers.has('Content-Type')) { + const contentType = extractContentType(inputBody); + if (contentType) { headers.append('Content-Type', contentType); } } diff --git a/src/response.js b/src/response.js index ce946f6ba..f29bfe296 100644 --- a/src/response.js +++ b/src/response.js @@ -8,7 +8,7 @@ import http from 'http'; import Headers from './headers.js'; -import Body, { clone } from './body'; +import Body, { clone, extractContentType } from './body'; const INTERNALS = Symbol('Response internals'); @@ -27,12 +27,20 @@ export default class Response { Body.call(this, body, opts); const status = opts.status || 200; + const headers = new Headers(opts.headers) + + if (body != null && !headers.has('Content-Type')) { + const contentType = extractContentType(body); + if (contentType) { + headers.append('Content-Type', contentType); + } + } this[INTERNALS] = { url: opts.url, status, statusText: opts.statusText || STATUS_CODES[status], - headers: new Headers(opts.headers) + headers }; } diff --git a/test/test.js b/test/test.js index e9b5db241..aebb955e4 100644 --- a/test/test.js +++ b/test/test.js @@ -1365,6 +1365,38 @@ describe('node-fetch', () => { }); const itUSP = typeof URLSearchParams === 'function' ? it : it.skip; + + itUSP('constructing a Response with URLSearchParams as body should have a Content-Type', function() { + const params = new URLSearchParams(); + const res = new Response(params); + res.headers.get('Content-Type'); + expect(res.headers.get('Content-Type')).to.equal('application/x-www-form-urlencoded;charset=UTF-8'); + }); + + itUSP('constructing a Request with URLSearchParams as body should have a Content-Type', function() { + const params = new URLSearchParams(); + const req = new Request(base, { method: 'POST', body: params }); + expect(req.headers.get('Content-Type')).to.equal('application/x-www-form-urlencoded;charset=UTF-8'); + }); + + itUSP('Reading a body with URLSearchParams should echo back the result', function() { + const params = new URLSearchParams(); + params.append('a','1'); + return new Response(params).text().then(text => { + expect(text).to.equal('a=1'); + }); + }); + + // Body should been cloned... + itUSP('constructing a Request/Response with URLSearchParams and mutating it should not affected body', function() { + const params = new URLSearchParams(); + const req = new Request(`${base}inspect`, { method: 'POST', body: params }) + params.append('a','1') + return req.text().then(text => { + expect(text).to.equal(''); + }); + }); + itUSP('should allow POST request with URLSearchParams as body', function() { const params = new URLSearchParams(); params.append('a','1'); From 35a4abe825750a31c9cf4d93b3545479e208ea6f Mon Sep 17 00:00:00 2001 From: Jonathan Puckey Date: Thu, 15 Nov 2018 04:38:19 +0100 Subject: [PATCH 184/223] Fix spelling mistake (#551) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b8b3926c8..f3b3f9678 100644 --- a/README.md +++ b/README.md @@ -258,7 +258,7 @@ You may cancel requests with `AbortController`. A suggested implementation is [` An example of timing out a request after 150ms could be achieved as follows: ```js -import AbortContoller from 'abort-controller'; +import AbortController from 'abort-controller'; const controller = new AbortController(); const timeout = setTimeout( From 7d3293200a91ad52b5ca7962f9d6fd1c04983edb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jimmy=20W=C3=A4rting?= Date: Thu, 15 Nov 2018 15:50:32 +0100 Subject: [PATCH 185/223] Unify internal body as buffer (#552) --- src/body.js | 72 ++++++----------------------------------------------- 1 file changed, 8 insertions(+), 64 deletions(-) diff --git a/src/body.js b/src/body.js index d9ef9218f..90cbcabfa 100644 --- a/src/body.js +++ b/src/body.js @@ -34,25 +34,26 @@ export default function Body(body, { if (body == null) { // body is undefined or null body = null; - } else if (typeof body === 'string') { - // body is string } else if (isURLSearchParams(body)) { - body = Buffer.from(body.toString()); // body is a URLSearchParams + body = Buffer.from(body.toString()); } else if (body instanceof Blob) { // body is blob + body = body[BUFFER]; } else if (Buffer.isBuffer(body)) { // body is Buffer } else if (Object.prototype.toString.call(body) === '[object ArrayBuffer]') { // body is ArrayBuffer + body = Buffer.from(body); } else if (ArrayBuffer.isView(body)) { // body is ArrayBufferView + body = Buffer.from(body.buffer, body.byteOffset, body.byteLength); } else if (body instanceof Stream) { // body is stream } else { // none of the above - // coerce to string - body = String(body); + // coerce to string then buffer + body = Buffer.from(String(body)); } this[INTERNALS] = { body, @@ -149,9 +150,7 @@ Body.prototype = { */ textConverted() { return consumeBody.call(this).then(buffer => convertBody(buffer, this.headers)); - }, - - + } }; // In browsers, all properties are enumerable. @@ -197,31 +196,11 @@ function consumeBody() { return Body.Promise.resolve(Buffer.alloc(0)); } - // body is string - if (typeof this.body === 'string') { - return Body.Promise.resolve(Buffer.from(this.body)); - } - - // body is blob - if (this.body instanceof Blob) { - return Body.Promise.resolve(this.body[BUFFER]); - } - // body is buffer if (Buffer.isBuffer(this.body)) { return Body.Promise.resolve(this.body); } - // body is ArrayBuffer - if (Object.prototype.toString.call(this.body) === '[object ArrayBuffer]') { - return Body.Promise.resolve(Buffer.from(this.body)); - } - - // body is ArrayBufferView - if (ArrayBuffer.isView(this.body)) { - return Body.Promise.resolve(Buffer.from(this.body.buffer, this.body.byteOffset, this.body.byteLength)); - } - // istanbul ignore if: should never happen if (!(this.body instanceof Stream)) { return Body.Promise.resolve(Buffer.alloc(0)); @@ -426,7 +405,7 @@ export function extractContentType(body) { // body is string return 'text/plain;charset=UTF-8'; } else if (isURLSearchParams(body)) { - // body is a URLSearchParams + // body is a URLSearchParams return 'application/x-www-form-urlencoded;charset=UTF-8'; } else if (body instanceof Blob) { // body is blob @@ -469,24 +448,9 @@ export function getTotalBytes(instance) { if (body === null) { // body is null return 0; - } else if (typeof body === 'string') { - // body is string - return Buffer.byteLength(body); - } else if (isURLSearchParams(body)) { - // body is URLSearchParams - return Buffer.byteLength(String(body)); - } else if (body instanceof Blob) { - // body is blob - return body.size; } else if (Buffer.isBuffer(body)) { // body is buffer return body.length; - } else if (Object.prototype.toString.call(body) === '[object ArrayBuffer]') { - // body is ArrayBuffer - return body.byteLength; - } else if (ArrayBuffer.isView(body)) { - // body is ArrayBufferView - return body.byteLength; } else if (body && typeof body.getLengthSync === 'function') { // detect form data input from form-data module if (body._lengthRetrievers && body._lengthRetrievers.length == 0 || // 1.x @@ -513,30 +477,10 @@ export function writeToStream(dest, instance) { if (body === null) { // body is null dest.end(); - } else if (typeof body === 'string') { - // body is string - dest.write(body); - dest.end(); - } else if (isURLSearchParams(body)) { - // body is URLSearchParams - dest.write(Buffer.from(String(body))); - dest.end(); - } else if (body instanceof Blob) { - // body is blob - dest.write(body[BUFFER]); - dest.end(); } else if (Buffer.isBuffer(body)) { // body is buffer dest.write(body); dest.end() - } else if (Object.prototype.toString.call(body) === '[object ArrayBuffer]') { - // body is ArrayBuffer - dest.write(Buffer.from(body)); - dest.end() - } else if (ArrayBuffer.isView(body)) { - // body is ArrayBufferView - dest.write(Buffer.from(body.buffer, body.byteOffset, body.byteLength)); - dest.end() } else { // body is stream body.pipe(dest); From 1c2f07ffb84fc3713f7c168a797e95d370f89c2d Mon Sep 17 00:00:00 2001 From: "Kevin (Kun) \"Kassimo\" Qian" Date: Sat, 29 Dec 2018 04:04:44 -0500 Subject: [PATCH 186/223] Headers should not accept empty field name (#562) --- src/headers.js | 2 +- test/test.js | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/headers.js b/src/headers.js index 6b75371e8..f449cb1a0 100644 --- a/src/headers.js +++ b/src/headers.js @@ -10,7 +10,7 @@ const invalidHeaderCharRegex = /[^\t\x20-\x7e\x80-\xff]/; function validateName(name) { name = `${name}`; - if (invalidTokenRegex.test(name)) { + if (invalidTokenRegex.test(name) || name === '') { throw new TypeError(`${name} is not a legal HTTP header name`); } } diff --git a/test/test.js b/test/test.js index aebb955e4..78301a4b7 100644 --- a/test/test.js +++ b/test/test.js @@ -2008,6 +2008,8 @@ describe('Headers', function () { expect(() => headers.get('Hé-y')) .to.throw(TypeError); expect(() => headers.has('Hé-y')) .to.throw(TypeError); expect(() => headers.set('Hé-y', 'ok')) .to.throw(TypeError); + // should reject empty header + expect(() => headers.append('', 'ok')) .to.throw(TypeError); // 'o k' is valid value but invalid name new Headers({ 'He-y': 'o k' }); From e996bdab73baf996cf2dbf25643c8fe2698c3249 Mon Sep 17 00:00:00 2001 From: David Frank Date: Wed, 16 Jan 2019 14:43:24 +0800 Subject: [PATCH 187/223] Quick readme update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f3b3f9678..e900c9719 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ node-fetch A light-weight module that brings `window.fetch` to Node.js -(We are looking for [v2 maintainers and collaborators](https://github.com/bitinn/node-fetch/issues/252)) +(We are looking for [v2 maintainers and collaborators](https://github.com/bitinn/node-fetch/issues/567)) From bee2ad8db7900654c5a4edc561d58d1660601c97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jimmy=20W=C3=A4rting?= Date: Mon, 15 Apr 2019 22:44:07 +0200 Subject: [PATCH 188/223] ignore buffers recalculation --- src/blob.js | 2 ++ src/body.js | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/blob.js b/src/blob.js index b5fa64565..e62fecd3f 100644 --- a/src/blob.js +++ b/src/blob.js @@ -12,6 +12,7 @@ export default class Blob { const options = arguments[1]; const buffers = []; + let size = 0; if (blobParts) { const a = blobParts; @@ -30,6 +31,7 @@ export default class Blob { } else { buffer = Buffer.from(typeof element === 'string' ? element : String(element)); } + size += buffer.length; buffers.push(buffer); } } diff --git a/src/body.js b/src/body.js index 90cbcabfa..c70b41052 100644 --- a/src/body.js +++ b/src/body.js @@ -258,7 +258,7 @@ function consumeBody() { clearTimeout(resTimeout); try { - resolve(Buffer.concat(accum)); + resolve(Buffer.concat(accum, accumBytes)); } catch (err) { // handle streams that have accumulated too much data (issue #414) reject(new FetchError(`Could not create Buffer from response body for ${this.url}: ${err.message}`, 'system', err)); From 0ad136d49f5a93dd0e7284bc42f24995e235b64f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jimmy=20W=C3=A4rting?= Date: Mon, 15 Apr 2019 22:46:11 +0200 Subject: [PATCH 189/223] Added new reading method to blob --- src/blob.js | 15 +++++++++++++++ test/test.js | 19 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/src/blob.js b/src/blob.js index e62fecd3f..284fadfe0 100644 --- a/src/blob.js +++ b/src/blob.js @@ -49,6 +49,21 @@ export default class Blob { get type() { return this[TYPE]; } + text() { + return Promise.resolve(this[BUFFER].toString()) + } + arrayBuffer() { + const buf = this[BUFFER]; + const ab = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength); + return Promise.resolve(ab); + } + // stream() { + // const readable = new Readable() + // readable._read = () => {} + // readable.push(this[BUFFER]) + // readable.push(null) + // return readable || whatwg stream? not decided + // } slice() { const size = this.size; diff --git a/test/test.js b/test/test.js index 78301a4b7..22b2a9173 100644 --- a/test/test.js +++ b/test/test.js @@ -1773,6 +1773,25 @@ describe('node-fetch', () => { }); }); + it('should support reading blob as text', function() { + return new Response(`hello`) + .blob() + .then(blob => blob.text()) + .then(body => { + expect(body).to.equal('hello'); + }) + }) + + it('should support reading blob as arrayBuffer', function() { + return new Response(`hello`) + .blob() + .then(blob => blob.arrayBuffer()) + .then(ab => { + const str = String.fromCharCode.apply(null, new Uint8Array(ab)); + expect(str).to.equal('hello'); + }) + }) + it('should support blob round-trip', function() { const url = `${base}hello`; From 432c9b01ea71cdf0513258ed128688437796e54a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jimmy=20W=C3=A4rting?= Date: Tue, 16 Apr 2019 12:29:17 +0200 Subject: [PATCH 190/223] support reading blob with stream (#608) --- src/blob.js | 19 ++++++++++++------- test/test.js | 17 +++++++++++++---- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/blob.js b/src/blob.js index 284fadfe0..e03a6d54f 100644 --- a/src/blob.js +++ b/src/blob.js @@ -1,6 +1,8 @@ // Based on https://github.com/tmpvar/jsdom/blob/aa85b2abf07766ff7bf5c1f6daafb3726f2f2db5/lib/jsdom/living/blob.js // (MIT licensed) +import { Readable } from 'stream'; + export const BUFFER = Symbol('buffer'); const TYPE = Symbol('type'); @@ -57,13 +59,16 @@ export default class Blob { const ab = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength); return Promise.resolve(ab); } - // stream() { - // const readable = new Readable() - // readable._read = () => {} - // readable.push(this[BUFFER]) - // readable.push(null) - // return readable || whatwg stream? not decided - // } + stream() { + const readable = new Readable(); + readable._read = () => {}; + readable.push(this[BUFFER]); + readable.push(null); + return readable; + } + toString() { + return '[object Blob]' + } slice() { const size = this.size; diff --git a/test/test.js b/test/test.js index 22b2a9173..7247fc93b 100644 --- a/test/test.js +++ b/test/test.js @@ -1779,8 +1779,8 @@ describe('node-fetch', () => { .then(blob => blob.text()) .then(body => { expect(body).to.equal('hello'); - }) - }) + }); + }); it('should support reading blob as arrayBuffer', function() { return new Response(`hello`) @@ -1789,8 +1789,17 @@ describe('node-fetch', () => { .then(ab => { const str = String.fromCharCode.apply(null, new Uint8Array(ab)); expect(str).to.equal('hello'); - }) - }) + }); + }); + + it('should support reading blob as stream', function() { + return new Response(`hello`) + .blob() + .then(blob => streamToPromise(blob.stream(), data => { + const str = data.toString(); + expect(str).to.equal('hello'); + })); + }); it('should support blob round-trip', function() { const url = `${base}hello`; From 05f5ac12a2d4d24a3e7abd3ce2677eb633d4efc2 Mon Sep 17 00:00:00 2001 From: David Frank Date: Sat, 27 Apr 2019 00:11:52 +0800 Subject: [PATCH 191/223] Node 12 compatibility (#614) * dev package bump * test invalid header differently as node 12 no longer accepts invalid headers in response * add node v10 in travis test list as node 12 has been released --- .travis.yml | 1 + package.json | 20 ++++++++++---------- test/server.js | 14 -------------- test/test.js | 22 ++++++++++------------ 4 files changed, 21 insertions(+), 36 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0f8a1e5f2..3bb109e15 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ node_js: - "4" - "6" - "8" + - "10" - "node" env: - FORMDATA_VERSION=1.0.0 diff --git a/package.json b/package.json index 3d591b0e1..57b0f9586 100644 --- a/package.json +++ b/package.json @@ -37,28 +37,28 @@ }, "homepage": "https://github.com/bitinn/node-fetch", "devDependencies": { - "abort-controller": "^1.0.2", - "abortcontroller-polyfill": "^1.1.9", - "babel-core": "^6.26.0", - "babel-plugin-istanbul": "^4.1.5", + "abort-controller": "^1.1.0", + "abortcontroller-polyfill": "^1.3.0", + "babel-core": "^6.26.3", + "babel-plugin-istanbul": "^4.1.6", "babel-preset-env": "^1.6.1", "babel-register": "^6.16.3", "chai": "^3.5.0", "chai-as-promised": "^7.1.1", "chai-iterator": "^1.1.1", "chai-string": "~1.3.0", - "codecov": "^3.0.0", - "cross-env": "^5.1.3", - "form-data": "^2.3.1", + "codecov": "^3.3.0", + "cross-env": "^5.2.0", + "form-data": "^2.3.3", "is-builtin-module": "^1.0.0", "mocha": "^5.0.0", "nyc": "11.9.0", "parted": "^0.1.1", - "promise": "^8.0.1", + "promise": "^8.0.3", "resumer": "0.0.0", "rollup": "^0.63.4", - "rollup-plugin-babel": "^3.0.3", - "string-to-arraybuffer": "^1.0.0", + "rollup-plugin-babel": "^3.0.7", + "string-to-arraybuffer": "^1.0.2", "url-search-params": "^1.0.2", "whatwg-url": "^5.0.0" }, diff --git a/test/server.js b/test/server.js index 4028f0cc4..e672a6e9a 100644 --- a/test/server.js +++ b/test/server.js @@ -117,20 +117,6 @@ export default class TestServer { res.end('fake gzip string'); } - if (p === '/invalid-header') { - res.setHeader('Content-Type', 'text/plain'); - res.writeHead(200); - // HACK: add a few invalid headers to the generated header string before - // it is actually sent to the socket. - res._header = res._header.replace(/\r\n$/, [ - 'Invalid-Header : abc\r\n', - 'Invalid-Header-Value: \x07k\r\n', - 'Set-Cookie: \x07k\r\n', - 'Set-Cookie: \x07kk\r\n', - ].join('') + '\r\n'); - res.end('hello world\n'); - } - if (p === '/timeout') { setTimeout(function() { res.statusCode = 200; diff --git a/test/test.js b/test/test.js index 7247fc93b..9384abd4e 100644 --- a/test/test.js +++ b/test/test.js @@ -45,7 +45,7 @@ import fetch, { Response } from '../src/'; import FetchErrorOrig from '../src/fetch-error.js'; -import HeadersOrig from '../src/headers.js'; +import HeadersOrig, { createHeadersLenient } from '../src/headers.js'; import RequestOrig from '../src/request.js'; import ResponseOrig from '../src/response.js'; import Body from '../src/body.js'; @@ -489,17 +489,15 @@ describe('node-fetch', () => { }); it('should ignore invalid headers', function() { - const url = `${base}invalid-header`; - return fetch(url).then(res => { - expect(res.headers.get('Invalid-Header')).to.be.null; - expect(res.headers.get('Invalid-Header-Value')).to.be.null; - expect(res.headers.get('Set-Cookie')).to.be.null; - expect(Array.from(res.headers.keys()).length).to.equal(4); - expect(res.headers.has('Connection')).to.be.true; - expect(res.headers.has('Content-Type')).to.be.true; - expect(res.headers.has('Date')).to.be.true; - expect(res.headers.has('Transfer-Encoding')).to.be.true; - }); + var headers = { + 'Invalid-Header ': 'abc\r\n', + 'Invalid-Header-Value': '\x07k\r\n', + 'Set-Cookie': ['\x07k\r\n', '\x07kk\r\n'] + }; + headers = createHeadersLenient(headers); + expect(headers).to.not.have.property('Invalid-Header '); + expect(headers).to.not.have.property('Invalid-Header-Value'); + expect(headers).to.not.have.property('Set-Cookie'); }); it('should handle client-error response', function() { From 2a2d4384afd601d8697277b0e737466418db53c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Muhammet=20=C3=96zt=C3=BCrk?= Date: Fri, 26 Apr 2019 19:20:15 +0300 Subject: [PATCH 192/223] Adding Brotli Support (#598) * adding brotli support * support old node versions * better test --- .gitignore | 3 +++ src/index.js | 8 ++++++++ test/server.js | 18 ++++++++++++++++++ test/test.js | 28 ++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+) diff --git a/.gitignore b/.gitignore index 97fd1c698..839eff401 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,6 @@ lib # Ignore package manager lock files package-lock.json yarn.lock + +# Ignore IDE +.idea diff --git a/src/index.js b/src/index.js index e22799840..6d2132ab7 100644 --- a/src/index.js +++ b/src/index.js @@ -244,6 +244,14 @@ export default function fetch(url, opts) { return; } + // for br + if (codings == 'br' && typeof zlib.createBrotliDecompress === 'function') { + body = body.pipe(zlib.createBrotliDecompress()); + response = new Response(body, response_options); + resolve(response); + return; + } + // otherwise, use response as-is response = new Response(body, response_options); resolve(response); diff --git a/test/server.js b/test/server.js index e672a6e9a..524208fcd 100644 --- a/test/server.js +++ b/test/server.js @@ -94,6 +94,18 @@ export default class TestServer { }); } + if (p === '/brotli') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/plain'); + if (typeof zlib.createBrotliDecompress === 'function') { + res.setHeader('Content-Encoding', 'br'); + zlib.brotliCompress('hello world', function (err, buffer) { + res.end(buffer); + }); + } + } + + if (p === '/deflate-raw') { res.statusCode = 200; res.setHeader('Content-Type', 'text/plain'); @@ -308,6 +320,12 @@ export default class TestServer { res.end(); } + if (p === '/no-content/brotli') { + res.statusCode = 204; + res.setHeader('Content-Encoding', 'br'); + res.end(); + } + if (p === '/not-modified') { res.statusCode = 304; res.end(); diff --git a/test/test.js b/test/test.js index 9384abd4e..019ba739a 100644 --- a/test/test.js +++ b/test/test.js @@ -50,6 +50,7 @@ import RequestOrig from '../src/request.js'; import ResponseOrig from '../src/response.js'; import Body from '../src/body.js'; import Blob from '../src/blob.js'; +import zlib from "zlib"; const supportToString = ({ [Symbol.toStringTag]: 'z' @@ -664,6 +665,33 @@ describe('node-fetch', () => { }); }); + it('should decompress brotli response', function() { + if(typeof zlib.createBrotliDecompress !== 'function') this.skip(); + const url = `${base}brotli`; + return fetch(url).then(res => { + expect(res.headers.get('content-type')).to.equal('text/plain'); + return res.text().then(result => { + expect(result).to.be.a('string'); + expect(result).to.equal('hello world'); + }); + }); + }); + + it('should handle no content response with brotli encoding', function() { + if(typeof zlib.createBrotliDecompress !== 'function') this.skip(); + const url = `${base}no-content/brotli`; + return fetch(url).then(res => { + expect(res.status).to.equal(204); + expect(res.statusText).to.equal('No Content'); + expect(res.headers.get('content-encoding')).to.equal('br'); + expect(res.ok).to.be.true; + return res.text().then(result => { + expect(result).to.be.a('string'); + expect(result).to.be.empty; + }); + }); + }); + it('should skip decompression if unsupported', function() { const url = `${base}sdch`; return fetch(url).then(res => { From cfc8e5bad29422189cda3f9c47cd294caac7b3be Mon Sep 17 00:00:00 2001 From: Andrew Leedham Date: Fri, 26 Apr 2019 17:27:31 +0100 Subject: [PATCH 193/223] Swap packagephobia badge for flat style (#592) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e900c9719..3e6ff459a 100644 --- a/README.md +++ b/README.md @@ -522,7 +522,7 @@ MIT [travis-url]: https://travis-ci.org/bitinn/node-fetch [codecov-image]: https://img.shields.io/codecov/c/github/bitinn/node-fetch.svg?style=flat-square [codecov-url]: https://codecov.io/gh/bitinn/node-fetch -[install-size-image]: https://packagephobia.now.sh/badge?p=node-fetch +[install-size-image]: https://flat.badgen.net/packagephobia/install/node-fetch [install-size-url]: https://packagephobia.now.sh/result?p=node-fetch [whatwg-fetch]: https://fetch.spec.whatwg.org/ [response-init]: https://fetch.spec.whatwg.org/#responseinit From 49d77600a7475dffbe7051f2c1f15d2e6921067e Mon Sep 17 00:00:00 2001 From: David Frank Date: Sat, 27 Apr 2019 00:46:53 +0800 Subject: [PATCH 194/223] Pass custom timeout to subsequent requests on redirect (#615) --- src/index.js | 1 + test/server.js | 8 ++++++++ test/test.js | 11 +++++++++++ 3 files changed, 20 insertions(+) diff --git a/src/index.js b/src/index.js index 6d2132ab7..b716550a8 100644 --- a/src/index.js +++ b/src/index.js @@ -153,6 +153,7 @@ export default function fetch(url, opts) { method: request.method, body: request.body, signal: request.signal, + timeout: request.timeout }; // HTTP-redirect fetch step 9 diff --git a/test/server.js b/test/server.js index 524208fcd..15347885f 100644 --- a/test/server.js +++ b/test/server.js @@ -275,6 +275,14 @@ export default class TestServer { }, 1000); } + if (p === '/redirect/slow-chain') { + res.statusCode = 301; + res.setHeader('Location', '/redirect/slow'); + setTimeout(function() { + res.end(); + }, 100); + } + if (p === '/redirect/slow-stream') { res.statusCode = 301; res.setHeader('Location', '/slow'); diff --git a/test/test.js b/test/test.js index 019ba739a..21d633252 100644 --- a/test/test.js +++ b/test/test.js @@ -799,6 +799,17 @@ describe('node-fetch', () => { }); }); + it('should allow custom timeout on redirected requests', function() { + this.timeout(2000); + const url = `${base}redirect/slow-chain`; + const opts = { + timeout: 200 + }; + return expect(fetch(url, opts)).to.eventually.be.rejected + .and.be.an.instanceOf(FetchError) + .and.have.property('type', 'request-timeout'); + }); + it('should clear internal timeout on fetch response', function (done) { this.timeout(2000); spawn('node', ['-e', `require('./')('${base}hello', { timeout: 10000 })`]) From c9805a2868bb0896be126acefdc2c11c4c586bf9 Mon Sep 17 00:00:00 2001 From: David Frank Date: Sat, 27 Apr 2019 01:20:20 +0800 Subject: [PATCH 195/223] 2.4.0 release (#616) * changelog update * package.json update --- CHANGELOG.md | 10 +++++++++- package.json | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85b2e2ad0..5166b53db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,9 +5,17 @@ Changelog # 2.x release +## v2.4.0 + +- Enhance: added `Brotli` compression support (using node's zlib). +- Enhance: updated `Blob` implementation per spec. +- Fix: set content type automatically for `URLSearchParams`. +- Fix: `Headers` now reject empty header names. +- Fix: test cases, as node 12+ no longer accepts invalid header response. + ## v2.3.0 -- New: `AbortSignal` support, with README example. +- Enhance: added `AbortSignal` support, with README example. - Enhance: handle invalid `Location` header during redirect by rejecting them explicitly with `FetchError`. - Fix: update `browser.js` to support react-native environment, where `self` isn't available globally. diff --git a/package.json b/package.json index 57b0f9586..0792d74f4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.3.0", + "version": "2.4.0", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index", "browser": "./browser.js", From 1a88481fbda4a3614adbb9f537e3e86494850414 Mon Sep 17 00:00:00 2001 From: mcuppi <46288829+mcuppi@users.noreply.github.com> Date: Sat, 27 Apr 2019 02:34:01 -0400 Subject: [PATCH 196/223] Fix Blob for older node versions and webpack. (#618) `Readable` isn't a named export --- src/blob.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/blob.js b/src/blob.js index e03a6d54f..e1151a955 100644 --- a/src/blob.js +++ b/src/blob.js @@ -1,7 +1,10 @@ // Based on https://github.com/tmpvar/jsdom/blob/aa85b2abf07766ff7bf5c1f6daafb3726f2f2db5/lib/jsdom/living/blob.js // (MIT licensed) -import { Readable } from 'stream'; +import Stream from 'stream'; + +// fix for "Readable" isn't a named export issue +const Readable = Stream.Readable; export const BUFFER = Symbol('buffer'); const TYPE = Symbol('type'); From b3ecba5e81016390eec57718636122459cc33a94 Mon Sep 17 00:00:00 2001 From: David Frank Date: Sat, 27 Apr 2019 14:50:25 +0800 Subject: [PATCH 197/223] 2.4.1 release (#619) * changelog update * package.json update --- CHANGELOG.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5166b53db..ef22c7748 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ Changelog # 2.x release +## v2.4.1 + +- Fix: `Blob` import rule for node < 10, as `Readable` isn't a named export. + ## v2.4.0 - Enhance: added `Brotli` compression support (using node's zlib). diff --git a/package.json b/package.json index 0792d74f4..34c583ab0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.4.0", + "version": "2.4.1", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index", "browser": "./browser.js", From a35dcd14a3dd90b0ed0062740d380aff3904a6a7 Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Tue, 30 Apr 2019 20:14:11 -0700 Subject: [PATCH 198/223] chore(deps): address deprecated url-search-params package (#622) --- package.json | 2 +- test/test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 34c583ab0..2c7ad0943 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ }, "homepage": "https://github.com/bitinn/node-fetch", "devDependencies": { + "@ungap/url-search-params": "^0.1.2", "abort-controller": "^1.1.0", "abortcontroller-polyfill": "^1.3.0", "babel-core": "^6.26.3", @@ -59,7 +60,6 @@ "rollup": "^0.63.4", "rollup-plugin-babel": "^3.0.7", "string-to-arraybuffer": "^1.0.2", - "url-search-params": "^1.0.2", "whatwg-url": "^5.0.0" }, "dependencies": {} diff --git a/test/test.js b/test/test.js index 21d633252..e96c85a65 100644 --- a/test/test.js +++ b/test/test.js @@ -8,7 +8,7 @@ import then from 'promise'; import resumer from 'resumer'; import FormData from 'form-data'; import stringToArrayBuffer from 'string-to-arraybuffer'; -import URLSearchParams_Polyfill from 'url-search-params'; +import URLSearchParams_Polyfill from '@ungap/url-search-params'; import { URL } from 'whatwg-url'; import { AbortController } from 'abortcontroller-polyfill/dist/abortcontroller'; import AbortController2 from 'abort-controller'; From 1fe1358642ad9bad5895747f2d9b4c1f6f7cc5f0 Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Tue, 30 Apr 2019 20:15:05 -0700 Subject: [PATCH 199/223] test: enable --throw-deprecation for tests (#625) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2c7ad0943..599e16e18 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "scripts": { "build": "cross-env BABEL_ENV=rollup rollup -c", "prepare": "npm run build", - "test": "cross-env BABEL_ENV=test mocha --require babel-register test/test.js", + "test": "cross-env BABEL_ENV=test mocha --require babel-register --throw-deprecation test/test.js", "report": "cross-env BABEL_ENV=coverage nyc --reporter lcov --reporter text mocha -R spec test/test.js", "coverage": "cross-env BABEL_ENV=coverage nyc --reporter json --reporter text mocha -R spec test/test.js && codecov -f coverage/coverage-final.json" }, From d8f5ba0e97fd9711940eac766951a1c8222383b0 Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Tue, 30 Apr 2019 20:19:06 -0700 Subject: [PATCH 200/223] build: disable generation of package-lock since it is not used (#623) --- .npmrc | 1 + 1 file changed, 1 insertion(+) create mode 100644 .npmrc diff --git a/.npmrc b/.npmrc new file mode 100644 index 000000000..43c97e719 --- /dev/null +++ b/.npmrc @@ -0,0 +1 @@ +package-lock=false From 0fc414c2a88e897fd941c06734993a1d9a2747e7 Mon Sep 17 00:00:00 2001 From: David Frank Date: Wed, 1 May 2019 11:44:27 +0800 Subject: [PATCH 201/223] Allow third party blob implementation (#629) * Support making request with any blob that have stream() method * don't clone blob when cloning request * check for blob api that node-fetch uses --- src/body.js | 49 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/src/body.js b/src/body.js index c70b41052..4e7d66d8e 100644 --- a/src/body.js +++ b/src/body.js @@ -37,9 +37,8 @@ export default function Body(body, { } else if (isURLSearchParams(body)) { // body is a URLSearchParams body = Buffer.from(body.toString()); - } else if (body instanceof Blob) { + } else if (isBlob(body)) { // body is blob - body = body[BUFFER]; } else if (Buffer.isBuffer(body)) { // body is Buffer } else if (Object.prototype.toString.call(body) === '[object ArrayBuffer]') { @@ -191,18 +190,25 @@ function consumeBody() { return Body.Promise.reject(this[INTERNALS].error); } + let body = this.body; + // body is null - if (this.body === null) { + if (body === null) { return Body.Promise.resolve(Buffer.alloc(0)); } + // body is blob + if (isBlob(body)) { + body = body.stream(); + } + // body is buffer - if (Buffer.isBuffer(this.body)) { - return Body.Promise.resolve(this.body); + if (Buffer.isBuffer(body)) { + return Body.Promise.resolve(body); } // istanbul ignore if: should never happen - if (!(this.body instanceof Stream)) { + if (!(body instanceof Stream)) { return Body.Promise.resolve(Buffer.alloc(0)); } @@ -224,7 +230,7 @@ function consumeBody() { } // handle stream errors - this.body.on('error', err => { + body.on('error', err => { if (err.name === 'AbortError') { // if the request was aborted, reject with this Error abort = true; @@ -235,7 +241,7 @@ function consumeBody() { } }); - this.body.on('data', chunk => { + body.on('data', chunk => { if (abort || chunk === null) { return; } @@ -250,7 +256,7 @@ function consumeBody() { accum.push(chunk); }); - this.body.on('end', () => { + body.on('end', () => { if (abort) { return; } @@ -355,6 +361,22 @@ function isURLSearchParams(obj) { typeof obj.sort === 'function'; } +/** + * Check if `obj` is a W3C `Blob` object (which `File` inherits from) + * @param {*} obj + * @return {boolean} + */ +function isBlob(obj) { + return typeof obj === 'object' && + typeof obj.arrayBuffer === 'function' && + typeof obj.type === 'string' && + typeof obj.stream === 'function' && + typeof obj.constructor === 'function' && + typeof obj.constructor.name === 'string' && + /^(Blob|File)$/.test(obj.constructor.name) && + /^(Blob|File)$/.test(obj[Symbol.toStringTag]) +} + /** * Clone body given Res/Req instance * @@ -407,7 +429,7 @@ export function extractContentType(body) { } else if (isURLSearchParams(body)) { // body is a URLSearchParams return 'application/x-www-form-urlencoded;charset=UTF-8'; - } else if (body instanceof Blob) { + } else if (isBlob(body)) { // body is blob return body.type || null; } else if (Buffer.isBuffer(body)) { @@ -448,6 +470,8 @@ export function getTotalBytes(instance) { if (body === null) { // body is null return 0; + } else if (isBlob(body)) { + return body.size; } else if (Buffer.isBuffer(body)) { // body is buffer return body.length; @@ -460,8 +484,7 @@ export function getTotalBytes(instance) { return null; } else { // body is stream - // can't really do much about this - return null; + return instance.size || null; } } @@ -477,6 +500,8 @@ export function writeToStream(dest, instance) { if (body === null) { // body is null dest.end(); + } else if (isBlob(body)) { + body.stream().pipe(dest); } else if (Buffer.isBuffer(body)) { // body is buffer dest.write(body); From 0c2294ec48fa5b84519f8bdd60f4e2672ebd9b06 Mon Sep 17 00:00:00 2001 From: David Frank Date: Wed, 1 May 2019 13:05:32 +0800 Subject: [PATCH 202/223] 2.5.0 release (#630) * redirected property * changelog update * readme update * 2.5.0 --- CHANGELOG.md | 8 ++++++++ README.md | 15 ++++++++++----- package.json | 2 +- src/index.js | 3 ++- src/response.js | 12 +++++++++--- test/test.js | 18 ++++++++++++++++-- 6 files changed, 46 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef22c7748..941b6a8d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ Changelog # 2.x release +## v2.5.0 + +- Enhance: `Response` object now includes `redirected` property. +- Enhance: `fetch()` now accepts third-party `Blob` implementation as body. +- Other: disable `package-lock.json` generation as we never commit them. +- Other: dev dependency update. +- Other: readme update. + ## v2.4.1 - Fix: `Blob` import rule for node < 10, as `Readable` isn't a named export. diff --git a/README.md b/README.md index 3e6ff459a..48f4215e4 100644 --- a/README.md +++ b/README.md @@ -381,7 +381,6 @@ The following properties are not implemented in node-fetch at this moment: - `Response.error()` - `Response.redirect()` - `type` -- `redirected` - `trailer` #### new Response([body[, options]]) @@ -401,6 +400,12 @@ Because Node.js does not implement service workers (for which this class was des Convenience property representing if the request ended normally. Will evaluate to true if the response status was greater than or equal to 200 but smaller than 300. +#### response.redirected + +*(spec-compliant)* + +Convenience property representing if the request has been redirected at least once. Will evaluate to true if the internal redirect counter is greater than 0. + ### Class: Headers @@ -510,17 +515,17 @@ An Error thrown when the request is aborted in response to an `AbortSignal`'s `a Thanks to [github/fetch](https://github.com/github/fetch) for providing a solid implementation reference. -`node-fetch` v1 was maintained by [@bitinn](https://github.com/bitinn), v2 is currently maintained by [@TimothyGu](https://github.com/timothygu), v2 readme is written by [@jkantr](https://github.com/jkantr). +`node-fetch` v1 was maintained by [@bitinn](https://github.com/bitinn); v2 was maintained by [@TimothyGu](https://github.com/timothygu), [@bitinn](https://github.com/bitinn) and [@jimmywarting](https://github.com/jimmywarting); v2 readme is written by [@jkantr](https://github.com/jkantr). ## License MIT -[npm-image]: https://img.shields.io/npm/v/node-fetch.svg?style=flat-square +[npm-image]: https://flat.badgen.net/npm/v/node-fetch [npm-url]: https://www.npmjs.com/package/node-fetch -[travis-image]: https://img.shields.io/travis/bitinn/node-fetch.svg?style=flat-square +[travis-image]: https://flat.badgen.net/travis/bitinn/node-fetch [travis-url]: https://travis-ci.org/bitinn/node-fetch -[codecov-image]: https://img.shields.io/codecov/c/github/bitinn/node-fetch.svg?style=flat-square +[codecov-image]: https://flat.badgen.net/codecov/c/github/bitinn/node-fetch/master [codecov-url]: https://codecov.io/gh/bitinn/node-fetch [install-size-image]: https://flat.badgen.net/packagephobia/install/node-fetch [install-size-url]: https://packagephobia.now.sh/result?p=node-fetch diff --git a/package.json b/package.json index 599e16e18..353f79322 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.4.1", + "version": "2.5.0", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index", "browser": "./browser.js", diff --git a/src/index.js b/src/index.js index b716550a8..907f47275 100644 --- a/src/index.js +++ b/src/index.js @@ -189,7 +189,8 @@ export default function fetch(url, opts) { statusText: res.statusMessage, headers: headers, size: request.size, - timeout: request.timeout + timeout: request.timeout, + counter: request.counter }; // HTTP-network fetch step 12.1.1.3 diff --git a/src/response.js b/src/response.js index f29bfe296..e2ca49c3e 100644 --- a/src/response.js +++ b/src/response.js @@ -40,7 +40,8 @@ export default class Response { url: opts.url, status, statusText: opts.statusText || STATUS_CODES[status], - headers + headers, + counter: opts.counter }; } @@ -59,6 +60,10 @@ export default class Response { return this[INTERNALS].status >= 200 && this[INTERNALS].status < 300; } + get redirected() { + return this[INTERNALS].counter > 0; + } + get statusText() { return this[INTERNALS].statusText; } @@ -78,9 +83,9 @@ export default class Response { status: this.status, statusText: this.statusText, headers: this.headers, - ok: this.ok + ok: this.ok, + redirected: this.redirected }); - } } @@ -90,6 +95,7 @@ Object.defineProperties(Response.prototype, { url: { enumerable: true }, status: { enumerable: true }, ok: { enumerable: true }, + redirected: { enumerable: true }, statusText: { enumerable: true }, headers: { enumerable: true }, clone: { enumerable: true } diff --git a/test/test.js b/test/test.js index e96c85a65..00f45353e 100644 --- a/test/test.js +++ b/test/test.js @@ -489,6 +489,20 @@ describe('node-fetch', () => { }); }); + it('should set redirected property on response when redirect', function() { + const url = `${base}redirect/301`; + return fetch(url).then(res => { + expect(res.redirected).to.be.true; + }); + }); + + it('should not set redirected property on response without redirect', function() { + const url = `${base}hello`; + return fetch(url).then(res => { + expect(res.redirected).to.be.false; + }); + }); + it('should ignore invalid headers', function() { var headers = { 'Invalid-Header ': 'abc\r\n', @@ -2196,12 +2210,12 @@ describe('Response', function () { } for (const toCheck of [ 'body', 'bodyUsed', 'arrayBuffer', 'blob', 'json', 'text', - 'url', 'status', 'ok', 'statusText', 'headers', 'clone' + 'url', 'status', 'ok', 'redirected', 'statusText', 'headers', 'clone' ]) { expect(enumerableProperties).to.contain(toCheck); } for (const toCheck of [ - 'body', 'bodyUsed', 'url', 'status', 'ok', 'statusText', + 'body', 'bodyUsed', 'url', 'status', 'ok', 'redirected', 'statusText', 'headers' ]) { expect(() => { From bf8b4e8db350ec76dbb9236620f774fcc21b8c12 Mon Sep 17 00:00:00 2001 From: edgraaff Date: Sun, 5 May 2019 14:12:33 +0200 Subject: [PATCH 203/223] Allow agent option to be a function (#632) Enable users to return HTTP/HTTPS-specific agent based on request url --- README.md | 2 +- src/request.js | 9 +++++++-- test/test.js | 24 ++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 48f4215e4..65d3be74b 100644 --- a/README.md +++ b/README.md @@ -317,7 +317,7 @@ The default values are shown after each option key. timeout: 0, // req/res timeout in ms, it resets on redirect. 0 to disable (OS limit applies). Signal is recommended instead. compress: true, // support gzip/deflate content encoding. false to disable size: 0, // maximum response body size in bytes. 0 to disable - agent: null // http(s).Agent instance, allows custom proxy, certificate, dns lookup etc. + agent: null // http(s).Agent instance (or function providing one), allows custom proxy, certificate, dns lookup etc. } ``` diff --git a/src/request.js b/src/request.js index 99aef257f..45a7eb7e4 100644 --- a/src/request.js +++ b/src/request.js @@ -230,7 +230,12 @@ export function getNodeRequestOptions(request) { headers.set('Accept-Encoding', 'gzip,deflate'); } - if (!headers.has('Connection') && !request.agent) { + let agent = request.agent; + if (typeof agent === 'function') { + agent = agent(parsedURL); + } + + if (!headers.has('Connection') && !agent) { headers.set('Connection', 'close'); } @@ -240,6 +245,6 @@ export function getNodeRequestOptions(request) { return Object.assign({}, parsedURL, { method: request.method, headers: exportNodeCompatibleHeaders(headers), - agent: request.agent + agent }); } diff --git a/test/test.js b/test/test.js index 00f45353e..b9ff01806 100644 --- a/test/test.js +++ b/test/test.js @@ -1978,6 +1978,30 @@ describe('node-fetch', () => { expect(families[1]).to.equal(family); }); }); + + it('should allow a function supplying the agent', function() { + const url = `${base}inspect`; + + const agent = http.Agent({ + keepAlive: true + }); + + let parsedURL; + + return fetch(url, { + agent: function(_parsedURL) { + parsedURL = _parsedURL; + return agent; + } + }).then(res => { + return res.json(); + }).then(res => { + // the agent provider should have been called + expect(parsedURL.protocol).to.equal('http:'); + // the agent we returned should have been used + expect(res.headers['connection']).to.equal('keep-alive'); + }); + }); }); describe('Headers', function () { From 95286f52bb866283bc69521a04efe1de37b26a33 Mon Sep 17 00:00:00 2001 From: David Frank Date: Thu, 16 May 2019 14:38:28 +0800 Subject: [PATCH 204/223] v2.6.0 (#638) * Update readme and changelog for `options.agent` - Fix content-length issue introduced in v2.5.0 * More test coverage for `extractContentType` * Slightly improve test performance * `Response.url` should not return null * Document `Headers.raw()` usage better * 2.6.0 --- CHANGELOG.md | 6 ++++ README.md | 47 ++++++++++++++++++++++++- package.json | 2 +- src/body.js | 7 ++-- src/response.js | 2 +- test/server.js | 6 ++-- test/test.js | 91 +++++++++++++++++++++++++++++++++++++++++-------- 7 files changed, 136 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 941b6a8d8..188fcd399 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ Changelog # 2.x release +## v2.6.0 + +- Enhance: `options.agent`, it now accepts a function that returns custom http(s).Agent instance based on current URL, see readme for more information. +- Fix: incorrect `Content-Length` was returned for stream body in 2.5.0 release; note that `node-fetch` doesn't calculate content length for stream body. +- Fix: `Response.url` should return empty string instead of `null` by default. + ## v2.5.0 - Enhance: `Response` object now includes `redirected` property. diff --git a/README.md b/README.md index 65d3be74b..cb1990120 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ A light-weight module that brings `window.fetch` to Node.js - [Streams](#streams) - [Buffer](#buffer) - [Accessing Headers and other Meta data](#accessing-headers-and-other-meta-data) + - [Extract Set-Cookie Header](#extract-set-cookie-header) - [Post data using a file stream](#post-data-using-a-file-stream) - [Post with form-data (detect multipart)](#post-with-form-data-detect-multipart) - [Request cancellation with AbortSignal](#request-cancellation-with-abortsignal) @@ -208,6 +209,17 @@ fetch('https://github.com/') }); ``` +#### Extract Set-Cookie Header + +Unlike browsers, you can access raw `Set-Cookie` headers manually using `Headers.raw()`, this is a `node-fetch` only API. + +```js +fetch(url).then(res => { + // returns an array of values, instead of a string of comma-separated values + console.log(res.headers.raw()['set-cookie']); +}); +``` + #### Post data using a file stream ```js @@ -317,7 +329,7 @@ The default values are shown after each option key. timeout: 0, // req/res timeout in ms, it resets on redirect. 0 to disable (OS limit applies). Signal is recommended instead. compress: true, // support gzip/deflate content encoding. false to disable size: 0, // maximum response body size in bytes. 0 to disable - agent: null // http(s).Agent instance (or function providing one), allows custom proxy, certificate, dns lookup etc. + agent: null // http(s).Agent instance or function that returns an instance (see below) } ``` @@ -334,6 +346,39 @@ Header | Value `Transfer-Encoding` | `chunked` _(when `req.body` is a stream)_ `User-Agent` | `node-fetch/1.0 (+https://github.com/bitinn/node-fetch)` +Note: when `body` is a `Stream`, `Content-Length` is not set automatically. + +##### Custom Agent + +The `agent` option allows you to specify networking related options that's out of the scope of Fetch. Including and not limit to: + +- Support self-signed certificate +- Use only IPv4 or IPv6 +- Custom DNS Lookup + +See [`http.Agent`](https://nodejs.org/api/http.html#http_new_agent_options) for more information. + +In addition, `agent` option accepts a function that returns http(s).Agent instance given current [URL](https://nodejs.org/api/url.html), this is useful during a redirection chain across HTTP and HTTPS protocol. + +```js +const httpAgent = new http.Agent({ + keepAlive: true +}); +const httpsAgent = new https.Agent({ + keepAlive: true +}); + +const options = { + agent: function (_parsedURL) { + if (_parsedURL.protocol == 'http:') { + return httpAgent; + } else { + return httpsAgent; + } + } +} +``` + ### Class: Request diff --git a/package.json b/package.json index 353f79322..8e5c883b2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.5.0", + "version": "2.6.0", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index", "browser": "./browser.js", diff --git a/src/body.js b/src/body.js index 4e7d66d8e..1b6eab1f8 100644 --- a/src/body.js +++ b/src/body.js @@ -415,11 +415,9 @@ export function clone(instance) { * * This function assumes that instance.body is present. * - * @param Mixed instance Response or Request instance + * @param Mixed instance Any options.body input */ export function extractContentType(body) { - // istanbul ignore if: Currently, because of a guard in Request, body - // can never be null. Included here for completeness. if (body === null) { // body is null return null; @@ -466,7 +464,6 @@ export function extractContentType(body) { export function getTotalBytes(instance) { const {body} = instance; - // istanbul ignore if: included for completion if (body === null) { // body is null return 0; @@ -484,7 +481,7 @@ export function getTotalBytes(instance) { return null; } else { // body is stream - return instance.size || null; + return null; } } diff --git a/src/response.js b/src/response.js index e2ca49c3e..e4801bb70 100644 --- a/src/response.js +++ b/src/response.js @@ -46,7 +46,7 @@ export default class Response { } get url() { - return this[INTERNALS].url; + return this[INTERNALS].url || ''; } get status() { diff --git a/test/server.js b/test/server.js index 15347885f..e6aaacbf9 100644 --- a/test/server.js +++ b/test/server.js @@ -157,10 +157,10 @@ export default class TestServer { res.setHeader('Content-Type', 'text/plain'); setTimeout(function() { res.write('test'); - }, 50); + }, 10); setTimeout(function() { res.end('test'); - }, 100); + }, 20); } if (p === '/size/long') { @@ -280,7 +280,7 @@ export default class TestServer { res.setHeader('Location', '/redirect/slow'); setTimeout(function() { res.end(); - }, 100); + }, 10); } if (p === '/redirect/slow-stream') { diff --git a/test/test.js b/test/test.js index b9ff01806..38d3ce050 100644 --- a/test/test.js +++ b/test/test.js @@ -48,7 +48,7 @@ import FetchErrorOrig from '../src/fetch-error.js'; import HeadersOrig, { createHeadersLenient } from '../src/headers.js'; import RequestOrig from '../src/request.js'; import ResponseOrig from '../src/response.js'; -import Body from '../src/body.js'; +import Body, { getTotalBytes, extractContentType } from '../src/body.js'; import Blob from '../src/blob.js'; import zlib from "zlib"; @@ -738,7 +738,7 @@ describe('node-fetch', () => { // Wait a few ms to see if a uncaught error occurs setTimeout(() => { done(); - }, 50); + }, 20); }); }); @@ -748,7 +748,7 @@ describe('node-fetch', () => { return new Promise((resolve) => { setTimeout(() => { resolve(value) - }, 100); + }, 20); }); } @@ -789,10 +789,9 @@ describe('node-fetch', () => { }); it('should allow custom timeout', function() { - this.timeout(500); const url = `${base}timeout`; const opts = { - timeout: 100 + timeout: 20 }; return expect(fetch(url, opts)).to.eventually.be.rejected .and.be.an.instanceOf(FetchError) @@ -800,10 +799,9 @@ describe('node-fetch', () => { }); it('should allow custom timeout on response body', function() { - this.timeout(500); const url = `${base}slow`; const opts = { - timeout: 100 + timeout: 20 }; return fetch(url, opts).then(res => { expect(res.ok).to.be.true; @@ -814,10 +812,9 @@ describe('node-fetch', () => { }); it('should allow custom timeout on redirected requests', function() { - this.timeout(2000); const url = `${base}redirect/slow-chain`; const opts = { - timeout: 200 + timeout: 20 }; return expect(fetch(url, opts)).to.eventually.be.rejected .and.be.an.instanceOf(FetchError) @@ -908,7 +905,7 @@ describe('node-fetch', () => { '${base}timeout', { signal: controller.signal, timeout: 10000 } ); - setTimeout(function () { controller.abort(); }, 100); + setTimeout(function () { controller.abort(); }, 20); ` spawn('node', ['-e', script]) .on('exit', () => { @@ -940,7 +937,7 @@ describe('node-fetch', () => { }); setTimeout(() => { abortController.abort(); - }, 50); + }, 20); return expect(fetch(request)).to.be.eventually.rejected .and.be.an.instanceOf(Error) .and.have.property('name', 'AbortError'); @@ -1914,8 +1911,8 @@ describe('node-fetch', () => { expect(err.type).to.equal('test-error'); expect(err.code).to.equal('ESOMEERROR'); expect(err.errno).to.equal('ESOMEERROR'); - expect(err.stack).to.include('funcName') - .and.to.startWith(`${err.name}: ${err.message}`); + // reading the stack is quite slow (~30-50ms) + expect(err.stack).to.include('funcName').and.to.startWith(`${err.name}: ${err.message}`); }); it('should support https request', function() { @@ -1982,7 +1979,7 @@ describe('node-fetch', () => { it('should allow a function supplying the agent', function() { const url = `${base}inspect`; - const agent = http.Agent({ + const agent = new http.Agent({ keepAlive: true }); @@ -2002,6 +1999,67 @@ describe('node-fetch', () => { expect(res.headers['connection']).to.equal('keep-alive'); }); }); + + it('should calculate content length and extract content type for each body type', function () { + const url = `${base}hello`; + const bodyContent = 'a=1'; + + let streamBody = resumer().queue(bodyContent).end(); + streamBody = streamBody.pipe(new stream.PassThrough()); + const streamRequest = new Request(url, { + method: 'POST', + body: streamBody, + size: 1024 + }); + + let blobBody = new Blob([bodyContent], { type: 'text/plain' }); + const blobRequest = new Request(url, { + method: 'POST', + body: blobBody, + size: 1024 + }); + + let formBody = new FormData(); + formBody.append('a', '1'); + const formRequest = new Request(url, { + method: 'POST', + body: formBody, + size: 1024 + }); + + let bufferBody = Buffer.from(bodyContent); + const bufferRequest = new Request(url, { + method: 'POST', + body: bufferBody, + size: 1024 + }); + + const stringRequest = new Request(url, { + method: 'POST', + body: bodyContent, + size: 1024 + }); + + const nullRequest = new Request(url, { + method: 'GET', + body: null, + size: 1024 + }); + + expect(getTotalBytes(streamRequest)).to.be.null; + expect(getTotalBytes(blobRequest)).to.equal(blobBody.size); + expect(getTotalBytes(formRequest)).to.not.be.null; + expect(getTotalBytes(bufferRequest)).to.equal(bufferBody.length); + expect(getTotalBytes(stringRequest)).to.equal(bodyContent.length); + expect(getTotalBytes(nullRequest)).to.equal(0); + + expect(extractContentType(streamBody)).to.be.null; + expect(extractContentType(blobBody)).to.equal('text/plain'); + expect(extractContentType(formBody)).to.startWith('multipart/form-data'); + expect(extractContentType(bufferBody)).to.be.null; + expect(extractContentType(bodyContent)).to.equal('text/plain;charset=UTF-8'); + expect(extractContentType(null)).to.be.null; + }); }); describe('Headers', function () { @@ -2387,6 +2445,11 @@ describe('Response', function () { const res = new Response(null); expect(res.status).to.equal(200); }); + + it('should default to empty string as url', function() { + const res = new Response(); + expect(res.url).to.equal(''); + }); }); describe('Request', function () { From 086be6fc74d8cc69faf76f65bf96d8f76b224dd1 Mon Sep 17 00:00:00 2001 From: Steve Moser Date: Fri, 9 Aug 2019 05:17:25 -0400 Subject: [PATCH 205/223] Remove --save option as it isn't required anymore (#581) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cb1990120..ecb5e487f 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ See Matt Andrews' [isomorphic-fetch](https://github.com/matthew-andrews/isomorph Current stable release (`2.x`) ```sh -$ npm install node-fetch --save +$ npm install node-fetch ``` ## Loading and configuring the module From eb3a57255b4eaa446d52e4cf3e77a1e560d61527 Mon Sep 17 00:00:00 2001 From: Richie Bendall Date: Sun, 8 Sep 2019 09:44:40 +1200 Subject: [PATCH 206/223] feat: Data URI support (#659) Adds support for Data URIs using native methods in Node 5.10.0+ --- src/index.js | 11 +++++++++++ test/test.js | 25 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/index.js b/src/index.js index 907f47275..56044fe41 100644 --- a/src/index.js +++ b/src/index.js @@ -38,6 +38,17 @@ export default function fetch(url, opts) { throw new Error('native promise missing, set fetch.Promise to your favorite alternative'); } + if (/^data:/.test(url)) { + const request = new Request(url, opts); + try { + const data = Buffer.from(url.split(',')[1], 'base64') + const res = new Response(data.body, { headers: { 'Content-Type': data.mimeType || url.match(/^data:(.+);base64,.*$/)[1] } }); + return fetch.Promise.resolve(res); + } catch (err) { + return fetch.Promise.reject(new FetchError(`[${request.method}] ${request.url} invalid URL, ${err.message}`, 'system', err)); + } + } + Body.Promise = fetch.Promise; // wrap http.request into fetch diff --git a/test/test.js b/test/test.js index 38d3ce050..b8c62dc6d 100644 --- a/test/test.js +++ b/test/test.js @@ -2834,4 +2834,29 @@ describe('external encoding', () => { }); }); }); + + describe('data uri', function() { + const dataUrl = 'data:image/gif;base64,R0lGODlhAQABAIAAAAUEBAAAACwAAAAAAQABAAACAkQBADs='; + + const invalidDataUrl = 'data:@@@@'; + + it('should accept data uri', function() { + return fetch(dataUrl).then(r => { + console.assert(r.status == 200); + console.assert(r.headers.get('Content-Type') == 'image/gif'); + + return r.buffer().then(b => { + console.assert(b instanceof Buffer); + }); + }); + }); + + it('should reject invalid data uri', function() { + return fetch(invalidDataUrl) + .catch(e => { + console.assert(e); + console.assert(e.message.includes('invalid URL')); + }); + }); + }); }); From 1d5778ad0d910dbd1584fb407a186f5a0bc1ea22 Mon Sep 17 00:00:00 2001 From: Richie Bendall Date: Sun, 8 Sep 2019 10:00:54 +1200 Subject: [PATCH 207/223] docs: Add Discord badge --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index ecb5e487f..eee288e0e 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ node-fetch [![build status][travis-image]][travis-url] [![coverage status][codecov-image]][codecov-url] [![install size][install-size-image]][install-size-url] +[![Discord][discord-image]][discord-url] A light-weight module that brings `window.fetch` to Node.js @@ -574,6 +575,8 @@ MIT [codecov-url]: https://codecov.io/gh/bitinn/node-fetch [install-size-image]: https://flat.badgen.net/packagephobia/install/node-fetch [install-size-url]: https://packagephobia.now.sh/result?p=node-fetch +[discord-image]: https://img.shields.io/discord/619915844268326952?color=%237289DA&label=Discord&style=flat-square +[discord-url]: https://discord.gg/Zxbndcm [whatwg-fetch]: https://fetch.spec.whatwg.org/ [response-init]: https://fetch.spec.whatwg.org/#responseinit [node-readable]: https://nodejs.org/api/stream.html#stream_readable_streams From 5535c2ed478d418969ecfd60c16453462de2a53f Mon Sep 17 00:00:00 2001 From: Boris Bosiljcic Date: Mon, 16 Sep 2019 13:52:22 +0200 Subject: [PATCH 208/223] fix: Check for global.fetch before binding it (#674) --- browser.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/browser.js b/browser.js index 0ad5de004..83c54c584 100644 --- a/browser.js +++ b/browser.js @@ -16,7 +16,9 @@ var global = getGlobal(); module.exports = exports = global.fetch; // Needed for TypeScript and Webpack. -exports.default = global.fetch.bind(global); +if (global.fetch) { + exports.default = global.fetch.bind(global); +} exports.Headers = global.Headers; exports.Request = global.Request; From 7b136627c537cb24430b0310638c9177a85acee1 Mon Sep 17 00:00:00 2001 From: Richie Bendall Date: Wed, 2 Oct 2019 21:50:00 +1300 Subject: [PATCH 209/223] chore: Add funding link --- .github/FUNDING.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 000000000..78f6bbf83 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,12 @@ +# These are supported funding model platforms + +github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +patreon: # Replace with a single Patreon username +open_collective: node-fetch # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +custom: # Replace with a single custom sponsorship URL From 47a24a03eb49a49d81b768892aee10074ed54a91 Mon Sep 17 00:00:00 2001 From: Richie Bendall Date: Wed, 2 Oct 2019 22:00:55 +1300 Subject: [PATCH 210/223] chore: Add opencollective badge --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index eee288e0e..7f48e026a 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ node-fetch [![coverage status][codecov-image]][codecov-url] [![install size][install-size-image]][install-size-url] [![Discord][discord-image]][discord-url] +[![Opencollective][opencollective-image]][opencollective-url] A light-weight module that brings `window.fetch` to Node.js @@ -577,6 +578,8 @@ MIT [install-size-url]: https://packagephobia.now.sh/result?p=node-fetch [discord-image]: https://img.shields.io/discord/619915844268326952?color=%237289DA&label=Discord&style=flat-square [discord-url]: https://discord.gg/Zxbndcm +[opencollective-image]: https://img.shields.io/opencollective/all/node-fetch?label=Sponsors&style=flat-square +[opencollective-url]: https://opencollective.com/node-fetch [whatwg-fetch]: https://fetch.spec.whatwg.org/ [response-init]: https://fetch.spec.whatwg.org/#responseinit [node-readable]: https://nodejs.org/api/stream.html#stream_readable_streams From 6a5d192034a0f438551dffb6d2d8df2c00921d16 Mon Sep 17 00:00:00 2001 From: dsuket Date: Mon, 7 Oct 2019 15:58:27 +0900 Subject: [PATCH 211/223] fix: Properly parse meta tag when parameters are reversed (#682) --- src/body.js | 6 ++++++ test/server.js | 6 ++++++ test/test.js | 10 ++++++++++ 3 files changed, 22 insertions(+) diff --git a/src/body.js b/src/body.js index 1b6eab1f8..a9d2e7973 100644 --- a/src/body.js +++ b/src/body.js @@ -306,6 +306,12 @@ function convertBody(buffer, headers) { // html4 if (!res && str) { res = /
中文
', 'gb2312')); } + if (p === '/encoding/gb2312-reverse') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/html'); + res.end(convert('
中文
', 'gb2312')); + } + if (p === '/encoding/shift-jis') { res.statusCode = 200; res.setHeader('Content-Type', 'text/html; charset=Shift-JIS'); diff --git a/test/test.js b/test/test.js index b8c62dc6d..c5d61c72a 100644 --- a/test/test.js +++ b/test/test.js @@ -2767,6 +2767,16 @@ describe('external encoding', () => { }); }); + it('should support encoding decode, html4 detect reverse http-equiv', function() { + const url = `${base}encoding/gb2312-reverse`; + return fetch(url).then(res => { + expect(res.status).to.equal(200); + return res.textConverted().then(result => { + expect(result).to.equal('
中文
'); + }); + }); + }); + it('should default to utf8 encoding', function() { const url = `${base}encoding/utf8`; return fetch(url).then(res => { From 244e6f63d42025465796e3ca4ce813bf2c31fc5b Mon Sep 17 00:00:00 2001 From: Richie Bendall Date: Mon, 7 Oct 2019 20:23:11 +1300 Subject: [PATCH 212/223] docs: Show backers in README --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7f48e026a..95c6cb6f7 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,13 @@ node-fetch [![coverage status][codecov-image]][codecov-url] [![install size][install-size-image]][install-size-url] [![Discord][discord-image]][discord-url] -[![Opencollective][opencollective-image]][opencollective-url] A light-weight module that brings `window.fetch` to Node.js (We are looking for [v2 maintainers and collaborators](https://github.com/bitinn/node-fetch/issues/567)) +[![Backers][opencollective-image]][opencollective-url] + - [Motivation](#motivation) @@ -578,7 +579,7 @@ MIT [install-size-url]: https://packagephobia.now.sh/result?p=node-fetch [discord-image]: https://img.shields.io/discord/619915844268326952?color=%237289DA&label=Discord&style=flat-square [discord-url]: https://discord.gg/Zxbndcm -[opencollective-image]: https://img.shields.io/opencollective/all/node-fetch?label=Sponsors&style=flat-square +[opencollective-image]: https://opencollective.com/node-fetch/backers.svg [opencollective-url]: https://opencollective.com/node-fetch [whatwg-fetch]: https://fetch.spec.whatwg.org/ [response-init]: https://fetch.spec.whatwg.org/#responseinit From 1e99050f944ac435fce26a9549eadcc2419a968a Mon Sep 17 00:00:00 2001 From: Ramit Mittal Date: Fri, 11 Oct 2019 01:56:58 +0530 Subject: [PATCH 213/223] fix: Change error message thrown with redirect mode set to error (#653) The original error message does not provide enough information about what went wrong. It simply states a configuration setting. --- src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.js b/src/index.js index 56044fe41..8bf9248fd 100644 --- a/src/index.js +++ b/src/index.js @@ -125,7 +125,7 @@ export default function fetch(url, opts) { // HTTP fetch step 5.5 switch (request.redirect) { case 'error': - reject(new FetchError(`redirect mode is set to error: ${request.url}`, 'no-redirect')); + reject(new FetchError(`uri requested responds with a redirect, redirect mode is set to error: ${request.url}`, 'no-redirect')); finalize(); return; case 'manual': From 8c197f8982a238b3c345c64b17bfa92e16b4f7c4 Mon Sep 17 00:00:00 2001 From: Sesamestrong Date: Sun, 20 Oct 2019 22:32:52 -0400 Subject: [PATCH 214/223] docs: Fix typos and grammatical errors in README.md (#686) --- README.md | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 95c6cb6f7..2dde74289 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ A light-weight module that brings `window.fetch` to Node.js ## Motivation -Instead of implementing `XMLHttpRequest` in Node.js to run browser-specific [Fetch polyfill](https://github.com/github/fetch), why not go from native `http` to `fetch` API directly? Hence `node-fetch`, minimal code for a `window.fetch` compatible API on Node.js runtime. +Instead of implementing `XMLHttpRequest` in Node.js to run browser-specific [Fetch polyfill](https://github.com/github/fetch), why not go from native `http` to `fetch` API directly? Hence, `node-fetch`, minimal code for a `window.fetch` compatible API on Node.js runtime. See Matt Andrews' [isomorphic-fetch](https://github.com/matthew-andrews/isomorphic-fetch) or Leonardo Quixada's [cross-fetch](https://github.com/lquixada/cross-fetch) for isomorphic usage (exports `node-fetch` for server-side, `whatwg-fetch` for client-side). @@ -59,9 +59,9 @@ See Matt Andrews' [isomorphic-fetch](https://github.com/matthew-andrews/isomorph - Stay consistent with `window.fetch` API. - Make conscious trade-off when following [WHATWG fetch spec][whatwg-fetch] and [stream spec](https://streams.spec.whatwg.org/) implementation details, document known differences. -- Use native promise, but allow substituting it with [insert your favorite promise library]. -- Use native Node streams for body, on both request and response. -- Decode content encoding (gzip/deflate) properly, and convert string output (such as `res.text()` and `res.json()`) to UTF-8 automatically. +- Use native promise but allow substituting it with [insert your favorite promise library]. +- Use native Node streams for body on both request and response. +- Decode content encoding (gzip/deflate) properly and convert string output (such as `res.text()` and `res.json()`) to UTF-8 automatically. - Useful extensions such as timeout, redirect limit, response size limit, [explicit errors](ERROR-HANDLING.md) for troubleshooting. ## Difference from client-side fetch @@ -79,12 +79,12 @@ $ npm install node-fetch ``` ## Loading and configuring the module -We suggest you load the module via `require`, pending the stabalizing of es modules in node: +We suggest you load the module via `require` until the stabilization of ES modules in node: ```js const fetch = require('node-fetch'); ``` -If you are using a Promise library other than native, set it through fetch.Promise: +If you are using a Promise library other than native, set it through `fetch.Promise`: ```js const Bluebird = require('bluebird'); @@ -93,7 +93,7 @@ fetch.Promise = Bluebird; ## Common Usage -NOTE: The documentation below is up-to-date with `2.x` releases, [see `1.x` readme](https://github.com/bitinn/node-fetch/blob/1.x/README.md), [changelog](https://github.com/bitinn/node-fetch/blob/1.x/CHANGELOG.md) and [2.x upgrade guide](UPGRADE-GUIDE.md) for the differences. +NOTE: The documentation below is up-to-date with `2.x` releases; see the [`1.x` readme](https://github.com/bitinn/node-fetch/blob/1.x/README.md), [changelog](https://github.com/bitinn/node-fetch/blob/1.x/CHANGELOG.md) and [2.x upgrade guide](UPGRADE-GUIDE.md) for the differences. #### Plain text or HTML ```js @@ -149,9 +149,9 @@ fetch('https://httpbin.org/post', { method: 'POST', body: params }) ``` #### Handling exceptions -NOTE: 3xx-5xx responses are *NOT* exceptions, and should be handled in `then()`, see the next section. +NOTE: 3xx-5xx responses are *NOT* exceptions and should be handled in `then()`; see the next section for more information. -Adding a catch to the fetch promise chain will catch *all* exceptions, such as errors originating from node core libraries, like network errors, and operational errors which are instances of FetchError. See the [error handling document](ERROR-HANDLING.md) for more details. +Adding a catch to the fetch promise chain will catch *all* exceptions, such as errors originating from node core libraries, network errors and operational errors, which are instances of FetchError. See the [error handling document](ERROR-HANDLING.md) for more details. ```js fetch('https://domain.invalid/') @@ -189,7 +189,7 @@ fetch('https://assets-cdn.github.com/images/modules/logos_page/Octocat.png') ``` #### Buffer -If you prefer to cache binary data in full, use buffer(). (NOTE: buffer() is a `node-fetch` only API) +If you prefer to cache binary data in full, use buffer(). (NOTE: `buffer()` is a `node-fetch`-only API) ```js const fileType = require('file-type'); @@ -214,7 +214,7 @@ fetch('https://github.com/') #### Extract Set-Cookie Header -Unlike browsers, you can access raw `Set-Cookie` headers manually using `Headers.raw()`, this is a `node-fetch` only API. +Unlike browsers, you can access raw `Set-Cookie` headers manually using `Headers.raw()`. This is a `node-fetch` only API. ```js fetch(url).then(res => { @@ -266,11 +266,11 @@ fetch('https://httpbin.org/post', options) #### Request cancellation with AbortSignal -> NOTE: You may only cancel streamed requests on Node >= v8.0.0 +> NOTE: You may cancel streamed requests only on Node >= v8.0.0 You may cancel requests with `AbortController`. A suggested implementation is [`abort-controller`](https://www.npmjs.com/package/abort-controller). -An example of timing out a request after 150ms could be achieved as follows: +An example of timing out a request after 150ms could be achieved as the following: ```js import AbortController from 'abort-controller'; @@ -311,7 +311,7 @@ See [test cases](https://github.com/bitinn/node-fetch/blob/master/test/test.js) Perform an HTTP(S) fetch. -`url` should be an absolute url, such as `https://example.com/`. A path-relative URL (`/file/under/root`) or protocol-relative URL (`//can-be-http-or-https.com/`) will result in a rejected promise. +`url` should be an absolute url, such as `https://example.com/`. A path-relative URL (`/file/under/root`) or protocol-relative URL (`//can-be-http-or-https.com/`) will result in a rejected `Promise`. ### Options @@ -353,7 +353,7 @@ Note: when `body` is a `Stream`, `Content-Length` is not set automatically. ##### Custom Agent -The `agent` option allows you to specify networking related options that's out of the scope of Fetch. Including and not limit to: +The `agent` option allows you to specify networking related options which are out of the scope of Fetch, including and not limited to the following: - Support self-signed certificate - Use only IPv4 or IPv6 @@ -361,7 +361,7 @@ The `agent` option allows you to specify networking related options that's out o See [`http.Agent`](https://nodejs.org/api/http.html#http_new_agent_options) for more information. -In addition, `agent` option accepts a function that returns http(s).Agent instance given current [URL](https://nodejs.org/api/url.html), this is useful during a redirection chain across HTTP and HTTPS protocol. +In addition, the `agent` option accepts a function that returns `http`(s)`.Agent` instance given current [URL](https://nodejs.org/api/url.html), this is useful during a redirection chain across HTTP and HTTPS protocol. ```js const httpAgent = new http.Agent({ @@ -435,7 +435,7 @@ The following properties are not implemented in node-fetch at this moment: *(spec-compliant)* -- `body` A string or [Readable stream][node-readable] +- `body` A `String` or [`Readable` stream][node-readable] - `options` A [`ResponseInit`][response-init] options dictionary Constructs a new `Response` object. The constructor is identical to that in the [browser](https://developer.mozilla.org/en-US/docs/Web/API/Response/Response). @@ -465,7 +465,7 @@ This class allows manipulating and iterating over a set of HTTP headers. All met - `init` Optional argument to pre-fill the `Headers` object -Construct a new `Headers` object. `init` can be either `null`, a `Headers` object, an key-value map object, or any iterable object. +Construct a new `Headers` object. `init` can be either `null`, a `Headers` object, an key-value map object or any iterable object. ```js // Example adapted from https://fetch.spec.whatwg.org/#example-headers-class @@ -506,7 +506,7 @@ The following methods are not yet implemented in node-fetch at this moment: * Node.js [`Readable` stream][node-readable] -The data encapsulated in the `Body` object. Note that while the [Fetch Standard][whatwg-fetch] requires the property to always be a WHATWG `ReadableStream`, in node-fetch it is a Node.js [`Readable` stream][node-readable]. +Data are encapsulated in the `Body` object. Note that while the [Fetch Standard][whatwg-fetch] requires the property to always be a WHATWG `ReadableStream`, in node-fetch it is a Node.js [`Readable` stream][node-readable]. #### body.bodyUsed @@ -514,7 +514,7 @@ The data encapsulated in the `Body` object. Note that while the [Fetch Standard] * `Boolean` -A boolean property for if this body has been consumed. Per spec, a consumed body cannot be used again. +A boolean property for if this body has been consumed. Per the specs, a consumed body cannot be used again. #### body.arrayBuffer() #### body.blob() @@ -541,9 +541,9 @@ Consume the body and return a promise that will resolve to a Buffer. * Returns: Promise<String> -Identical to `body.text()`, except instead of always converting to UTF-8, encoding sniffing will be performed and text converted to UTF-8, if possible. +Identical to `body.text()`, except instead of always converting to UTF-8, encoding sniffing will be performed and text converted to UTF-8 if possible. -(This API requires an optional dependency on npm package [encoding](https://www.npmjs.com/package/encoding), which you need to install manually. `webpack` users may see [a warning message](https://github.com/bitinn/node-fetch/issues/412#issuecomment-379007792) due to this optional dependency.) +(This API requires an optional dependency of the npm package [encoding](https://www.npmjs.com/package/encoding), which you need to install manually. `webpack` users may see [a warning message](https://github.com/bitinn/node-fetch/issues/412#issuecomment-379007792) due to this optional dependency.) ### Class: FetchError From 2358a6c2563d1730a0cdaccc197c611949f6a334 Mon Sep 17 00:00:00 2001 From: Antoni Kepinski Date: Sat, 5 Sep 2020 14:55:39 +0200 Subject: [PATCH 215/223] Honor the `size` option after following a redirect and revert data uri support Co-authored-by: Richie Bendall --- CHANGELOG.md | 6 ++++++ src/index.js | 14 ++------------ test/test.js | 25 ------------------------- 3 files changed, 8 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 188fcd399..543d3d947 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ Changelog # 2.x release +## v2.6.1 + +**This is an important security release. It is strongly recommended to update as soon as possible.** + +- Fix: honor the `size` option after following a redirect. + ## v2.6.0 - Enhance: `options.agent`, it now accepts a function that returns custom http(s).Agent instance based on current URL, see readme for more information. diff --git a/src/index.js b/src/index.js index 8bf9248fd..03b56f733 100644 --- a/src/index.js +++ b/src/index.js @@ -38,17 +38,6 @@ export default function fetch(url, opts) { throw new Error('native promise missing, set fetch.Promise to your favorite alternative'); } - if (/^data:/.test(url)) { - const request = new Request(url, opts); - try { - const data = Buffer.from(url.split(',')[1], 'base64') - const res = new Response(data.body, { headers: { 'Content-Type': data.mimeType || url.match(/^data:(.+);base64,.*$/)[1] } }); - return fetch.Promise.resolve(res); - } catch (err) { - return fetch.Promise.reject(new FetchError(`[${request.method}] ${request.url} invalid URL, ${err.message}`, 'system', err)); - } - } - Body.Promise = fetch.Promise; // wrap http.request into fetch @@ -164,7 +153,8 @@ export default function fetch(url, opts) { method: request.method, body: request.body, signal: request.signal, - timeout: request.timeout + timeout: request.timeout, + size: request.size }; // HTTP-redirect fetch step 9 diff --git a/test/test.js b/test/test.js index c5d61c72a..d3cf2fc97 100644 --- a/test/test.js +++ b/test/test.js @@ -2844,29 +2844,4 @@ describe('external encoding', () => { }); }); }); - - describe('data uri', function() { - const dataUrl = 'data:image/gif;base64,R0lGODlhAQABAIAAAAUEBAAAACwAAAAAAQABAAACAkQBADs='; - - const invalidDataUrl = 'data:@@@@'; - - it('should accept data uri', function() { - return fetch(dataUrl).then(r => { - console.assert(r.status == 200); - console.assert(r.headers.get('Content-Type') == 'image/gif'); - - return r.buffer().then(b => { - console.assert(b instanceof Buffer); - }); - }); - }); - - it('should reject invalid data uri', function() { - return fetch(invalidDataUrl) - .catch(e => { - console.assert(e); - console.assert(e.message.includes('invalid URL')); - }); - }); - }); }); From b5e2e41b2b50bf2997720d6125accaf0dd68c0ab Mon Sep 17 00:00:00 2001 From: Antoni Kepinski Date: Sat, 5 Sep 2020 14:58:33 +0200 Subject: [PATCH 216/223] update version number --- package.json | 128 +++++++++++++++++++++++++-------------------------- 1 file changed, 64 insertions(+), 64 deletions(-) diff --git a/package.json b/package.json index 8e5c883b2..216046916 100644 --- a/package.json +++ b/package.json @@ -1,66 +1,66 @@ { - "name": "node-fetch", - "version": "2.6.0", - "description": "A light-weight module that brings window.fetch to node.js", - "main": "lib/index", - "browser": "./browser.js", - "module": "lib/index.mjs", - "files": [ - "lib/index.js", - "lib/index.mjs", - "lib/index.es.js", - "browser.js" - ], - "engines": { - "node": "4.x || >=6.0.0" - }, - "scripts": { - "build": "cross-env BABEL_ENV=rollup rollup -c", - "prepare": "npm run build", - "test": "cross-env BABEL_ENV=test mocha --require babel-register --throw-deprecation test/test.js", - "report": "cross-env BABEL_ENV=coverage nyc --reporter lcov --reporter text mocha -R spec test/test.js", - "coverage": "cross-env BABEL_ENV=coverage nyc --reporter json --reporter text mocha -R spec test/test.js && codecov -f coverage/coverage-final.json" - }, - "repository": { - "type": "git", - "url": "https://github.com/bitinn/node-fetch.git" - }, - "keywords": [ - "fetch", - "http", - "promise" - ], - "author": "David Frank", - "license": "MIT", - "bugs": { - "url": "https://github.com/bitinn/node-fetch/issues" - }, - "homepage": "https://github.com/bitinn/node-fetch", - "devDependencies": { - "@ungap/url-search-params": "^0.1.2", - "abort-controller": "^1.1.0", - "abortcontroller-polyfill": "^1.3.0", - "babel-core": "^6.26.3", - "babel-plugin-istanbul": "^4.1.6", - "babel-preset-env": "^1.6.1", - "babel-register": "^6.16.3", - "chai": "^3.5.0", - "chai-as-promised": "^7.1.1", - "chai-iterator": "^1.1.1", - "chai-string": "~1.3.0", - "codecov": "^3.3.0", - "cross-env": "^5.2.0", - "form-data": "^2.3.3", - "is-builtin-module": "^1.0.0", - "mocha": "^5.0.0", - "nyc": "11.9.0", - "parted": "^0.1.1", - "promise": "^8.0.3", - "resumer": "0.0.0", - "rollup": "^0.63.4", - "rollup-plugin-babel": "^3.0.7", - "string-to-arraybuffer": "^1.0.2", - "whatwg-url": "^5.0.0" - }, - "dependencies": {} + "name": "node-fetch", + "version": "2.6.1", + "description": "A light-weight module that brings window.fetch to node.js", + "main": "lib/index", + "browser": "./browser.js", + "module": "lib/index.mjs", + "files": [ + "lib/index.js", + "lib/index.mjs", + "lib/index.es.js", + "browser.js" + ], + "engines": { + "node": "4.x || >=6.0.0" + }, + "scripts": { + "build": "cross-env BABEL_ENV=rollup rollup -c", + "prepare": "npm run build", + "test": "cross-env BABEL_ENV=test mocha --require babel-register --throw-deprecation test/test.js", + "report": "cross-env BABEL_ENV=coverage nyc --reporter lcov --reporter text mocha -R spec test/test.js", + "coverage": "cross-env BABEL_ENV=coverage nyc --reporter json --reporter text mocha -R spec test/test.js && codecov -f coverage/coverage-final.json" + }, + "repository": { + "type": "git", + "url": "https://github.com/bitinn/node-fetch.git" + }, + "keywords": [ + "fetch", + "http", + "promise" + ], + "author": "David Frank", + "license": "MIT", + "bugs": { + "url": "https://github.com/bitinn/node-fetch/issues" + }, + "homepage": "https://github.com/bitinn/node-fetch", + "devDependencies": { + "@ungap/url-search-params": "^0.1.2", + "abort-controller": "^1.1.0", + "abortcontroller-polyfill": "^1.3.0", + "babel-core": "^6.26.3", + "babel-plugin-istanbul": "^4.1.6", + "babel-preset-env": "^1.6.1", + "babel-register": "^6.16.3", + "chai": "^3.5.0", + "chai-as-promised": "^7.1.1", + "chai-iterator": "^1.1.1", + "chai-string": "~1.3.0", + "codecov": "^3.3.0", + "cross-env": "^5.2.0", + "form-data": "^2.3.3", + "is-builtin-module": "^1.0.0", + "mocha": "^5.0.0", + "nyc": "11.9.0", + "parted": "^0.1.1", + "promise": "^8.0.3", + "resumer": "0.0.0", + "rollup": "^0.63.4", + "rollup-plugin-babel": "^3.0.7", + "string-to-arraybuffer": "^1.0.2", + "whatwg-url": "^5.0.0" + }, + "dependencies": {} } From 152214ca2f6e2a5a17d71e4638114625d3be30c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jimmy=20W=C3=A4rting?= Date: Mon, 6 Sep 2021 14:58:28 +0200 Subject: [PATCH 217/223] Fix(package.json): Corrected main file path in package.json (#1274) * fix main configuration in package.json * pinned a breaking change in codecov & teeny-request --- CHANGELOG.md | 5 +++++ package.json | 10 +++++----- rollup.config.js | 6 +----- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 543d3d947..812a96309 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ Changelog # 2.x release +## v2.6.2 + +- Fix: used full filename for main in package.json +- Other: pinned codecov & teeny-request (had one breaking change with spread operators) + ## v2.6.1 **This is an important security release. It is strongly recommended to update as soon as possible.** diff --git a/package.json b/package.json index 216046916..4bb7d9640 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,8 @@ { "name": "node-fetch", - "version": "2.6.1", + "version": "2.6.2", "description": "A light-weight module that brings window.fetch to node.js", - "main": "lib/index", + "main": "lib/index.js", "browser": "./browser.js", "module": "lib/index.mjs", "files": [ @@ -48,7 +48,7 @@ "chai-as-promised": "^7.1.1", "chai-iterator": "^1.1.1", "chai-string": "~1.3.0", - "codecov": "^3.3.0", + "codecov": "3.3.0", "cross-env": "^5.2.0", "form-data": "^2.3.3", "is-builtin-module": "^1.0.0", @@ -60,7 +60,7 @@ "rollup": "^0.63.4", "rollup-plugin-babel": "^3.0.7", "string-to-arraybuffer": "^1.0.2", + "teeny-request": "3.7.0", "whatwg-url": "^5.0.0" - }, - "dependencies": {} + } } diff --git a/rollup.config.js b/rollup.config.js index a201ee455..1bc88f8db 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -18,10 +18,6 @@ export default { tweakDefault() ], external: function (id) { - if (isBuiltin(id)) { - return true; - } - id = id.split('/').slice(0, id[0] === '@' ? 2 : 1).join('/'); - return !!require('./package.json').dependencies[id]; + return isBuiltin(id); } }; From ace7536c955556be742d9910566738630cc3c2a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20Unneb=C3=A4ck?= Date: Mon, 20 Sep 2021 16:09:10 +0200 Subject: [PATCH 218/223] fix: properly encode url with unicode characters (#1291) * fix: properly encode url with unicode characters * release: 2.6.3 --- CHANGELOG.md | 4 ++++ package.json | 8 +++++--- rollup.config.js | 5 ++++- src/request.js | 40 +++++++++++++++++++++++++++++++++++++--- test/server.js | 8 +++++++- test/test.js | 22 ++++++++++++++++++++++ 6 files changed, 79 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 812a96309..671da7654 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ Changelog # 2.x release +## v2.6.3 + +- Fix: properly encode url with unicode characters + ## v2.6.2 - Fix: used full filename for main in package.json diff --git a/package.json b/package.json index 4bb7d9640..c5edc7991 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.6.2", + "version": "2.6.3", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", "browser": "./browser.js", @@ -36,6 +36,9 @@ "url": "https://github.com/bitinn/node-fetch/issues" }, "homepage": "https://github.com/bitinn/node-fetch", + "dependencies": { + "whatwg-url": "^5.0.0" + }, "devDependencies": { "@ungap/url-search-params": "^0.1.2", "abort-controller": "^1.1.0", @@ -60,7 +63,6 @@ "rollup": "^0.63.4", "rollup-plugin-babel": "^3.0.7", "string-to-arraybuffer": "^1.0.2", - "teeny-request": "3.7.0", - "whatwg-url": "^5.0.0" + "teeny-request": "3.7.0" } } diff --git a/rollup.config.js b/rollup.config.js index 1bc88f8db..d5951bd2e 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -1,9 +1,12 @@ import isBuiltin from 'is-builtin-module'; import babel from 'rollup-plugin-babel'; +import packageJson from './package.json'; import tweakDefault from './build/rollup-plugin'; process.env.BABEL_ENV = 'rollup'; +const dependencies = Object.keys(packageJson.dependencies); + export default { input: 'src/index.js', output: [ @@ -18,6 +21,6 @@ export default { tweakDefault() ], external: function (id) { - return isBuiltin(id); + return dependencies.includes(id) || isBuiltin(id); } }; diff --git a/src/request.js b/src/request.js index 45a7eb7e4..6fa8e77b6 100644 --- a/src/request.js +++ b/src/request.js @@ -9,6 +9,7 @@ import Url from 'url'; import Stream from 'stream'; +import {URL} from 'whatwg-url'; import Headers, { exportNodeCompatibleHeaders } from './headers.js'; import Body, { clone, extractContentType, getTotalBytes } from './body'; @@ -18,6 +19,39 @@ const INTERNALS = Symbol('Request internals'); const parse_url = Url.parse; const format_url = Url.format; +/** + * Wrapper around `new URL` to handle arbitrary URLs + * + * @param {string} urlStr + * @return {void} + */ +function parseURL(urlStr) { + /* + Check whether the URL is absolute or not + + Scheme: https://tools.ietf.org/html/rfc3986#section-3.1 + Absolute URL: https://tools.ietf.org/html/rfc3986#section-4.3 + */ + if (/^[a-zA-Z][a-zA-Z\d+\-.]*:/.exec(urlStr)) { + const url = new URL(urlStr); + + return { + path: url.pathname, + pathname: url.pathname, + hostname: url.hostname, + protocol: url.protocol, + port: url.port, + hash: url.hash, + search: url.search, + query: url.query, + href: url.href, + } + } + + // Fallback to old implementation for arbitrary URLs + return parse_url(urlStr); +} + const streamDestructionSupported = 'destroy' in Stream.Readable.prototype; /** @@ -59,14 +93,14 @@ export default class Request { // in order to support Node.js' Url objects; though WHATWG's URL objects // will fall into this branch also (since their `toString()` will return // `href` property anyway) - parsedURL = parse_url(input.href); + parsedURL = parseURL(input.href); } else { // coerce input to a string before attempting to parse - parsedURL = parse_url(`${input}`); + parsedURL = parseURL(`${input}`); } input = {}; } else { - parsedURL = parse_url(input.url); + parsedURL = parseURL(input.url); } let method = init.method || input.method || 'GET'; diff --git a/test/server.js b/test/server.js index 06c715d65..ebd311d9c 100644 --- a/test/server.js +++ b/test/server.js @@ -32,7 +32,7 @@ export default class TestServer { } router(req, res) { - let p = parse(req.url).pathname; + let p = decodeURIComponent(parse(req.url).pathname); if (p === '/hello') { res.statusCode = 200; @@ -384,6 +384,12 @@ export default class TestServer { }); req.pipe(parser); } + + if (p === '/issues/1290/ひらがな') { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/plain'); + res.end('Success'); + } } } diff --git a/test/test.js b/test/test.js index d3cf2fc97..9220cbd4a 100644 --- a/test/test.js +++ b/test/test.js @@ -2845,3 +2845,25 @@ describe('external encoding', () => { }); }); }); + +describe('issue #1290', function() { + it('should handle escaped unicode in URLs', () => { + const url = `${base}issues/1290/%E3%81%B2%E3%82%89%E3%81%8C%E3%81%AA`; + return fetch(url).then((res) => { + expect(res.status).to.equal(200); + return res.text().then(result => { + expect(result).to.equal('Success'); + }); + }); + }); + + it('should handle unicode in URLs', () => { + const url = `${base}issues/1290/ひらがな`; + return fetch(url).then((res) => { + expect(res.status).to.equal(200); + return res.text().then(result => { + expect(result).to.equal('Success'); + }); + }); + }); +}); From 18193c5922c64046b922e18faf41821290535f06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jimmy=20W=C3=A4rting?= Date: Tue, 21 Sep 2021 16:42:50 +0200 Subject: [PATCH 219/223] fix v2.6.3 that did not sending query params (#1301) --- CHANGELOG.md | 4 ++++ package.json | 2 +- src/request.js | 14 +------------- test/test.js | 9 +++++++++ 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 671da7654..46eef0ff0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ Changelog # 2.x release +## v2.6.4 + +- Hotfix: fix v2.6.3 that did not sending query params + ## v2.6.3 - Fix: properly encode url with unicode characters diff --git a/package.json b/package.json index c5edc7991..98cf5f4c5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.6.3", + "version": "2.6.4", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", "browser": "./browser.js", diff --git a/src/request.js b/src/request.js index 6fa8e77b6..3c27c81cf 100644 --- a/src/request.js +++ b/src/request.js @@ -33,19 +33,7 @@ function parseURL(urlStr) { Absolute URL: https://tools.ietf.org/html/rfc3986#section-4.3 */ if (/^[a-zA-Z][a-zA-Z\d+\-.]*:/.exec(urlStr)) { - const url = new URL(urlStr); - - return { - path: url.pathname, - pathname: url.pathname, - hostname: url.hostname, - protocol: url.protocol, - port: url.port, - hash: url.hash, - search: url.search, - query: url.query, - href: url.href, - } + urlStr = new URL(urlStr).toString() } // Fallback to old implementation for arbitrary URLs diff --git a/test/test.js b/test/test.js index 9220cbd4a..9568489e3 100644 --- a/test/test.js +++ b/test/test.js @@ -2847,6 +2847,15 @@ describe('external encoding', () => { }); describe('issue #1290', function() { + + it('should keep query params', function() { + return fetch(`${base}inspect?month=2021-09`) + .then(res => res.json()) + .then(json => { + expect(json.url).to.equal('/inspect?month=2021-09') + }) + }) + it('should handle escaped unicode in URLs', () => { const url = `${base}issues/1290/%E3%81%B2%E3%82%89%E3%81%8C%E3%81%AA`; return fetch(url).then((res) => { From b5417aea6a3275932283a200214522e6ab53f1ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20Unneb=C3=A4ck?= Date: Wed, 22 Sep 2021 11:16:53 +0200 Subject: [PATCH 220/223] fix: import whatwg-url in a way compatible with ESM Node (#1303) * fix: import whatwg-url in a way compatible with ESM Node * release: 2.6.5 --- CHANGELOG.md | 4 ++++ package.json | 2 +- src/request.js | 3 ++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46eef0ff0..29d168cf7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ Changelog # 2.x release +## v2.6.5 + +- Fix: import `whatwg-url` in a way compatible with ESM + ## v2.6.4 - Hotfix: fix v2.6.3 that did not sending query params diff --git a/package.json b/package.json index 98cf5f4c5..4178c1a32 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.6.4", + "version": "2.6.5", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", "browser": "./browser.js", diff --git a/src/request.js b/src/request.js index 3c27c81cf..59850fd95 100644 --- a/src/request.js +++ b/src/request.js @@ -9,11 +9,12 @@ import Url from 'url'; import Stream from 'stream'; -import {URL} from 'whatwg-url'; +import whatwgUrl from 'whatwg-url'; import Headers, { exportNodeCompatibleHeaders } from './headers.js'; import Body, { clone, extractContentType, getTotalBytes } from './body'; const INTERNALS = Symbol('Request internals'); +const URL = whatwgUrl.URL; // fix an issue where "format", "parse" aren't a named export for node <10 const parse_url = Url.parse; From f56b0c66d3dd2ef185436de1f2fd40f66bfea8f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jimmy=20W=C3=A4rting?= Date: Sun, 31 Oct 2021 16:40:17 +0100 Subject: [PATCH 221/223] fix(URL): prefer built in URL version when available and fallback to whatwg (#1352) * fix(URL): prefer built in URL version when available and fallback to whatwg * bump minor --- package.json | 2 +- src/request.js | 2 +- test/test.js | 7 +++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 4178c1a32..ec0510513 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.6.5", + "version": "2.6.6", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", "browser": "./browser.js", diff --git a/src/request.js b/src/request.js index 59850fd95..739ba9071 100644 --- a/src/request.js +++ b/src/request.js @@ -14,7 +14,7 @@ import Headers, { exportNodeCompatibleHeaders } from './headers.js'; import Body, { clone, extractContentType, getTotalBytes } from './body'; const INTERNALS = Symbol('Request internals'); -const URL = whatwgUrl.URL; +const URL = Url.URL || whatwgUrl.URL; // fix an issue where "format", "parse" aren't a named export for node <10 const parse_url = Url.parse; diff --git a/test/test.js b/test/test.js index 9568489e3..6427ae21e 100644 --- a/test/test.js +++ b/test/test.js @@ -2875,4 +2875,11 @@ describe('issue #1290', function() { }); }); }); + + // #1342 + it('should not throw with a valid URL', () => { + const url = 'https://r2---sn-n4v7sney.example.com'; + new Request(url); + }); + }); From 8fe5c4ea66b9b8187600e6d5ec9b1b6781f44009 Mon Sep 17 00:00:00 2001 From: Ciffelia Date: Fri, 5 Nov 2021 22:42:53 +0900 Subject: [PATCH 222/223] 2.x: Specify encoding as an optional peer dependency in package.json (#1310) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Specify `encoding` as an optional peer dependency * Update package.json Co-authored-by: Linus Unnebäck Co-authored-by: Linus Unnebäck --- package.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/package.json b/package.json index ec0510513..6f0ac4302 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,14 @@ "dependencies": { "whatwg-url": "^5.0.0" }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + }, "devDependencies": { "@ungap/url-search-params": "^0.1.2", "abort-controller": "^1.1.0", From 1ef4b560a17e644a02a3bfdea7631ffeee578b35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jimmy=20W=C3=A4rting?= Date: Sun, 16 Jan 2022 12:45:33 +0100 Subject: [PATCH 223/223] backport of #1449 (#1453) * backport of #1449 * bump patch version --- package.json | 2 +- src/index.js | 49 ++++++++++++++++++++++++++++++++++++++++--------- test/server.js | 7 ++++++- test/test.js | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 94 insertions(+), 11 deletions(-) diff --git a/package.json b/package.json index 6f0ac4302..3c1bd8da7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-fetch", - "version": "2.6.6", + "version": "2.6.7", "description": "A light-weight module that brings window.fetch to node.js", "main": "lib/index.js", "browser": "./browser.js", diff --git a/src/index.js b/src/index.js index 03b56f733..b210d28e4 100644 --- a/src/index.js +++ b/src/index.js @@ -13,16 +13,29 @@ import https from 'https'; import zlib from 'zlib'; import Stream from 'stream'; -import Body, { writeToStream, getTotalBytes } from './body'; -import Response from './response'; -import Headers, { createHeadersLenient } from './headers'; -import Request, { getNodeRequestOptions } from './request'; -import FetchError from './fetch-error'; -import AbortError from './abort-error'; +import Body, { writeToStream, getTotalBytes } from './body.js'; +import Response from './response.js'; +import Headers, { createHeadersLenient } from './headers.js'; +import Request, { getNodeRequestOptions } from './request.js'; +import FetchError from './fetch-error.js'; +import AbortError from './abort-error.js'; + +import whatwgUrl from 'whatwg-url'; + +const URL = Url.URL || whatwgUrl.URL; // fix an issue where "PassThrough", "resolve" aren't a named export for node <10 const PassThrough = Stream.PassThrough; -const resolve_url = Url.resolve; + +const isDomainOrSubdomain = (destination, original) => { + const orig = new URL(original).hostname; + const dest = new URL(destination).hostname; + + return orig === dest || ( + orig[orig.length - dest.length - 1] === '.' && orig.endsWith(dest) + ); +}; + /** * Fetch function @@ -109,7 +122,19 @@ export default function fetch(url, opts) { const location = headers.get('Location'); // HTTP fetch step 5.3 - const locationURL = location === null ? null : resolve_url(request.url, location); + let locationURL = null; + try { + locationURL = location === null ? null : new URL(location, request.url).toString(); + } catch (err) { + // error here can only be invalid URL in Location: header + // do not throw when options.redirect == manual + // let the user extract the errorneous redirect URL + if (request.redirect !== 'manual') { + reject(new FetchError(`uri requested responds with an invalid redirect URL: ${location}`, 'invalid-redirect')); + finalize(); + return; + } + } // HTTP fetch step 5.5 switch (request.redirect) { @@ -154,9 +179,15 @@ export default function fetch(url, opts) { body: request.body, signal: request.signal, timeout: request.timeout, - size: request.size + size: request.size }; + if (!isDomainOrSubdomain(request.url, locationURL)) { + for (const name of ['authorization', 'www-authenticate', 'cookie', 'cookie2']) { + requestOpts.headers.delete(name); + } + } + // HTTP-redirect fetch step 9 if (res.statusCode !== 303 && request.body && getTotalBytes(request) === null) { reject(new FetchError('Cannot follow redirect with body being a readable stream', 'unsupported-redirect')); diff --git a/test/server.js b/test/server.js index ebd311d9c..2f0baf8cd 100644 --- a/test/server.js +++ b/test/server.js @@ -1,7 +1,6 @@ import * as http from 'http'; import { parse } from 'url'; import * as zlib from 'zlib'; -import * as stream from 'stream'; import { multipart as Multipart } from 'parted'; let convert; @@ -66,6 +65,12 @@ export default class TestServer { })); } + if (p.startsWith('/redirect-to/3')) { + res.statusCode = p.slice(13, 16); + res.setHeader('Location', p.slice(17)); + res.end(); + } + if (p === '/gzip') { res.statusCode = 200; res.setHeader('Content-Type', 'text/plain'); diff --git a/test/test.js b/test/test.js index 6427ae21e..cdeb51f00 100644 --- a/test/test.js +++ b/test/test.js @@ -1569,6 +1569,53 @@ describe('node-fetch', () => { }); }); + it('should not forward secure headers to 3th party', () => { + return fetch(`${base}redirect-to/302/https://httpbin.org/get`, { + headers: new Headers({ + cookie: 'gets=removed', + cookie2: 'gets=removed', + authorization: 'gets=removed', + 'www-authenticate': 'gets=removed', + 'other-safe-headers': 'stays', + 'x-foo': 'bar' + }) + }).then(res => res.json()).then(json => { + const headers = new Headers(json.headers); + // Safe headers are not removed + expect(headers.get('other-safe-headers')).to.equal('stays'); + expect(headers.get('x-foo')).to.equal('bar'); + // Unsafe headers should not have been sent to httpbin + expect(headers.get('cookie')).to.equal(null); + expect(headers.get('cookie2')).to.equal(null); + expect(headers.get('www-authenticate')).to.equal(null); + expect(headers.get('authorization')).to.equal(null); + }); + }); + + it('should forward secure headers to same host', () => { + return fetch(`${base}redirect-to/302/${base}inspect`, { + headers: new Headers({ + cookie: 'is=cookie', + cookie2: 'is=cookie2', + authorization: 'is=authorization', + 'other-safe-headers': 'stays', + 'www-authenticate': 'is=www-authenticate', + 'x-foo': 'bar' + }) + }).then(res => res.json().then(json => { + const headers = new Headers(json.headers); + // Safe headers are not removed + expect(res.url).to.equal(`${base}inspect`); + expect(headers.get('other-safe-headers')).to.equal('stays'); + expect(headers.get('x-foo')).to.equal('bar'); + // Unsafe headers should not have been sent to httpbin + expect(headers.get('cookie')).to.equal('is=cookie'); + expect(headers.get('cookie2')).to.equal('is=cookie2'); + expect(headers.get('www-authenticate')).to.equal('is=www-authenticate'); + expect(headers.get('authorization')).to.equal('is=authorization'); + })); + }); + it('should allow PATCH request', function() { const url = `${base}inspect`; const opts = {