index.js 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621
  1. var url = require("url");
  2. var URL = url.URL;
  3. var http = require("http");
  4. var https = require("https");
  5. var Writable = require("stream").Writable;
  6. var assert = require("assert");
  7. var debug = require("./debug");
  8. // Create handlers that pass events from native requests
  9. var events = ["abort", "aborted", "connect", "error", "socket", "timeout"];
  10. var eventHandlers = Object.create(null);
  11. events.forEach(function (event) {
  12. eventHandlers[event] = function (arg1, arg2, arg3) {
  13. this._redirectable.emit(event, arg1, arg2, arg3);
  14. };
  15. });
  16. var InvalidUrlError = createErrorType(
  17. "ERR_INVALID_URL",
  18. "Invalid URL",
  19. TypeError
  20. );
  21. // Error types with codes
  22. var RedirectionError = createErrorType(
  23. "ERR_FR_REDIRECTION_FAILURE",
  24. "Redirected request failed"
  25. );
  26. var TooManyRedirectsError = createErrorType(
  27. "ERR_FR_TOO_MANY_REDIRECTS",
  28. "Maximum number of redirects exceeded"
  29. );
  30. var MaxBodyLengthExceededError = createErrorType(
  31. "ERR_FR_MAX_BODY_LENGTH_EXCEEDED",
  32. "Request body larger than maxBodyLength limit"
  33. );
  34. var WriteAfterEndError = createErrorType(
  35. "ERR_STREAM_WRITE_AFTER_END",
  36. "write after end"
  37. );
  38. // An HTTP(S) request that can be redirected
  39. function RedirectableRequest(options, responseCallback) {
  40. // Initialize the request
  41. Writable.call(this);
  42. this._sanitizeOptions(options);
  43. this._options = options;
  44. this._ended = false;
  45. this._ending = false;
  46. this._redirectCount = 0;
  47. this._redirects = [];
  48. this._requestBodyLength = 0;
  49. this._requestBodyBuffers = [];
  50. // Attach a callback if passed
  51. if (responseCallback) {
  52. this.on("response", responseCallback);
  53. }
  54. // React to responses of native requests
  55. var self = this;
  56. this._onNativeResponse = function (response) {
  57. self._processResponse(response);
  58. };
  59. // Perform the first request
  60. this._performRequest();
  61. }
  62. RedirectableRequest.prototype = Object.create(Writable.prototype);
  63. RedirectableRequest.prototype.abort = function () {
  64. abortRequest(this._currentRequest);
  65. this.emit("abort");
  66. };
  67. // Writes buffered data to the current native request
  68. RedirectableRequest.prototype.write = function (data, encoding, callback) {
  69. // Writing is not allowed if end has been called
  70. if (this._ending) {
  71. throw new WriteAfterEndError();
  72. }
  73. // Validate input and shift parameters if necessary
  74. if (!isString(data) && !isBuffer(data)) {
  75. throw new TypeError("data should be a string, Buffer or Uint8Array");
  76. }
  77. if (isFunction(encoding)) {
  78. callback = encoding;
  79. encoding = null;
  80. }
  81. // Ignore empty buffers, since writing them doesn't invoke the callback
  82. // https://github.com/nodejs/node/issues/22066
  83. if (data.length === 0) {
  84. if (callback) {
  85. callback();
  86. }
  87. return;
  88. }
  89. // Only write when we don't exceed the maximum body length
  90. if (this._requestBodyLength + data.length <= this._options.maxBodyLength) {
  91. this._requestBodyLength += data.length;
  92. this._requestBodyBuffers.push({ data: data, encoding: encoding });
  93. this._currentRequest.write(data, encoding, callback);
  94. }
  95. // Error when we exceed the maximum body length
  96. else {
  97. this.emit("error", new MaxBodyLengthExceededError());
  98. this.abort();
  99. }
  100. };
  101. // Ends the current native request
  102. RedirectableRequest.prototype.end = function (data, encoding, callback) {
  103. // Shift parameters if necessary
  104. if (isFunction(data)) {
  105. callback = data;
  106. data = encoding = null;
  107. }
  108. else if (isFunction(encoding)) {
  109. callback = encoding;
  110. encoding = null;
  111. }
  112. // Write data if needed and end
  113. if (!data) {
  114. this._ended = this._ending = true;
  115. this._currentRequest.end(null, null, callback);
  116. }
  117. else {
  118. var self = this;
  119. var currentRequest = this._currentRequest;
  120. this.write(data, encoding, function () {
  121. self._ended = true;
  122. currentRequest.end(null, null, callback);
  123. });
  124. this._ending = true;
  125. }
  126. };
  127. // Sets a header value on the current native request
  128. RedirectableRequest.prototype.setHeader = function (name, value) {
  129. this._options.headers[name] = value;
  130. this._currentRequest.setHeader(name, value);
  131. };
  132. // Clears a header value on the current native request
  133. RedirectableRequest.prototype.removeHeader = function (name) {
  134. delete this._options.headers[name];
  135. this._currentRequest.removeHeader(name);
  136. };
  137. // Global timeout for all underlying requests
  138. RedirectableRequest.prototype.setTimeout = function (msecs, callback) {
  139. var self = this;
  140. // Destroys the socket on timeout
  141. function destroyOnTimeout(socket) {
  142. socket.setTimeout(msecs);
  143. socket.removeListener("timeout", socket.destroy);
  144. socket.addListener("timeout", socket.destroy);
  145. }
  146. // Sets up a timer to trigger a timeout event
  147. function startTimer(socket) {
  148. if (self._timeout) {
  149. clearTimeout(self._timeout);
  150. }
  151. self._timeout = setTimeout(function () {
  152. self.emit("timeout");
  153. clearTimer();
  154. }, msecs);
  155. destroyOnTimeout(socket);
  156. }
  157. // Stops a timeout from triggering
  158. function clearTimer() {
  159. // Clear the timeout
  160. if (self._timeout) {
  161. clearTimeout(self._timeout);
  162. self._timeout = null;
  163. }
  164. // Clean up all attached listeners
  165. self.removeListener("abort", clearTimer);
  166. self.removeListener("error", clearTimer);
  167. self.removeListener("response", clearTimer);
  168. if (callback) {
  169. self.removeListener("timeout", callback);
  170. }
  171. if (!self.socket) {
  172. self._currentRequest.removeListener("socket", startTimer);
  173. }
  174. }
  175. // Attach callback if passed
  176. if (callback) {
  177. this.on("timeout", callback);
  178. }
  179. // Start the timer if or when the socket is opened
  180. if (this.socket) {
  181. startTimer(this.socket);
  182. }
  183. else {
  184. this._currentRequest.once("socket", startTimer);
  185. }
  186. // Clean up on events
  187. this.on("socket", destroyOnTimeout);
  188. this.on("abort", clearTimer);
  189. this.on("error", clearTimer);
  190. this.on("response", clearTimer);
  191. return this;
  192. };
  193. // Proxy all other public ClientRequest methods
  194. [
  195. "flushHeaders", "getHeader",
  196. "setNoDelay", "setSocketKeepAlive",
  197. ].forEach(function (method) {
  198. RedirectableRequest.prototype[method] = function (a, b) {
  199. return this._currentRequest[method](a, b);
  200. };
  201. });
  202. // Proxy all public ClientRequest properties
  203. ["aborted", "connection", "socket"].forEach(function (property) {
  204. Object.defineProperty(RedirectableRequest.prototype, property, {
  205. get: function () { return this._currentRequest[property]; },
  206. });
  207. });
  208. RedirectableRequest.prototype._sanitizeOptions = function (options) {
  209. // Ensure headers are always present
  210. if (!options.headers) {
  211. options.headers = {};
  212. }
  213. // Since http.request treats host as an alias of hostname,
  214. // but the url module interprets host as hostname plus port,
  215. // eliminate the host property to avoid confusion.
  216. if (options.host) {
  217. // Use hostname if set, because it has precedence
  218. if (!options.hostname) {
  219. options.hostname = options.host;
  220. }
  221. delete options.host;
  222. }
  223. // Complete the URL object when necessary
  224. if (!options.pathname && options.path) {
  225. var searchPos = options.path.indexOf("?");
  226. if (searchPos < 0) {
  227. options.pathname = options.path;
  228. }
  229. else {
  230. options.pathname = options.path.substring(0, searchPos);
  231. options.search = options.path.substring(searchPos);
  232. }
  233. }
  234. };
  235. // Executes the next native request (initial or redirect)
  236. RedirectableRequest.prototype._performRequest = function () {
  237. // Load the native protocol
  238. var protocol = this._options.protocol;
  239. var nativeProtocol = this._options.nativeProtocols[protocol];
  240. if (!nativeProtocol) {
  241. this.emit("error", new TypeError("Unsupported protocol " + protocol));
  242. return;
  243. }
  244. // If specified, use the agent corresponding to the protocol
  245. // (HTTP and HTTPS use different types of agents)
  246. if (this._options.agents) {
  247. var scheme = protocol.slice(0, -1);
  248. this._options.agent = this._options.agents[scheme];
  249. }
  250. // Create the native request and set up its event handlers
  251. var request = this._currentRequest =
  252. nativeProtocol.request(this._options, this._onNativeResponse);
  253. request._redirectable = this;
  254. for (var event of events) {
  255. request.on(event, eventHandlers[event]);
  256. }
  257. // RFC7230§5.3.1: When making a request directly to an origin server, […]
  258. // a client MUST send only the absolute path […] as the request-target.
  259. this._currentUrl = /^\//.test(this._options.path) ?
  260. url.format(this._options) :
  261. // When making a request to a proxy, […]
  262. // a client MUST send the target URI in absolute-form […].
  263. this._options.path;
  264. // End a redirected request
  265. // (The first request must be ended explicitly with RedirectableRequest#end)
  266. if (this._isRedirect) {
  267. // Write the request entity and end
  268. var i = 0;
  269. var self = this;
  270. var buffers = this._requestBodyBuffers;
  271. (function writeNext(error) {
  272. // Only write if this request has not been redirected yet
  273. /* istanbul ignore else */
  274. if (request === self._currentRequest) {
  275. // Report any write errors
  276. /* istanbul ignore if */
  277. if (error) {
  278. self.emit("error", error);
  279. }
  280. // Write the next buffer if there are still left
  281. else if (i < buffers.length) {
  282. var buffer = buffers[i++];
  283. /* istanbul ignore else */
  284. if (!request.finished) {
  285. request.write(buffer.data, buffer.encoding, writeNext);
  286. }
  287. }
  288. // End the request if `end` has been called on us
  289. else if (self._ended) {
  290. request.end();
  291. }
  292. }
  293. }());
  294. }
  295. };
  296. // Processes a response from the current native request
  297. RedirectableRequest.prototype._processResponse = function (response) {
  298. // Store the redirected response
  299. var statusCode = response.statusCode;
  300. if (this._options.trackRedirects) {
  301. this._redirects.push({
  302. url: this._currentUrl,
  303. headers: response.headers,
  304. statusCode: statusCode,
  305. });
  306. }
  307. // RFC7231§6.4: The 3xx (Redirection) class of status code indicates
  308. // that further action needs to be taken by the user agent in order to
  309. // fulfill the request. If a Location header field is provided,
  310. // the user agent MAY automatically redirect its request to the URI
  311. // referenced by the Location field value,
  312. // even if the specific status code is not understood.
  313. // If the response is not a redirect; return it as-is
  314. var location = response.headers.location;
  315. if (!location || this._options.followRedirects === false ||
  316. statusCode < 300 || statusCode >= 400) {
  317. response.responseUrl = this._currentUrl;
  318. response.redirects = this._redirects;
  319. this.emit("response", response);
  320. // Clean up
  321. this._requestBodyBuffers = [];
  322. return;
  323. }
  324. // The response is a redirect, so abort the current request
  325. abortRequest(this._currentRequest);
  326. // Discard the remainder of the response to avoid waiting for data
  327. response.destroy();
  328. // RFC7231§6.4: A client SHOULD detect and intervene
  329. // in cyclical redirections (i.e., "infinite" redirection loops).
  330. if (++this._redirectCount > this._options.maxRedirects) {
  331. this.emit("error", new TooManyRedirectsError());
  332. return;
  333. }
  334. // Store the request headers if applicable
  335. var requestHeaders;
  336. var beforeRedirect = this._options.beforeRedirect;
  337. if (beforeRedirect) {
  338. requestHeaders = Object.assign({
  339. // The Host header was set by nativeProtocol.request
  340. Host: response.req.getHeader("host"),
  341. }, this._options.headers);
  342. }
  343. // RFC7231§6.4: Automatic redirection needs to done with
  344. // care for methods not known to be safe, […]
  345. // RFC7231§6.4.2–3: For historical reasons, a user agent MAY change
  346. // the request method from POST to GET for the subsequent request.
  347. var method = this._options.method;
  348. if ((statusCode === 301 || statusCode === 302) && this._options.method === "POST" ||
  349. // RFC7231§6.4.4: The 303 (See Other) status code indicates that
  350. // the server is redirecting the user agent to a different resource […]
  351. // A user agent can perform a retrieval request targeting that URI
  352. // (a GET or HEAD request if using HTTP) […]
  353. (statusCode === 303) && !/^(?:GET|HEAD)$/.test(this._options.method)) {
  354. this._options.method = "GET";
  355. // Drop a possible entity and headers related to it
  356. this._requestBodyBuffers = [];
  357. removeMatchingHeaders(/^content-/i, this._options.headers);
  358. }
  359. // Drop the Host header, as the redirect might lead to a different host
  360. var currentHostHeader = removeMatchingHeaders(/^host$/i, this._options.headers);
  361. // If the redirect is relative, carry over the host of the last request
  362. var currentUrlParts = url.parse(this._currentUrl);
  363. var currentHost = currentHostHeader || currentUrlParts.host;
  364. var currentUrl = /^\w+:/.test(location) ? this._currentUrl :
  365. url.format(Object.assign(currentUrlParts, { host: currentHost }));
  366. // Determine the URL of the redirection
  367. var redirectUrl;
  368. try {
  369. redirectUrl = url.resolve(currentUrl, location);
  370. }
  371. catch (cause) {
  372. this.emit("error", new RedirectionError({ cause: cause }));
  373. return;
  374. }
  375. // Create the redirected request
  376. debug("redirecting to", redirectUrl);
  377. this._isRedirect = true;
  378. var redirectUrlParts = url.parse(redirectUrl);
  379. Object.assign(this._options, redirectUrlParts);
  380. // Drop confidential headers when redirecting to a less secure protocol
  381. // or to a different domain that is not a superdomain
  382. if (redirectUrlParts.protocol !== currentUrlParts.protocol &&
  383. redirectUrlParts.protocol !== "https:" ||
  384. redirectUrlParts.host !== currentHost &&
  385. !isSubdomain(redirectUrlParts.host, currentHost)) {
  386. removeMatchingHeaders(/^(?:authorization|cookie)$/i, this._options.headers);
  387. }
  388. // Evaluate the beforeRedirect callback
  389. if (isFunction(beforeRedirect)) {
  390. var responseDetails = {
  391. headers: response.headers,
  392. statusCode: statusCode,
  393. };
  394. var requestDetails = {
  395. url: currentUrl,
  396. method: method,
  397. headers: requestHeaders,
  398. };
  399. try {
  400. beforeRedirect(this._options, responseDetails, requestDetails);
  401. }
  402. catch (err) {
  403. this.emit("error", err);
  404. return;
  405. }
  406. this._sanitizeOptions(this._options);
  407. }
  408. // Perform the redirected request
  409. try {
  410. this._performRequest();
  411. }
  412. catch (cause) {
  413. this.emit("error", new RedirectionError({ cause: cause }));
  414. }
  415. };
  416. // Wraps the key/value object of protocols with redirect functionality
  417. function wrap(protocols) {
  418. // Default settings
  419. var exports = {
  420. maxRedirects: 21,
  421. maxBodyLength: 10 * 1024 * 1024,
  422. };
  423. // Wrap each protocol
  424. var nativeProtocols = {};
  425. Object.keys(protocols).forEach(function (scheme) {
  426. var protocol = scheme + ":";
  427. var nativeProtocol = nativeProtocols[protocol] = protocols[scheme];
  428. var wrappedProtocol = exports[scheme] = Object.create(nativeProtocol);
  429. // Executes a request, following redirects
  430. function request(input, options, callback) {
  431. // Parse parameters
  432. if (isString(input)) {
  433. var parsed;
  434. try {
  435. parsed = urlToOptions(new URL(input));
  436. }
  437. catch (err) {
  438. /* istanbul ignore next */
  439. parsed = url.parse(input);
  440. }
  441. if (!isString(parsed.protocol)) {
  442. throw new InvalidUrlError({ input });
  443. }
  444. input = parsed;
  445. }
  446. else if (URL && (input instanceof URL)) {
  447. input = urlToOptions(input);
  448. }
  449. else {
  450. callback = options;
  451. options = input;
  452. input = { protocol: protocol };
  453. }
  454. if (isFunction(options)) {
  455. callback = options;
  456. options = null;
  457. }
  458. // Set defaults
  459. options = Object.assign({
  460. maxRedirects: exports.maxRedirects,
  461. maxBodyLength: exports.maxBodyLength,
  462. }, input, options);
  463. options.nativeProtocols = nativeProtocols;
  464. if (!isString(options.host) && !isString(options.hostname)) {
  465. options.hostname = "::1";
  466. }
  467. assert.equal(options.protocol, protocol, "protocol mismatch");
  468. debug("options", options);
  469. return new RedirectableRequest(options, callback);
  470. }
  471. // Executes a GET request, following redirects
  472. function get(input, options, callback) {
  473. var wrappedRequest = wrappedProtocol.request(input, options, callback);
  474. wrappedRequest.end();
  475. return wrappedRequest;
  476. }
  477. // Expose the properties on the wrapped protocol
  478. Object.defineProperties(wrappedProtocol, {
  479. request: { value: request, configurable: true, enumerable: true, writable: true },
  480. get: { value: get, configurable: true, enumerable: true, writable: true },
  481. });
  482. });
  483. return exports;
  484. }
  485. /* istanbul ignore next */
  486. function noop() { /* empty */ }
  487. // from https://github.com/nodejs/node/blob/master/lib/internal/url.js
  488. function urlToOptions(urlObject) {
  489. var options = {
  490. protocol: urlObject.protocol,
  491. hostname: urlObject.hostname.startsWith("[") ?
  492. /* istanbul ignore next */
  493. urlObject.hostname.slice(1, -1) :
  494. urlObject.hostname,
  495. hash: urlObject.hash,
  496. search: urlObject.search,
  497. pathname: urlObject.pathname,
  498. path: urlObject.pathname + urlObject.search,
  499. href: urlObject.href,
  500. };
  501. if (urlObject.port !== "") {
  502. options.port = Number(urlObject.port);
  503. }
  504. return options;
  505. }
  506. function removeMatchingHeaders(regex, headers) {
  507. var lastValue;
  508. for (var header in headers) {
  509. if (regex.test(header)) {
  510. lastValue = headers[header];
  511. delete headers[header];
  512. }
  513. }
  514. return (lastValue === null || typeof lastValue === "undefined") ?
  515. undefined : String(lastValue).trim();
  516. }
  517. function createErrorType(code, message, baseClass) {
  518. // Create constructor
  519. function CustomError(properties) {
  520. Error.captureStackTrace(this, this.constructor);
  521. Object.assign(this, properties || {});
  522. this.code = code;
  523. this.message = this.cause ? message + ": " + this.cause.message : message;
  524. }
  525. // Attach constructor and set default properties
  526. CustomError.prototype = new (baseClass || Error)();
  527. CustomError.prototype.constructor = CustomError;
  528. CustomError.prototype.name = "Error [" + code + "]";
  529. return CustomError;
  530. }
  531. function abortRequest(request) {
  532. for (var event of events) {
  533. request.removeListener(event, eventHandlers[event]);
  534. }
  535. request.on("error", noop);
  536. request.abort();
  537. }
  538. function isSubdomain(subdomain, domain) {
  539. assert(isString(subdomain) && isString(domain));
  540. var dot = subdomain.length - domain.length - 1;
  541. return dot > 0 && subdomain[dot] === "." && subdomain.endsWith(domain);
  542. }
  543. function isString(value) {
  544. return typeof value === "string" || value instanceof String;
  545. }
  546. function isFunction(value) {
  547. return typeof value === "function";
  548. }
  549. function isBuffer(value) {
  550. return typeof value === "object" && ("length" in value);
  551. }
  552. // Exports
  553. module.exports = wrap({ http: http, https: https });
  554. module.exports.wrap = wrap;