%PDF-
%PDF-
Mini Shell
Mini Shell
'use strict';
/**
* Module dependencies.
*/
const debug = require('debug')('superagent');
const formidable = require('formidable');
const FormData = require('form-data');
const Response = require('./response');
const parse = require('url').parse;
const format = require('url').format;
const resolve = require('url').resolve;
let methods = require('methods');
const Stream = require('stream');
const utils = require('../utils');
const unzip = require('./unzip').unzip;
const extend = require('extend');
const mime = require('mime');
const https = require('https');
const http = require('http');
const fs = require('fs');
const qs = require('qs');
const zlib = require('zlib');
const util = require('util');
const pkg = require('../../package.json');
const RequestBase = require('../request-base');
const CookieJar = require('cookiejar');
function request(method, url) {
// callback
if ('function' == typeof url) {
return new exports.Request('GET', method).end(url);
}
// url first
if (1 == arguments.length) {
return new exports.Request('GET', method);
}
return new exports.Request(method, url);
}
exports = module.exports = request;
/**
* Expose `Request`.
*/
exports.Request = Request;
/**
* Expose the agent function
*/
exports.agent = require('./agent');
/**
* Noop.
*/
function noop(){};
/**
* Expose `Response`.
*/
exports.Response = Response;
/**
* Define "form" mime type.
*/
mime.define({
'application/x-www-form-urlencoded': ['form', 'urlencoded', 'form-data']
}, true);
/**
* Protocol map.
*/
exports.protocols = {
'http:': http,
'https:': https,
};
/**
* Default serialization map.
*
* superagent.serialize['application/xml'] = function(obj){
* return 'generated xml here';
* };
*
*/
exports.serialize = {
'application/x-www-form-urlencoded': qs.stringify,
'application/json': JSON.stringify,
};
/**
* Default parsers.
*
* superagent.parse['application/xml'] = function(res, fn){
* fn(null, res);
* };
*
*/
exports.parse = require('./parsers');
/**
* Initialize internal header tracking properties on a request instance.
*
* @param {Object} req the instance
* @api private
*/
function _initHeaders(req) {
const ua = `node-superagent/${pkg.version}`;
req._header = { // coerces header names to lowercase
'user-agent': ua
};
req.header = { // preserves header name case
'User-Agent': ua
};
}
/**
* Initialize a new `Request` with the given `method` and `url`.
*
* @param {String} method
* @param {String|Object} url
* @api public
*/
function Request(method, url) {
Stream.call(this);
if ('string' != typeof url) url = format(url);
this._agent = false;
this._formData = null;
this.method = method;
this.url = url;
_initHeaders(this);
this.writable = true;
this._redirects = 0;
this.redirects(method === 'HEAD' ? 0 : 5);
this.cookies = '';
this.qs = {};
this._query = [];
this.qsRaw = this._query; // Unused, for backwards compatibility only
this._redirectList = [];
this._streamRequest = false;
this.once('end', this.clearTimeout.bind(this));
}
/**
* Inherit from `Stream` (which inherits from `EventEmitter`).
* Mixin `RequestBase`.
*/
util.inherits(Request, Stream);
RequestBase(Request.prototype);
/**
* Queue the given `file` as an attachment to the specified `field`,
* with optional `options` (or filename).
*
* ``` js
* request.post('http://localhost/upload')
* .attach('field', Buffer.from('<b>Hello world</b>'), 'hello.html')
* .end(callback);
* ```
*
* A filename may also be used:
*
* ``` js
* request.post('http://localhost/upload')
* .attach('files', 'image.jpg')
* .end(callback);
* ```
*
* @param {String} field
* @param {String|fs.ReadStream|Buffer} file
* @param {String|Object} options
* @return {Request} for chaining
* @api public
*/
Request.prototype.attach = function(field, file, options){
if (file) {
if (this._data) {
throw Error("superagent can't mix .send() and .attach()");
}
let o = options || {};
if ('string' == typeof options) {
o = { filename: options };
}
if ('string' == typeof file) {
if (!o.filename) o.filename = file;
debug('creating `fs.ReadStream` instance for file: %s', file);
file = fs.createReadStream(file);
} else if (!o.filename && file.path) {
o.filename = file.path;
}
this._getFormData().append(field, file, o);
}
return this;
};
Request.prototype._getFormData = function() {
if (!this._formData) {
this._formData = new FormData();
this._formData.on('error', err => {
this.emit('error', err);
this.abort();
});
}
return this._formData;
};
/**
* Gets/sets the `Agent` to use for this HTTP request. The default (if this
* function is not called) is to opt out of connection pooling (`agent: false`).
*
* @param {http.Agent} agent
* @return {http.Agent}
* @api public
*/
Request.prototype.agent = function(agent){
if (!arguments.length) return this._agent;
this._agent = agent;
return this;
};
/**
* Set _Content-Type_ response header passed through `mime.lookup()`.
*
* Examples:
*
* request.post('/')
* .type('xml')
* .send(xmlstring)
* .end(callback);
*
* request.post('/')
* .type('json')
* .send(jsonstring)
* .end(callback);
*
* request.post('/')
* .type('application/json')
* .send(jsonstring)
* .end(callback);
*
* @param {String} type
* @return {Request} for chaining
* @api public
*/
Request.prototype.type = function(type) {
return this.set(
'Content-Type',
~type.indexOf('/') ? type : mime.lookup(type)
);
};
/**
* Set _Accept_ response header passed through `mime.lookup()`.
*
* Examples:
*
* superagent.types.json = 'application/json';
*
* request.get('/agent')
* .accept('json')
* .end(callback);
*
* request.get('/agent')
* .accept('application/json')
* .end(callback);
*
* @param {String} accept
* @return {Request} for chaining
* @api public
*/
Request.prototype.accept = function(type){
return this.set('Accept', ~type.indexOf('/')
? type
: mime.lookup(type));
};
/**
* Add query-string `val`.
*
* Examples:
*
* request.get('/shoes')
* .query('size=10')
* .query({ color: 'blue' })
*
* @param {Object|String} val
* @return {Request} for chaining
* @api public
*/
Request.prototype.query = function(val){
if ('string' == typeof val) {
this._query.push(val);
} else {
extend(this.qs, val);
}
return this;
};
/**
* Write raw `data` / `encoding` to the socket.
*
* @param {Buffer|String} data
* @param {String} encoding
* @return {Boolean}
* @api public
*/
Request.prototype.write = function(data, encoding){
const req = this.request();
if (!this._streamRequest) {
this._streamRequest = true;
}
return req.write(data, encoding);
};
/**
* Pipe the request body to `stream`.
*
* @param {Stream} stream
* @param {Object} options
* @return {Stream}
* @api public
*/
Request.prototype.pipe = function(stream, options){
this.piped = true; // HACK...
this.buffer(false);
this.end();
return this._pipeContinue(stream, options);
};
Request.prototype._pipeContinue = function(stream, options){
this.req.once('response', res => {
// redirect
const redirect = isRedirect(res.statusCode);
if (redirect && this._redirects++ != this._maxRedirects) {
return this._redirect(res)._pipeContinue(stream, options);
}
this.res = res;
this._emitResponse();
if (this._aborted) return;
if (this._shouldUnzip(res)) {
const unzipObj = zlib.createUnzip();
unzipObj.on('error', err => {
if (err && err.code === 'Z_BUF_ERROR') { // unexpected end of file is ignored by browsers and curl
stream.emit('end');
return;
}
stream.emit('error', err);
});
res.pipe(unzipObj).pipe(stream, options);
} else {
res.pipe(stream, options);
}
res.once('end', () => {
this.emit('end');
});
});
return stream;
};
/**
* Enable / disable buffering.
*
* @return {Boolean} [val]
* @return {Request} for chaining
* @api public
*/
Request.prototype.buffer = function(val){
this._buffer = (false !== val);
return this;
};
/**
* Redirect to `url
*
* @param {IncomingMessage} res
* @return {Request} for chaining
* @api private
*/
Request.prototype._redirect = function(res){
let url = res.headers.location;
if (!url) {
return this.callback(new Error('No location header for redirect'), res);
}
debug('redirect %s -> %s', this.url, url);
// location
url = resolve(this.url, url);
// ensure the response is being consumed
// this is required for Node v0.10+
res.resume();
let headers = this.req._headers;
const changesOrigin = parse(url).host !== parse(this.url).host;
// implementation of 302 following defacto standard
if (res.statusCode == 301 || res.statusCode == 302){
// strip Content-* related fields
// in case of POST etc
headers = utils.cleanHeader(this.req._headers, changesOrigin);
// force GET
this.method = 'HEAD' == this.method
? 'HEAD'
: 'GET';
// clear data
this._data = null;
}
// 303 is always GET
if (res.statusCode == 303) {
// strip Content-* related fields
// in case of POST etc
headers = utils.cleanHeader(this.req._headers, changesOrigin);
// force method
this.method = 'GET';
// clear data
this._data = null;
}
// 307 preserves method
// 308 preserves method
delete headers.host;
delete this.req;
delete this._formData;
// remove all add header except User-Agent
_initHeaders(this);
// redirect
this._endCalled = false;
this.url = url;
this.qs = {};
this._query.length = 0;
this.set(headers);
this.emit('redirect', res);
this._redirectList.push(this.url);
this.end(this._callback);
return this;
};
/**
* Set Authorization field value with `user` and `pass`.
*
* Examples:
*
* .auth('tobi', 'learnboost')
* .auth('tobi:learnboost')
* .auth('tobi')
* .auth(accessToken, { type: 'bearer' })
*
* @param {String} user
* @param {String} [pass]
* @param {Object} [options] options with authorization type 'basic' or 'bearer' ('basic' is default)
* @return {Request} for chaining
* @api public
*/
Request.prototype.auth = function(user, pass, options){
if (1 === arguments.length) pass = '';
if (typeof pass === 'object' && pass !== null) { // pass is optional and can be replaced with options
options = pass;
pass = '';
}
if (!options) {
options = { type: 'basic' };
}
var encoder = function(string) {
return new Buffer(string).toString('base64');
};
return this._auth(user, pass, options, encoder);
};
/**
* Set the certificate authority option for https request.
*
* @param {Buffer | Array} cert
* @return {Request} for chaining
* @api public
*/
Request.prototype.ca = function(cert){
this._ca = cert;
return this;
};
/**
* Set the client certificate key option for https request.
*
* @param {Buffer | String} cert
* @return {Request} for chaining
* @api public
*/
Request.prototype.key = function(cert){
this._key = cert;
return this;
};
/**
* Set the key, certificate, and CA certs of the client in PFX or PKCS12 format.
*
* @param {Buffer | String} cert
* @return {Request} for chaining
* @api public
*/
Request.prototype.pfx = function(cert) {
if (typeof cert === 'object' && !Buffer.isBuffer(cert)) {
this._pfx = cert.pfx;
this._passphrase = cert.passphrase;
} else {
this._pfx = cert;
}
return this;
};
/**
* Set the client certificate option for https request.
*
* @param {Buffer | String} cert
* @return {Request} for chaining
* @api public
*/
Request.prototype.cert = function(cert){
this._cert = cert;
return this;
};
/**
* Return an http[s] request.
*
* @return {OutgoingMessage}
* @api private
*/
Request.prototype.request = function(){
if (this.req) return this.req;
const options = {};
try {
const query = qs.stringify(this.qs, {
indices: false,
strictNullHandling: true,
});
if (query) {
this.qs = {};
this._query.push(query);
}
this._finalizeQueryString();
} catch (e) {
return this.emit('error', e);
}
let url = this.url;
const retries = this._retries;
// default to http://
if (0 != url.indexOf('http')) url = `http://${url}`;
url = parse(url);
// support unix sockets
if (/^https?\+unix:/.test(url.protocol) === true) {
// get the protocol
url.protocol = `${url.protocol.split('+')[0]}:`;
// get the socket, path
const unixParts = url.path.match(/^([^/]+)(.+)$/);
options.socketPath = unixParts[1].replace(/%2F/g, '/');
url.path = unixParts[2];
}
// options
options.method = this.method;
options.port = url.port;
options.path = url.path;
options.host = url.hostname;
options.ca = this._ca;
options.key = this._key;
options.pfx = this._pfx;
options.cert = this._cert;
options.passphrase = this._passphrase;
options.agent = this._agent;
// initiate request
const mod = exports.protocols[url.protocol];
// request
const req = (this.req = mod.request(options));
// set tcp no delay
req.setNoDelay(true);
if ('HEAD' != options.method) {
req.setHeader('Accept-Encoding', 'gzip, deflate');
}
this.protocol = url.protocol;
this.host = url.host;
// expose events
req.once('drain', () => { this.emit('drain'); });
req.once('error', err => {
// flag abortion here for out timeouts
// because node will emit a faux-error "socket hang up"
// when request is aborted before a connection is made
if (this._aborted) return;
// if not the same, we are in the **old** (cancelled) request,
// so need to continue (same as for above)
if (this._retries !== retries) return;
// if we've received a response then we don't want to let
// an error in the request blow up the response
if (this.response) return;
this.callback(err);
});
// auth
if (url.auth) {
const auth = url.auth.split(':');
this.auth(auth[0], auth[1]);
}
if (this.username && this.password) {
this.auth(this.username, this.password);
}
for (const key in this.header) {
if (this.header.hasOwnProperty(key))
req.setHeader(key, this.header[key]);
}
// add cookies
if (this.cookies) {
if(this.header.hasOwnProperty('cookie')) {
// merge
const tmpJar = new CookieJar.CookieJar();
tmpJar.setCookies(this.header.cookie.split(';'));
tmpJar.setCookies(this.cookies.split(';'));
req.setHeader('Cookie',tmpJar.getCookies(CookieJar.CookieAccessInfo.All).toValueString());
} else {
req.setHeader('Cookie', this.cookies);
}
}
return req;
};
/**
* Invoke the callback with `err` and `res`
* and handle arity check.
*
* @param {Error} err
* @param {Response} res
* @api private
*/
Request.prototype.callback = function(err, res){
if (this._shouldRetry(err, res)) {
return this._retry();
}
// Avoid the error which is emitted from 'socket hang up' to cause the fn undefined error on JS runtime.
const fn = this._callback || noop;
this.clearTimeout();
if (this.called) return console.warn('superagent: double callback bug');
this.called = true;
if (!err) {
try {
if (!this._isResponseOK(res)) {
let msg = 'Unsuccessful HTTP response';
if (res) {
msg = http.STATUS_CODES[res.status] || msg;
}
err = new Error(msg);
err.status = res ? res.status : undefined;
}
} catch (new_err) {
err = new_err;
}
}
// It's important that the callback is called outside try/catch
// to avoid double callback
if (!err) {
return fn(null, res);
}
err.response = res;
if (this._maxRetries) err.retries = this._retries - 1;
// only emit error event if there is a listener
// otherwise we assume the callback to `.end()` will get the error
if (err && this.listeners('error').length > 0) {
this.emit('error', err);
}
fn(err, res);
};
/**
* Check if `obj` is a host object,
*
* @param {Object} obj
* @return {Boolean}
* @api private
*/
Request.prototype._isHost = function _isHost(obj) {
return Buffer.isBuffer(obj) || obj instanceof Stream || obj instanceof FormData;
}
/**
* Initiate request, invoking callback `fn(err, res)`
* with an instanceof `Response`.
*
* @param {Function} fn
* @return {Request} for chaining
* @api public
*/
Request.prototype._emitResponse = function(body, files) {
const response = new Response(this);
this.response = response;
response.redirects = this._redirectList;
if (undefined !== body) {
response.body = body;
}
response.files = files;
this.emit('response', response);
return response;
};
Request.prototype.end = function(fn) {
this.request();
debug('%s %s', this.method, this.url);
if (this._endCalled) {
console.warn(
'Warning: .end() was called twice. This is not supported in superagent'
);
}
this._endCalled = true;
// store callback
this._callback = fn || noop;
return this._end();
};
Request.prototype._end = function() {
let data = this._data;
const req = this.req;
let buffer = this._buffer;
const method = this.method;
this._setTimeouts();
// body
if ('HEAD' != method && !req._headerSent) {
// serialize stuff
if ('string' != typeof data) {
let contentType = req.getHeader('Content-Type');
// Parse out just the content type from the header (ignore the charset)
if (contentType) contentType = contentType.split(';')[0];
let serialize = exports.serialize[contentType];
if (!serialize && isJSON(contentType)) {
serialize = exports.serialize['application/json'];
}
if (serialize) data = serialize(data);
}
// content-length
if (data && !req.getHeader('Content-Length')) {
req.setHeader('Content-Length', Buffer.isBuffer(data) ? data.length : Buffer.byteLength(data));
}
}
// response
req.once('response', res => {
debug('%s %s -> %s', this.method, this.url, res.statusCode);
if (this._responseTimeoutTimer) {
clearTimeout(this._responseTimeoutTimer);
}
if (this.piped) {
return;
}
const max = this._maxRedirects;
const mime = utils.type(res.headers['content-type'] || '') || 'text/plain';
const type = mime.split('/')[0];
const multipart = 'multipart' == type;
const redirect = isRedirect(res.statusCode);
let parser = this._parser;
const responseType = this._responseType;
this.res = res;
// redirect
if (redirect && this._redirects++ != max) {
return this._redirect(res);
}
if ('HEAD' == this.method) {
this.emit('end');
this.callback(null, this._emitResponse());
return;
}
// zlib support
if (this._shouldUnzip(res)) {
unzip(req, res);
}
if (!parser) {
if (responseType) {
parser = exports.parse.image; // It's actually a generic Buffer
buffer = true;
} else if (multipart) {
const form = new formidable.IncomingForm();
parser = form.parse.bind(form);
buffer = true;
} else if (isImageOrVideo(mime)) {
parser = exports.parse.image;
buffer = true; // For backwards-compatibility buffering default is ad-hoc MIME-dependent
} else if (exports.parse[mime]) {
parser = exports.parse[mime];
} else if ('text' == type) {
parser = exports.parse.text;
buffer = (buffer !== false);
// everyone wants their own white-labeled json
} else if (isJSON(mime)) {
parser = exports.parse['application/json'];
buffer = (buffer !== false);
} else if (buffer) {
parser = exports.parse.text;
}
}
// by default only buffer text/*, json and messed up thing from hell
if ((undefined === buffer && isText(mime)) || isJSON(mime)) {
buffer = true;
}
let parserHandlesEnd = false;
if (buffer) {
// Protectiona against zip bombs and other nuisance
let responseBytesLeft = this._maxResponseSize || 200000000;
res.on('data', buf => {
responseBytesLeft -= buf.byteLength || buf.length;
if (responseBytesLeft < 0) {
// This will propagate through error event
const err = Error("Maximum response size reached");
err.code = "ETOOLARGE";
// Parsers aren't required to observe error event,
// so would incorrectly report success
parserHandlesEnd = false;
// Will emit error event
res.destroy(err);
}
});
}
if (parser) {
try {
// Unbuffered parsers are supposed to emit response early,
// which is weird BTW, because response.body won't be there.
parserHandlesEnd = buffer;
parser(res, (err, obj, files) => {
if (this.timedout) {
// Timeout has already handled all callbacks
return;
}
// Intentional (non-timeout) abort is supposed to preserve partial response,
// even if it doesn't parse.
if (err && !this._aborted) {
return this.callback(err);
}
if (parserHandlesEnd) {
this.emit('end');
this.callback(null, this._emitResponse(obj, files));
}
});
} catch (err) {
this.callback(err);
return;
}
}
this.res = res;
// unbuffered
if (!buffer) {
debug('unbuffered %s %s', this.method, this.url);
this.callback(null, this._emitResponse());
if (multipart) return; // allow multipart to handle end event
res.once('end', () => {
debug('end %s %s', this.method, this.url);
this.emit('end');
});
return;
}
// terminating events
res.once('error', err => {
parserHandlesEnd = false;
this.callback(err, null);
});
if (!parserHandlesEnd)
res.once('end', () => {
debug('end %s %s', this.method, this.url);
// TODO: unless buffering emit earlier to stream
this.emit('end');
this.callback(null, this._emitResponse());
});
});
this.emit('request', this);
const getProgressMonitor = () => {
const lengthComputable = true;
const total = req.getHeader('Content-Length');
let loaded = 0;
const progress = new Stream.Transform();
progress._transform = (chunk, encoding, cb) => {
loaded += chunk.length;
this.emit('progress', {
direction: 'upload',
lengthComputable,
loaded,
total,
});
cb(null, chunk);
};
return progress;
};
const bufferToChunks = (buffer) => {
const chunkSize = 16 * 1024; // default highWaterMark value
const chunking = new Stream.Readable();
const totalLength = buffer.length;
const remainder = totalLength % chunkSize;
const cutoff = totalLength - remainder;
for (let i = 0; i < cutoff; i += chunkSize) {
const chunk = buffer.slice(i, i + chunkSize);
chunking.push(chunk);
}
if (remainder > 0) {
const remainderBuffer = buffer.slice(-remainder);
chunking.push(remainderBuffer);
}
chunking.push(null); // no more data
return chunking;
}
// if a FormData instance got created, then we send that as the request body
const formData = this._formData;
if (formData) {
// set headers
const headers = formData.getHeaders();
for (const i in headers) {
debug('setting FormData header: "%s: %s"', i, headers[i]);
req.setHeader(i, headers[i]);
}
// attempt to get "Content-Length" header
formData.getLength((err, length) => {
// TODO: Add chunked encoding when no length (if err)
debug('got FormData Content-Length: %s', length);
if ('number' == typeof length) {
req.setHeader('Content-Length', length);
}
formData.pipe(getProgressMonitor()).pipe(req);
});
} else if (Buffer.isBuffer(data)) {
bufferToChunks(data).pipe(getProgressMonitor()).pipe(req);
} else {
req.end(data);
}
return this;
};
/**
* Check whether response has a non-0-sized gzip-encoded body
*/
Request.prototype._shouldUnzip = res => {
if (res.statusCode === 204 || res.statusCode === 304) {
// These aren't supposed to have any body
return false;
}
// header content is a string, and distinction between 0 and no information is crucial
if ('0' === res.headers['content-length']) {
// We know that the body is empty (unfortunately, this check does not cover chunked encoding)
return false;
}
// console.log(res);
return /^\s*(?:deflate|gzip)\s*$/.test(res.headers['content-encoding']);
};
// generate HTTP verb methods
if (methods.indexOf('del') == -1) {
// create a copy so we don't cause conflicts with
// other packages using the methods package and
// npm 3.x
methods = methods.slice(0);
methods.push('del');
}
methods.forEach(method => {
const name = method;
method = 'del' == method ? 'delete' : method;
method = method.toUpperCase();
request[name] = (url, data, fn) => {
const req = request(method, url);
if ('function' == typeof data) (fn = data), (data = null);
if (data) {
if (method === 'GET' || method === 'HEAD') {
req.query(data);
} else {
req.send(data);
}
}
fn && req.end(fn);
return req;
};
});
/**
* Check if `mime` is text and should be buffered.
*
* @param {String} mime
* @return {Boolean}
* @api public
*/
function isText(mime) {
const parts = mime.split('/');
const type = parts[0];
const subtype = parts[1];
return 'text' == type || 'x-www-form-urlencoded' == subtype;
}
function isImageOrVideo(mime) {
const type = mime.split('/')[0];
return 'image' == type || 'video' == type;
}
/**
* Check if `mime` is json or has +json structured syntax suffix.
*
* @param {String} mime
* @return {Boolean}
* @api private
*/
function isJSON(mime) {
// should match /json or +json
// but not /json-seq
return /[\/+]json($|[^-\w])/.test(mime);
}
/**
* Check if we should follow the redirect `code`.
*
* @param {Number} code
* @return {Boolean}
* @api private
*/
function isRedirect(code) {
return ~[301, 302, 303, 305, 307, 308].indexOf(code);
}
Zerion Mini Shell 1.0