From 041a4b16a183694f88bbd8e13a3eef85035cc8b8 Mon Sep 17 00:00:00 2001 From: xiaojunnuo Date: Wed, 18 Nov 2020 14:46:42 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E6=8F=92=E5=85=A5?= =?UTF-8?q?=E8=84=9A=E6=9C=AC=E6=97=B6=E9=83=A8=E5=88=86=E5=AD=97=E7=AC=A6?= =?UTF-8?q?=E4=B9=B1=E7=A0=81=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/core/src/config/index.js | 2 +- .../src/lib/interceptor/impl/script.js | 2 +- ...iddleware.js => InsertScriptMiddleware.js} | 81 ++++++++++++++----- .../lib/proxy/middleware/ScriptMiddleware.js | 23 ------ .../proxy/mitmproxy/createRequestHandler.js | 24 +++--- .../lib/proxy/mitmproxy/middlewareHandler.js | 20 ----- packages/mitmproxy/test/matchTest.js | 10 +++ 7 files changed, 82 insertions(+), 80 deletions(-) rename packages/mitmproxy/src/lib/proxy/middleware/{HtmlMiddleware.js => InsertScriptMiddleware.js} (56%) delete mode 100644 packages/mitmproxy/src/lib/proxy/middleware/ScriptMiddleware.js delete mode 100644 packages/mitmproxy/src/lib/proxy/mitmproxy/middlewareHandler.js diff --git a/packages/core/src/config/index.js b/packages/core/src/config/index.js index 680f277..d305c2a 100644 --- a/packages/core/src/config/index.js +++ b/packages/core/src/config/index.js @@ -29,7 +29,7 @@ module.exports = { '/.*': { proxy: 'github.com', backup: [ - 'github.docmirror.cn/_proxy' + 'gh.docmirror.top/_proxy' ] } }, diff --git a/packages/mitmproxy/src/lib/interceptor/impl/script.js b/packages/mitmproxy/src/lib/interceptor/impl/script.js index 354e16e..19de3ce 100644 --- a/packages/mitmproxy/src/lib/interceptor/impl/script.js +++ b/packages/mitmproxy/src/lib/interceptor/impl/script.js @@ -28,7 +28,7 @@ module.exports = { const scriptTag = getScript(key, script.script) tags += '\r\n' + scriptTag } - log.info('responseIntercept: append script', rOptions.hostname, rOptions.path) + log.info('responseIntercept: insert script', rOptions.hostname, rOptions.path) return { head: tags } diff --git a/packages/mitmproxy/src/lib/proxy/middleware/HtmlMiddleware.js b/packages/mitmproxy/src/lib/proxy/middleware/InsertScriptMiddleware.js similarity index 56% rename from packages/mitmproxy/src/lib/proxy/middleware/HtmlMiddleware.js rename to packages/mitmproxy/src/lib/proxy/middleware/InsertScriptMiddleware.js index f345bcd..a85925f 100644 --- a/packages/mitmproxy/src/lib/proxy/middleware/HtmlMiddleware.js +++ b/packages/mitmproxy/src/lib/proxy/middleware/InsertScriptMiddleware.js @@ -4,6 +4,15 @@ const zlib = require('zlib') const url = require('url') var httpUtil = {} +httpUtil.getCharset = function (res) { + const contentType = res.getHeader('content-type') + const reg = /charset=(.*)/ + const matched = contentType.match(reg) + if (matched) { + return matched[1] + } + return 'utf-8' +} httpUtil.isGzip = function (res) { var contentEncoding = res.headers['content-encoding'] return !!(contentEncoding && contentEncoding.toLowerCase() === 'gzip') @@ -12,34 +21,63 @@ httpUtil.isHtml = function (res) { var contentType = res.headers['content-type'] return (typeof contentType !== 'undefined') && /text\/html|application\/xhtml\+xml/.test(contentType) } +const HEAD = Buffer.from('') +const HEAD_UP = Buffer.from('') +const BODY = Buffer.from('') +const BODY_UP = Buffer.from('') -function injectScriptIntoHeadHtml (html, script) { - html = html.replace(/(<\/head>)/i, function (match) { - return script + match - }) - return html -} - -function injectScriptIntoBodyHtml (html, script) { - html = html.replace(/(<\/body>)/i, function (match) { - return script + match - }) - return html -} - -function chunkReplace (_this, chunk, enc, callback, append) { - let chunkString = chunk.toString() +function chunkByteReplace (_this, chunk, enc, callback, append) { if (append && append.head) { - chunkString = injectScriptIntoHeadHtml(chunkString, append.head) + const ret = injectScriptIntoHtml([HEAD, HEAD_UP], chunk, append.head) + if (ret != null) { + chunk = ret + } } if (append && append.body) { - chunkString = injectScriptIntoBodyHtml(chunkString, append.body) + const ret = injectScriptIntoHtml([BODY, BODY_UP], chunk, append.body) + if (ret != null) { + chunk = ret + } } - _this.push(Buffer.from(chunkString)) + _this.push(chunk) callback() } +function injectScriptIntoHtml (tags, chunk, script) { + for (const tag of tags) { + const index = chunk.indexOf(tag) + if (index < 0) { + continue + } + const scriptBuf = Buffer.from(script) + const chunkNew = Buffer.alloc(chunk.length + scriptBuf.length) + chunk.copy(chunkNew, 0, 0, index) + scriptBuf.copy(chunkNew, index, 0) + chunk.copy(chunkNew, index + scriptBuf.length, index) + return chunkNew + } + return null +} +const contextPath = '/____ds_script____/' +const monkey = require('../../monkey') module.exports = { + + requestIntercept (context, req, res, ssl, next) { + const { rOptions, log } = context + if (rOptions.path.indexOf(contextPath) !== 0) { + return + } + const urlPath = rOptions.path + const filename = urlPath.replace(contextPath, '') + + const script = monkey.get()[filename] + + log.info('ds_script', filename, script != null) + res.writeHead(200) + res.write(script.script) + res.end() + return true + }, responseInterceptor (req, res, proxyReq, proxyRes, ssl, next, append) { if (!append.head && !append.body) { next() @@ -52,6 +90,7 @@ module.exports = { })() if (!isHtml || contentLengthIsZero) { next() + return } else { Object.keys(proxyRes.headers).forEach(function (key) { if (proxyRes.headers[key] !== undefined) { @@ -88,11 +127,11 @@ module.exports = { if (isGzip) { proxyRes.pipe(new zlib.Gunzip()) .pipe(through(function (chunk, enc, callback) { - chunkReplace(this, chunk, enc, callback, append) + chunkByteReplace(this, chunk, enc, callback, append) })).pipe(new zlib.Gzip()).pipe(res) } else { proxyRes.pipe(through(function (chunk, enc, callback) { - chunkReplace(this, chunk, enc, callback, append) + chunkByteReplace(this, chunk, enc, callback, append) })).pipe(res) } } diff --git a/packages/mitmproxy/src/lib/proxy/middleware/ScriptMiddleware.js b/packages/mitmproxy/src/lib/proxy/middleware/ScriptMiddleware.js deleted file mode 100644 index 47c8513..0000000 --- a/packages/mitmproxy/src/lib/proxy/middleware/ScriptMiddleware.js +++ /dev/null @@ -1,23 +0,0 @@ -const contextPath = '/____ds_script____/' -const monkey = require('../../monkey') -module.exports = { - requestIntercept (context, req, res, ssl, next) { - const { rOptions, log } = context - const urlPath = rOptions.path - const filename = urlPath.replace(contextPath, '') - - const script = monkey.get()[filename] - - log.info('ds_script', filename, script != null) - res.writeHead(200) - res.write(script.script) - res.end() - return true - }, - is (rOptions) { - if (rOptions.path.indexOf(contextPath) !== 0) { - return false - } - return true - } -} diff --git a/packages/mitmproxy/src/lib/proxy/mitmproxy/createRequestHandler.js b/packages/mitmproxy/src/lib/proxy/mitmproxy/createRequestHandler.js index e702d08..2c3a577 100644 --- a/packages/mitmproxy/src/lib/proxy/mitmproxy/createRequestHandler.js +++ b/packages/mitmproxy/src/lib/proxy/mitmproxy/createRequestHandler.js @@ -4,9 +4,10 @@ const commonUtil = require('../common/util') // const upgradeHeader = /(^|,)\s*upgrade\s*($|,)/i const DnsUtil = require('../../dns/index') const log = require('../../../utils/util.log') -const HtmlMiddleware = require('../middleware/HtmlMiddleware') const RequestCounter = require('../../choice/RequestCounter') -const ScriptMiddleware = require('../middleware/ScriptMiddleware') +const InsertScriptMiddleware = require('../middleware/InsertScriptMiddleware') +const defaultDns = require('dns') +const MAX_SLOW_TIME = 8000 // 超过此时间 则认为太慢了 // create requestHandler function module.exports = function createRequestHandler (createIntercepts, externalProxy, dnsConfig) { // return @@ -30,7 +31,7 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy, if (interceptors == null) { interceptors = [] } - let reqIncpts = interceptors.filter(item => { return item.requestIntercept != null }) + const reqIncpts = interceptors.filter(item => { return item.requestIntercept != null }) const resIncpts = interceptors.filter(item => { return item.responseIntercept != null }) const requestInterceptorPromise = () => { @@ -39,13 +40,7 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy, resolve() } try { - if (ScriptMiddleware.is(rOptions)) { - if (reqIncpts == null) { - reqIncpts = [] - } - reqIncpts.unshift(ScriptMiddleware) - } - + reqIncpts.unshift(InsertScriptMiddleware) if (reqIncpts && reqIncpts.length > 0) { for (const reqIncpt of reqIncpts) { const goNext = reqIncpt.requestIntercept(context, req, res, ssl, next) @@ -105,19 +100,20 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy, if (ip !== hostname) { callback(null, ip, 4) } else { - rOptions.lookup(hostname, options, callback) + defaultDns.lookup(hostname, options, callback) } }) } } } + proxyReq = (rOptions.protocol === 'https:' ? https : http).request(rOptions, (proxyRes) => { const end = new Date().getTime() const cost = end - start if (rOptions.protocol === 'https:') { log.info('代理请求返回:', url, cost + 'ms') } - if (cost > 8000) { + if (cost > MAX_SLOW_TIME) { countSlow(isDnsIntercept) } resolve(proxyRes) @@ -146,7 +142,7 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy, const cost = end - start log.error('代理请求被取消', rOptions.hostname, rOptions.path, cost + 'ms') - if (cost > 8000) { + if (cost > MAX_SLOW_TIME) { countSlow(isDnsIntercept) } @@ -212,7 +208,7 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy, body += append.body } } - HtmlMiddleware.responseInterceptor(req, res, proxyReq, proxyRes, ssl, next, { head, body }) + InsertScriptMiddleware.responseInterceptor(req, res, proxyReq, proxyRes, ssl, next, { head, body }) } else { next() } diff --git a/packages/mitmproxy/src/lib/proxy/mitmproxy/middlewareHandler.js b/packages/mitmproxy/src/lib/proxy/mitmproxy/middlewareHandler.js deleted file mode 100644 index 2820f76..0000000 --- a/packages/mitmproxy/src/lib/proxy/mitmproxy/middlewareHandler.js +++ /dev/null @@ -1,20 +0,0 @@ -const _ = require('lodash') -module.exports = (middlewares) => { - if (middlewares) { - if (Object.prototype.toString.call(middlewares) !== '[object Array]') { - throw new TypeError('middlewares must be a array') - } - } - // - // const sslConnectInterceptors = [] - // const requestInterceptors = [] - // const responseInterceptors = [] - - _.each(middlewares, (m) => { - if (m.buildIn === false || m.buildIn === 'false') { - - } else { - // m.name - } - }) -} diff --git a/packages/mitmproxy/test/matchTest.js b/packages/mitmproxy/test/matchTest.js index 44cd1ec..67348d5 100644 --- a/packages/mitmproxy/test/matchTest.js +++ b/packages/mitmproxy/test/matchTest.js @@ -5,3 +5,13 @@ console.log(ret) const reg = new RegExp('^/[^/]+/[^/]+$') console.log('/greper/d2-crud-plus/blob/master/.eslintignore'.match(reg)) + +const chunk = Buffer.from('') +const script = '' +const index = chunk.indexOf('') +const scriptBuf = Buffer.from(script) +const chunkNew = Buffer.alloc(chunk.length + scriptBuf.length) +chunk.copy(chunkNew, 0, 0, index) +scriptBuf.copy(chunkNew, index, 0) +chunk.copy(chunkNew, index + scriptBuf.length, index) +console.log(chunkNew.toString())