diff --git a/packages/core/src/config/index.js b/packages/core/src/config/index.js index cc767c8..40648cf 100644 --- a/packages/core/src/config/index.js +++ b/packages/core/src/config/index.js @@ -67,6 +67,25 @@ module.exports = { } } }, + compatible: { + // **** 自定义兼容配置 **** // + // connect阶段所需的兼容性配置 + connect: { + // 参考配置（无path） + // 'xxx.xxx.xxx.xxx:443': { + // ssl: false + // } + }, + // request阶段所需的兼容性配置 + request: { + // 参考配置（配置方式同 `拦截配置`） + // 'xxx.xxx.xxx.xxx:443': { + // '.*': { + // rejectUnauthorized: false + // } + // } + } + }, intercept: { enabled: true }, diff --git a/packages/gui/src/view/pages/server.vue b/packages/gui/src/view/pages/server.vue index 2eb0953..b89207b 100644 --- a/packages/gui/src/view/pages/server.vue +++ b/packages/gui/src/view/pages/server.vue @@ -106,7 +106,16 @@ - + +
+
+ 说明：兼容程序会自动根据错误信息进行兼容性调整，并将兼容设置保存在 ~/.dev-sidecar/automaticCompatibleConfig.json 文件中。但并不是所有的兼容设置都是正确的，所以需要通过以下配置来覆盖错误的兼容设置。 +
+ +
+ +
提示：IP预设置功能，优先级高于 DNS设置 @@ -116,11 +125,11 @@ :show-btns="false" :expandedOnStart="true">
- + - +
@@ -148,7 +157,7 @@
- +
@@ -384,7 +393,7 @@ export default { }, 5000) }, async handleTabChange (key) { - if (key !== '2' && key !== '3' && key !== '5' && key !== '6') { + if (key !== '2' && key !== '3' && key !== '5' && key !== '6' && key !== '7') { return } diff --git a/packages/mitmproxy/src/lib/proxy/common/util.js b/packages/mitmproxy/src/lib/proxy/common/util.js index 1a0f04a..f3efec1 100644 --- a/packages/mitmproxy/src/lib/proxy/common/util.js +++ b/packages/mitmproxy/src/lib/proxy/common/util.js @@ -98,7 +98,7 @@ util.parseHostnameAndPort = (host, defaultPort) => { return arr } -util.getOptionsFromRequest = (req, ssl, externalProxy = null, serverSetting) => { +util.getOptionsFromRequest = (req, ssl, externalProxy = null, serverSetting, compatibleConfig = null) => { // eslint-disable-next-line node/no-deprecated-api const urlObject = url.parse(req.url) const defaultPort = ssl ? 443 : 80 @@ -148,7 +148,8 @@ util.getOptionsFromRequest = (req, ssl, externalProxy = null, serverSetting) => port, path: urlObject.path, headers: req.headers, - agent + agent, + compatibleConfig } // eslint-disable-next-line node/no-deprecated-api diff --git a/packages/mitmproxy/src/lib/proxy/compatible/compatible.js b/packages/mitmproxy/src/lib/proxy/compatible/compatible.js new file mode 100644 index 0000000..15647c6 --- /dev/null +++ b/packages/mitmproxy/src/lib/proxy/compatible/compatible.js @@ -0,0 +1,143 @@ +/** + * 兼容程序自适应生成配置 + * 此脚本会针对各种兼容性问题，为对应域名生成相应的兼容性配置，并将自适应配置写入到 `~/.dev-sidecar/automaticCompatibleConfig.json` 文件中。 + * 当然，也有可能会生成错误的配置，导致无法兼容，这时候可以通过 `config.server.compatible` 配置项，来覆盖这里生成的配置，达到主动适配的效果。 + * + * @author WangLiang + */ +const fs = require('fs') +const path = require('path') +const jsonApi = require('../../../json') +const log = require('../../../utils/util.log') +const matchUtil = require('../../../utils/util.match') + +const defaultConfig = { + // connect阶段所需的兼容性配置 + connect: { + // 参考配置 + // 'xxx.xxx.xxx.xxx:443': { + // ssl: false + // } + }, + // request阶段所需的兼容性配置 + request: { + // 参考配置 + // 'xxx.xxx.xxx.xxx:443': { + // rejectUnauthorized: false + // } + } +} + +const config = _loadFromFile(defaultConfig) + +function _getConnectConfig (hostname, port) { + const connectConfig = config.connect[`${hostname}:${port}`] + log.info(`getConnectConfig: ${hostname}:${port}, ${jsonApi.stringify2(connectConfig)}`) + return connectConfig +} +function _getRequestConfig (hostname, port) { + const requestConfig = config.request[`${hostname}:${port}`] + log.info(`getRequestConfig: ${hostname}:${port}, ${jsonApi.stringify2(requestConfig)}`) + return requestConfig +} + +// region 本地配置文件所需函数 + +function _getConfigPath () { + const userHome = process.env.USERPROFILE || process.env.HOME || '/' + const dir = path.resolve(userHome, './.dev-sidecar') + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir) + } + return path.join(dir, '/automaticCompatibleConfig.json') +} + +function _loadFromFile (defaultConfig) { + const configPath = _getConfigPath() + let config + if (!fs.existsSync(configPath)) { + config = defaultConfig + log.info('automaticCompatibleConfig.json 文件不存在，使用默认配置:', configPath) + } else { + const file = fs.readFileSync(configPath) + log.info('读取 automaticCompatibleConfig.json 成功:', configPath) + const fileStr = file.toString() + config = fileStr && fileStr.length > 2 ? jsonApi.parse(fileStr) : {} + } + + return config +} + +function _saveConfigToFile () { + const filePath = _getConfigPath() + try { + fs.writeFileSync(filePath, jsonApi.stringify(config)) + log.info('保存 automaticCompatibleConfig.json 成功:', filePath) + } catch (e) { + log.error('保存 automaticCompatibleConfig.json 失败:', filePath, e) + } +} + +// endregion + +module.exports = { + /** + * 获取 connect 阶段所需的兼容性配置 + * + * @param hostname 域名 + * @param port 端口 + * @param manualCompatibleConfig 手动兼容性配置 + * @returns connect阶段所需的兼容性配置 + */ + getConnectCompatibleConfig (hostname, port, manualCompatibleConfig = null) { + let connectCompatibleConfig = manualCompatibleConfig == null ? null : matchUtil.matchHostname(manualCompatibleConfig.connect, `${hostname}:${port}`, 'getConnectCompatibleConfig') + if (connectCompatibleConfig == null) { + connectCompatibleConfig = _getConnectConfig(hostname, port) + } + return connectCompatibleConfig + }, + + setConnectSsl (hostname, port, ssl, autoSave = true) { + const connectCompatibleConfig = this.getConnectCompatibleConfig(hostname, port) + if (connectCompatibleConfig) { + connectCompatibleConfig.ssl = ssl + } else { + config.connect[`${hostname}:${port}`] = { ssl } + } + + // 配置保存到文件 + if (autoSave) _saveConfigToFile() + + log.info(`【兼容程序】${hostname}:${port}: 设置 connect.ssl = ${ssl}`) + }, + + // -------------------------------------------------------------------------------------------------------------------------- + + /** + * 获取 request 阶段所需的兼容性配置 + * + * @param rOptions + * @param manualCompatibleConfig + */ + getRequestCompatibleConfig (rOptions, manualCompatibleConfig = null) { + let requestCompatibleConfig = manualCompatibleConfig == null ? null : matchUtil.matchHostname(manualCompatibleConfig.request, `${rOptions.hostname}:${rOptions.port}`, 'getRequestCompatibleConfig') + if (requestCompatibleConfig == null) { + requestCompatibleConfig = _getRequestConfig(rOptions.hostname, rOptions.port) + } + return requestCompatibleConfig + }, + + setRequestRejectUnauthorized (rOptions, rejectUnauthorized, autoSave = true) { + const requestCompatibleConfig = this.getRequestCompatibleConfig(rOptions.hostname, rOptions.port) + if (requestCompatibleConfig) { + requestCompatibleConfig.rejectUnauthorized = rejectUnauthorized + } else { + config.request[`${rOptions.hostname}:${rOptions.port}`] = { rejectUnauthorized } + } + + // 配置保存到文件 + if (autoSave) _saveConfigToFile() + + log.info(`【兼容程序】${rOptions.hostname}:${rOptions.port}: 设置 request.rejectUnauthorized = ${rejectUnauthorized}`) + } +} diff --git a/packages/mitmproxy/src/lib/proxy/mitmproxy/createConnectHandler.js b/packages/mitmproxy/src/lib/proxy/mitmproxy/createConnectHandler.js index 61c5989..ba038b3 100644 --- a/packages/mitmproxy/src/lib/proxy/mitmproxy/createConnectHandler.js +++ b/packages/mitmproxy/src/lib/proxy/mitmproxy/createConnectHandler.js @@ -17,7 +17,7 @@ function isSslConnect (sslConnectInterceptors, req, cltSocket, head) { } // create connectHandler function -module.exports = function createConnectHandler (sslConnectInterceptor, middlewares, fakeServerCenter, dnsConfig) { +module.exports = function createConnectHandler (sslConnectInterceptor, middlewares, fakeServerCenter, dnsConfig, compatibleConfig) { // return const sslConnectInterceptors = [] sslConnectInterceptors.push(sslConnectInterceptor) @@ -27,14 +27,14 @@ module.exports = function createConnectHandler (sslConnectInterceptor, middlewar } } - return function connectHandler (req, cltSocket, head) { + return function connectHandler (req, cltSocket, head, ssl) { // eslint-disable-next-line node/no-deprecated-api - let { hostname, port } = url.parse(`https://${req.url}`) + let { hostname, port } = url.parse(`${ssl ? 'https' : 'http'}://${req.url}`) port = parseInt(port) if (isSslConnect(sslConnectInterceptors, req, cltSocket, head)) { // 需要拦截，代替目标服务器，让客户端连接DS在本地启动的代理服务 - fakeServerCenter.getServerPromise(hostname, port).then((serverObj) => { + fakeServerCenter.getServerPromise(hostname, port, ssl, compatibleConfig).then((serverObj) => { log.info(`----- fakeServer connect: ${localIP}:${serverObj.port} ➜ ${req.url} -----`) connect(req, cltSocket, head, localIP, serverObj.port) }, (e) => { diff --git a/packages/mitmproxy/src/lib/proxy/mitmproxy/createRequestHandler.js b/packages/mitmproxy/src/lib/proxy/mitmproxy/createRequestHandler.js index ec3727f..89948cb 100644 --- a/packages/mitmproxy/src/lib/proxy/mitmproxy/createRequestHandler.js +++ b/packages/mitmproxy/src/lib/proxy/mitmproxy/createRequestHandler.js @@ -8,15 +8,16 @@ const log = require('../../../utils/util.log') const RequestCounter = require('../../choice/RequestCounter') const InsertScriptMiddleware = require('../middleware/InsertScriptMiddleware') const dnsLookup = require('./dnsLookup') +const compatible = require('../compatible/compatible') const MAX_SLOW_TIME = 8000 // 超过此时间 则认为太慢了 // create requestHandler function -module.exports = function createRequestHandler (createIntercepts, middlewares, externalProxy, dnsConfig, setting) { +module.exports = function createRequestHandler (createIntercepts, middlewares, externalProxy, dnsConfig, setting, compatibleConfig) { // return return function requestHandler (req, res, ssl) { let proxyReq - const rOptions = commonUtil.getOptionsFromRequest(req, ssl, externalProxy, setting) + const rOptions = commonUtil.getOptionsFromRequest(req, ssl, externalProxy, setting, compatibleConfig) let url = `${rOptions.method} ➜ ${rOptions.protocol}//${rOptions.hostname}:${rOptions.port}${rOptions.path}` if (rOptions.headers.connection === 'close') { @@ -130,6 +131,19 @@ module.exports = function createRequestHandler (createIntercepts, middlewares, e // log.debug('agent:', rOptions.agent) // log.debug('agent.options:', rOptions.agent.options) res.setHeader('DS-Proxy-Request', rOptions.hostname) + + // 兼容程序：2 + if (rOptions.agent) { + const compatibleConfig = compatible.getRequestCompatibleConfig(rOptions, rOptions.compatibleConfig) + if (compatibleConfig && compatibleConfig.rejectUnauthorized != null && rOptions.agent.options.rejectUnauthorized !== compatibleConfig.rejectUnauthorized) { + if (compatibleConfig.rejectUnauthorized === false && rOptions.agent.unVerifySslAgent) { + log.info(`【兼容程序】${rOptions.hostname}:${rOptions.port}: 设置 'rOptions.agent.options.rejectUnauthorized = ${compatibleConfig.rejectUnauthorized}'`) + rOptions.agent = rOptions.agent.unVerifySslAgent + res.setHeader('DS-Compatible', 'unVerifySsl') + } + } + } + proxyReq = (rOptions.protocol === 'https:' ? https : http).request(rOptions, (proxyRes) => { const cost = new Date() - start if (rOptions.protocol === 'https:') { @@ -163,6 +177,11 @@ module.exports = function createRequestHandler (createIntercepts, middlewares, e log.error(`代理请求错误: ${url}, cost: ${cost} ms, error:`, e, ', rOptions:', jsonApi.stringify2(rOptions)) countSlow(isDnsIntercept, '代理请求错误: ' + e.message) reject(e) + + // 兼容程序：2 + if (e.code === 'DEPTH_ZERO_SELF_SIGNED_CERT') { + compatible.setRequestRejectUnauthorized(rOptions, false) + } }) proxyReq.on('aborted', () => { const cost = new Date() - start diff --git a/packages/mitmproxy/src/lib/proxy/tls/FakeServersCenter.js b/packages/mitmproxy/src/lib/proxy/tls/FakeServersCenter.js index 51dc2b3..85b8180 100644 --- a/packages/mitmproxy/src/lib/proxy/tls/FakeServersCenter.js +++ b/packages/mitmproxy/src/lib/proxy/tls/FakeServersCenter.js @@ -1,4 +1,5 @@ const https = require('https') +const http = require('http') const tlsUtils = require('./tlsUtils') const CertAndKeyContainer = require('./CertAndKeyContainer') const forge = require('node-forge') @@ -6,6 +7,7 @@ const pki = forge.pki // const colors = require('colors') const tls = require('tls') const log = require('../../../utils/util.log') +const compatible = require('../compatible/compatible') function arraysHaveSameElements (arr1, arr2) { if (arr1.length !== arr2.length) { @@ -43,15 +45,28 @@ module.exports = class FakeServersCenter { return serverPromiseObj } - getServerPromise (hostname, port) { + getServerPromise (hostname, port, ssl, manualCompatibleConfig) { + if (port === 443 || port === 80) { + ssl = port === 443 + } else { + // 兼容程序：1 + const compatibleConfig = compatible.getConnectCompatibleConfig(hostname, port, manualCompatibleConfig) + if (compatibleConfig && compatibleConfig.ssl != null) { + ssl = compatibleConfig.ssl + } + } + + log.info(`getServerPromise, hostname: ${hostname}:${port}, ssl: ${ssl}, protocol: ${ssl ? 'https' : 'http'}`) + for (let i = 0; i < this.queue.length; i++) { const serverPromiseObj = this.queue[i] - if (serverPromiseObj.port === port) { + if (serverPromiseObj.port === port && serverPromiseObj.ssl === ssl) { const mappingHostNames = serverPromiseObj.mappingHostNames for (let j = 0; j < mappingHostNames.length; j++) { const DNSName = mappingHostNames[j] if (tlsUtils.isMappingHostName(DNSName, hostname)) { this.reRankServer(i) + log.info(`Load promise from cache, hostname: ${hostname}:${port}, ssl: ${ssl}, serverPromiseObj: {"ssl":${serverPromiseObj.ssl},"port":${serverPromiseObj.port},"mappingHostNames":${JSON.stringify(serverPromiseObj.mappingHostNames)}}`) return serverPromiseObj.promise } } @@ -60,30 +75,38 @@ module.exports = class FakeServersCenter { const serverPromiseObj = { port, + ssl, mappingHostNames: [hostname] // temporary hostname } const promise = new Promise((resolve, reject) => { (async () => { - const certObj = await this.certAndKeyContainer.getCertPromise(hostname, port) - const cert = certObj.cert - const key = certObj.key - const certPem = pki.certificateToPem(cert) - const keyPem = pki.privateKeyToPem(key) - const fakeServer = new https.Server({ - key: keyPem, - cert: certPem, - SNICallback: (hostname, done) => { - (async () => { - const certObj = await this.certAndKeyContainer.getCertPromise(hostname, port) - log.info(`sni callback: ${hostname}:${port}`) - done(null, tls.createSecureContext({ - key: pki.privateKeyToPem(certObj.key), - cert: pki.certificateToPem(certObj.cert) - })) - })() - } - }) + let fakeServer + let cert + let key + if (ssl) { + const certObj = await this.certAndKeyContainer.getCertPromise(hostname, port) + cert = certObj.cert + key = certObj.key + const certPem = pki.certificateToPem(cert) + const keyPem = pki.privateKeyToPem(key) + fakeServer = new https.Server({ + key: keyPem, + cert: certPem, + SNICallback: (hostname, done) => { + (async () => { + const certObj = await this.certAndKeyContainer.getCertPromise(hostname, port) + log.info(`fakeServer SNICallback: ${hostname}:${port}`) + done(null, tls.createSecureContext({ + key: pki.privateKeyToPem(certObj.key), + cert: pki.certificateToPem(certObj.cert) + })) + })() + } + }) + } else { + fakeServer = new http.Server() + } const serverObj = { cert, key, @@ -97,7 +120,6 @@ module.exports = class FakeServersCenter { serverObj.port = address.port }) fakeServer.on('request', (req, res) => { - const ssl = true log.debug(`【fakeServer request - ${hostname}:${port}】\r\n----- req -----\r\n`, req, '\r\n----- res -----\r\n', res) this.requestHandler(req, res, ssl) }) @@ -116,7 +138,6 @@ module.exports = class FakeServersCenter { resolve(serverObj) }) fakeServer.on('upgrade', (req, socket, head) => { - const ssl = true if (process.env.NODE_ENV === 'development') { log.debug(`【fakeServer upgrade - ${hostname}:${port}】\r\n----- req -----\r\n`, req, '\r\n----- socket -----\r\n', socket, '\r\n----- head -----\r\n', head) } else { @@ -132,29 +153,47 @@ module.exports = class FakeServersCenter { fakeServer.on('clientError', (err, socket) => { // log.error(`【fakeServer clientError - ${hostname}:${port}】\r\n----- error -----\r\n`, err, '\r\n----- socket -----\r\n', socket) log.error(`【fakeServer clientError - ${hostname}:${port}】\r\n`, err) + + // 兼容程序：1 + if (port !== 443 && port !== 80) { + if (ssl === true && err.code.indexOf('ERR_SSL_') === 0) { + compatible.setConnectSsl(hostname, port, false) + log.error(`兼容程序：SSL异常，现设置为禁用ssl: ${hostname}:${port}, ssl = false`) + } else if (ssl === false && err.code === 'HPE_INVALID_METHOD') { + compatible.setConnectSsl(hostname, port, true) + log.error(`兼容程序：${err.code}，现设置为启用ssl: ${hostname}:${port}, ssl = true`) + } + } }) - fakeServer.on('tlsClientError', (err, tlsSocket) => { - // log.error(`【fakeServer tlsClientError - ${hostname}:${port}】\r\n----- error -----\r\n`, err, '\r\n----- tlsSocket -----\r\n', tlsSocket) - log.error(`【fakeServer tlsClientError - ${hostname}:${port}】\r\n`, err) - }) + if (ssl) { + fakeServer.on('tlsClientError', (err, tlsSocket) => { + if (err.code === 'ECONNRESET' || err.code === 'ETIMEDOUT') { + return // 在tlsClientError事件中，以上异常不记录日志 + } + // log.error(`【fakeServer tlsClientError - ${hostname}:${port}】\r\n----- error -----\r\n`, err, '\r\n----- tlsSocket -----\r\n', tlsSocket) + log.error(`【fakeServer tlsClientError - ${hostname}:${port}】\r\n`, err) + }) + } // 其他监听事件，只打印debug日志 if (process.env.NODE_ENV === 'development') { - fakeServer.on('keylog', (line, tlsSocket) => { - log.debug(`【fakeServer keylog - ${hostname}:${port}】\r\n----- line -----\r\n`, line, '\r\n----- tlsSocket -----\r\n', tlsSocket) - }) - // fakeServer.on('newSession', (sessionId, sessionData, callback) => { - // log.debug('【fakeServer newSession - ${hostname}:${port}】\r\n----- sessionId -----\r\n', sessionId, '\r\n----- sessionData -----\r\n', sessionData, '\r\n----- callback -----\r\n', callback) - // }) - // fakeServer.on('OCSPRequest', (certificate, issuer, callback) => { - // log.debug('【fakeServer OCSPRequest - ${hostname}:${port}】\r\n----- certificate -----\r\n', certificate, '\r\n----- issuer -----\r\n', issuer, '\r\n----- callback -----\r\n', callback) - // }) - // fakeServer.on('resumeSession', (sessionId, callback) => { - // log.debug('【fakeServer resumeSession - ${hostname}:${port}】\r\n----- sessionId -----\r\n', sessionId, '\r\n----- callback -----\r\n', callback) - // }) - fakeServer.on('secureConnection', (tlsSocket) => { - log.debug(`【fakeServer secureConnection - ${hostname}:${port}】\r\n----- tlsSocket -----\r\n`, tlsSocket) - }) + if (ssl) { + fakeServer.on('keylog', (line, tlsSocket) => { + log.debug(`【fakeServer keylog - ${hostname}:${port}】\r\n----- line -----\r\n`, line, '\r\n----- tlsSocket -----\r\n', tlsSocket) + }) + // fakeServer.on('newSession', (sessionId, sessionData, callback) => { + // log.debug(`【fakeServer newSession - ${hostname}:${port}】\r\n----- sessionId -----\r\n`, sessionId, '\r\n----- sessionData -----\r\n', sessionData, '\r\n----- callback -----\r\n', callback) + // }) + // fakeServer.on('OCSPRequest', (certificate, issuer, callback) => { + // log.debug(`【fakeServer OCSPRequest - ${hostname}:${port}】\r\n----- certificate -----\r\n`, certificate, '\r\n----- issuer -----\r\n', issuer, '\r\n----- callback -----\r\n', callback) + // }) + // fakeServer.on('resumeSession', (sessionId, callback) => { + // log.debug(`【fakeServer resumeSession - ${hostname}:${port}】\r\n----- sessionId -----\r\n`, sessionId, '\r\n----- callback -----\r\n', callback) + // }) + fakeServer.on('secureConnection', (tlsSocket) => { + log.debug(`【fakeServer secureConnection - ${hostname}:${port}】\r\n----- tlsSocket -----\r\n`, tlsSocket) + }) + } fakeServer.on('close', () => { log.debug(`【fakeServer close - ${hostname}:${port}】no arguments...`) }) diff --git a/packages/mitmproxy/src/options.js b/packages/mitmproxy/src/options.js index 6cfb4e7..05f531d 100644 --- a/packages/mitmproxy/src/options.js +++ b/packages/mitmproxy/src/options.js @@ -99,6 +99,10 @@ module.exports = (serverConfig) => { speedTest: serverConfig.dns.speedTest }, setting, + compatibleConfig: { + connect: serverConfig.compatible ? matchUtil.domainMapRegexply(serverConfig.compatible.connect) : {}, + request: serverConfig.compatible ? matchUtil.domainMapRegexply(serverConfig.compatible.request) : {} + }, middlewares, sslConnectInterceptor: (req, cltSocket, head) => { const hostname = req.url.split(':')[0]