Browse Source

feature: 新增兼容程序,在遇到部分特定异常时,通过自动调整参数达到规避异常的目的 (#375)

pull/376/head
王良 1 month ago committed by GitHub
parent
commit
2218e808dd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 19
      packages/core/src/config/index.js
  2. 19
      packages/gui/src/view/pages/server.vue
  3. 5
      packages/mitmproxy/src/lib/proxy/common/util.js
  4. 143
      packages/mitmproxy/src/lib/proxy/compatible/compatible.js
  5. 8
      packages/mitmproxy/src/lib/proxy/mitmproxy/createConnectHandler.js
  6. 23
      packages/mitmproxy/src/lib/proxy/mitmproxy/createRequestHandler.js
  7. 123
      packages/mitmproxy/src/lib/proxy/tls/FakeServersCenter.js
  8. 4
      packages/mitmproxy/src/options.js

19
packages/core/src/config/index.js

@ -67,6 +67,25 @@ module.exports = {
}
}
},
compatible: {
// **** 自定义兼容配置 **** //
// connect阶段所需的兼容性配置
connect: {
// 参考配置(无path)
// 'xxx.xxx.xxx.xxx:443': {
// ssl: false
// }
},
// request阶段所需的兼容性配置
request: {
// 参考配置(配置方式同 `拦截配置`)
// 'xxx.xxx.xxx.xxx:443': {
// '.*': {
// rejectUnauthorized: false
// }
// }
}
},
intercept: {
enabled: true
},

19
packages/gui/src/view/pages/server.vue

@ -106,7 +106,16 @@
</a-col>
</a-row>
</a-tab-pane>
<a-tab-pane tab="IP预设置" key="5">
<a-tab-pane tab="兼容程序" key="5">
<div style="height:100%;display:flex;flex-direction:column">
<div>
说明<code>兼容程序</code>会自动根据错误信息进行兼容性调整并将兼容设置保存在 <code>~/.dev-sidecar/automaticCompatibleConfig.json</code>
</div>
<vue-json-editor style="flex-grow:1;min-height:300px;margin-top:10px;" ref="editor" v-model="config.server.compatible" mode="code"
:show-btns="false" :expandedOnStart="true"></vue-json-editor>
</div>
</a-tab-pane>
<a-tab-pane tab="IP预设置" key="6">
<div style="height:100%;display:flex;flex-direction:column">
<div>
提示<code>IP预设置</code>功能优先级高于 <code>DNS设置</code>
@ -116,11 +125,11 @@
:show-btns="false" :expandedOnStart="true"></vue-json-editor>
</div>
</a-tab-pane>
<a-tab-pane tab="DNS服务管理" key="6">
<a-tab-pane tab="DNS服务管理" key="7">
<vue-json-editor style="height:100%" ref="editor" v-model="config.server.dns.providers" mode="code"
:show-btns="false" :expandedOnStart="true"></vue-json-editor>
</a-tab-pane>
<a-tab-pane tab="DNS设置" key="7">
<a-tab-pane tab="DNS设置" key="8">
<div>
<a-row style="margin-top:10px">
<a-col span="19">
@ -148,7 +157,7 @@
</a-row>
</div>
</a-tab-pane>
<a-tab-pane tab="IP测速" key="8">
<a-tab-pane tab="IP测速" key="9">
<div class="ip-tester" style="padding-right: 10px">
<a-alert type="info" message="对从DNS获取到的IP进行测速,使用速度最快的IP进行访问(注意:对使用了增强功能的域名没啥用)"></a-alert>
<a-form-item label="开启DNS测速" :label-col="labelCol" :wrapper-col="wrapperCol">
@ -384,7 +393,7 @@ export default {
}, 5000)
},
async handleTabChange (key) {
if (key !== '2' && key !== '3' && key !== '5' && key !== '6') {
if (key !== '2' && key !== '3' && key !== '5' && key !== '6' && key !== '7') {
return
}

5
packages/mitmproxy/src/lib/proxy/common/util.js

@ -98,7 +98,7 @@ util.parseHostnameAndPort = (host, defaultPort) => {
return arr
}
util.getOptionsFromRequest = (req, ssl, externalProxy = null, serverSetting) => {
util.getOptionsFromRequest = (req, ssl, externalProxy = null, serverSetting, compatibleConfig = null) => {
// eslint-disable-next-line node/no-deprecated-api
const urlObject = url.parse(req.url)
const defaultPort = ssl ? 443 : 80
@ -148,7 +148,8 @@ util.getOptionsFromRequest = (req, ssl, externalProxy = null, serverSetting) =>
port,
path: urlObject.path,
headers: req.headers,
agent
agent,
compatibleConfig
}
// eslint-disable-next-line node/no-deprecated-api

143
packages/mitmproxy/src/lib/proxy/compatible/compatible.js

@ -0,0 +1,143 @@
/**
* 兼容程序自适应生成配置
* 此脚本会针对各种兼容性问题为对应域名生成相应的兼容性配置并将自适应配置写入到 `~/.dev-sidecar/automaticCompatibleConfig.json` 文件中
* 当然也有可能会生成错误的配置导致无法兼容这时候可以通过 `config.server.compatible` 配置项来覆盖这里生成的配置达到主动适配的效果
*
* @author WangLiang
*/
const fs = require('fs')
const path = require('path')
const jsonApi = require('../../../json')
const log = require('../../../utils/util.log')
const matchUtil = require('../../../utils/util.match')
const defaultConfig = {
// connect阶段所需的兼容性配置
connect: {
// 参考配置
// 'xxx.xxx.xxx.xxx:443': {
// ssl: false
// }
},
// request阶段所需的兼容性配置
request: {
// 参考配置
// 'xxx.xxx.xxx.xxx:443': {
// rejectUnauthorized: false
// }
}
}
const config = _loadFromFile(defaultConfig)
function _getConnectConfig (hostname, port) {
const connectConfig = config.connect[`${hostname}:${port}`]
log.info(`getConnectConfig: ${hostname}:${port}, ${jsonApi.stringify2(connectConfig)}`)
return connectConfig
}
function _getRequestConfig (hostname, port) {
const requestConfig = config.request[`${hostname}:${port}`]
log.info(`getRequestConfig: ${hostname}:${port}, ${jsonApi.stringify2(requestConfig)}`)
return requestConfig
}
// region 本地配置文件所需函数
function _getConfigPath () {
const userHome = process.env.USERPROFILE || process.env.HOME || '/'
const dir = path.resolve(userHome, './.dev-sidecar')
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir)
}
return path.join(dir, '/automaticCompatibleConfig.json')
}
function _loadFromFile (defaultConfig) {
const configPath = _getConfigPath()
let config
if (!fs.existsSync(configPath)) {
config = defaultConfig
log.info('automaticCompatibleConfig.json 文件不存在,使用默认配置:', configPath)
} else {
const file = fs.readFileSync(configPath)
log.info('读取 automaticCompatibleConfig.json 成功:', configPath)
const fileStr = file.toString()
config = fileStr && fileStr.length > 2 ? jsonApi.parse(fileStr) : {}
}
return config
}
function _saveConfigToFile () {
const filePath = _getConfigPath()
try {
fs.writeFileSync(filePath, jsonApi.stringify(config))
log.info('保存 automaticCompatibleConfig.json 成功:', filePath)
} catch (e) {
log.error('保存 automaticCompatibleConfig.json 失败:', filePath, e)
}
}
// endregion
module.exports = {
/**
* 获取 connect 阶段所需的兼容性配置
*
* @param hostname 域名
* @param port 端口
* @param manualCompatibleConfig 手动兼容性配置
* @returns connect阶段所需的兼容性配置
*/
getConnectCompatibleConfig (hostname, port, manualCompatibleConfig = null) {
let connectCompatibleConfig = manualCompatibleConfig == null ? null : matchUtil.matchHostname(manualCompatibleConfig.connect, `${hostname}:${port}`, 'getConnectCompatibleConfig')
if (connectCompatibleConfig == null) {
connectCompatibleConfig = _getConnectConfig(hostname, port)
}
return connectCompatibleConfig
},
setConnectSsl (hostname, port, ssl, autoSave = true) {
const connectCompatibleConfig = this.getConnectCompatibleConfig(hostname, port)
if (connectCompatibleConfig) {
connectCompatibleConfig.ssl = ssl
} else {
config.connect[`${hostname}:${port}`] = { ssl }
}
// 配置保存到文件
if (autoSave) _saveConfigToFile()
log.info(`【兼容程序】${hostname}:${port}: 设置 connect.ssl = ${ssl}`)
},
// --------------------------------------------------------------------------------------------------------------------------
/**
* 获取 request 阶段所需的兼容性配置
*
* @param rOptions
* @param manualCompatibleConfig
*/
getRequestCompatibleConfig (rOptions, manualCompatibleConfig = null) {
let requestCompatibleConfig = manualCompatibleConfig == null ? null : matchUtil.matchHostname(manualCompatibleConfig.request, `${rOptions.hostname}:${rOptions.port}`, 'getRequestCompatibleConfig')
if (requestCompatibleConfig == null) {
requestCompatibleConfig = _getRequestConfig(rOptions.hostname, rOptions.port)
}
return requestCompatibleConfig
},
setRequestRejectUnauthorized (rOptions, rejectUnauthorized, autoSave = true) {
const requestCompatibleConfig = this.getRequestCompatibleConfig(rOptions.hostname, rOptions.port)
if (requestCompatibleConfig) {
requestCompatibleConfig.rejectUnauthorized = rejectUnauthorized
} else {
config.request[`${rOptions.hostname}:${rOptions.port}`] = { rejectUnauthorized }
}
// 配置保存到文件
if (autoSave) _saveConfigToFile()
log.info(`【兼容程序】${rOptions.hostname}:${rOptions.port}: 设置 request.rejectUnauthorized = ${rejectUnauthorized}`)
}
}

8
packages/mitmproxy/src/lib/proxy/mitmproxy/createConnectHandler.js

@ -17,7 +17,7 @@ function isSslConnect (sslConnectInterceptors, req, cltSocket, head) {
}
// create connectHandler function
module.exports = function createConnectHandler (sslConnectInterceptor, middlewares, fakeServerCenter, dnsConfig) {
module.exports = function createConnectHandler (sslConnectInterceptor, middlewares, fakeServerCenter, dnsConfig, compatibleConfig) {
// return
const sslConnectInterceptors = []
sslConnectInterceptors.push(sslConnectInterceptor)
@ -27,14 +27,14 @@ module.exports = function createConnectHandler (sslConnectInterceptor, middlewar
}
}
return function connectHandler (req, cltSocket, head) {
return function connectHandler (req, cltSocket, head, ssl) {
// eslint-disable-next-line node/no-deprecated-api
let { hostname, port } = url.parse(`https://${req.url}`)
let { hostname, port } = url.parse(`${ssl ? 'https' : 'http'}://${req.url}`)
port = parseInt(port)
if (isSslConnect(sslConnectInterceptors, req, cltSocket, head)) {
// 需要拦截,代替目标服务器,让客户端连接DS在本地启动的代理服务
fakeServerCenter.getServerPromise(hostname, port).then((serverObj) => {
fakeServerCenter.getServerPromise(hostname, port, ssl, compatibleConfig).then((serverObj) => {
log.info(`----- fakeServer connect: ${localIP}:${serverObj.port}${req.url} -----`)
connect(req, cltSocket, head, localIP, serverObj.port)
}, (e) => {

23
packages/mitmproxy/src/lib/proxy/mitmproxy/createRequestHandler.js

@ -8,15 +8,16 @@ const log = require('../../../utils/util.log')
const RequestCounter = require('../../choice/RequestCounter')
const InsertScriptMiddleware = require('../middleware/InsertScriptMiddleware')
const dnsLookup = require('./dnsLookup')
const compatible = require('../compatible/compatible')
const MAX_SLOW_TIME = 8000 // 超过此时间 则认为太慢了
// create requestHandler function
module.exports = function createRequestHandler (createIntercepts, middlewares, externalProxy, dnsConfig, setting) {
module.exports = function createRequestHandler (createIntercepts, middlewares, externalProxy, dnsConfig, setting, compatibleConfig) {
// return
return function requestHandler (req, res, ssl) {
let proxyReq
const rOptions = commonUtil.getOptionsFromRequest(req, ssl, externalProxy, setting)
const rOptions = commonUtil.getOptionsFromRequest(req, ssl, externalProxy, setting, compatibleConfig)
let url = `${rOptions.method}${rOptions.protocol}//${rOptions.hostname}:${rOptions.port}${rOptions.path}`
if (rOptions.headers.connection === 'close') {
@ -130,6 +131,19 @@ module.exports = function createRequestHandler (createIntercepts, middlewares, e
// log.debug('agent:', rOptions.agent)
// log.debug('agent.options:', rOptions.agent.options)
res.setHeader('DS-Proxy-Request', rOptions.hostname)
// 兼容程序:2
if (rOptions.agent) {
const compatibleConfig = compatible.getRequestCompatibleConfig(rOptions, rOptions.compatibleConfig)
if (compatibleConfig && compatibleConfig.rejectUnauthorized != null && rOptions.agent.options.rejectUnauthorized !== compatibleConfig.rejectUnauthorized) {
if (compatibleConfig.rejectUnauthorized === false && rOptions.agent.unVerifySslAgent) {
log.info(`【兼容程序】${rOptions.hostname}:${rOptions.port}: 设置 'rOptions.agent.options.rejectUnauthorized = ${compatibleConfig.rejectUnauthorized}'`)
rOptions.agent = rOptions.agent.unVerifySslAgent
res.setHeader('DS-Compatible', 'unVerifySsl')
}
}
}
proxyReq = (rOptions.protocol === 'https:' ? https : http).request(rOptions, (proxyRes) => {
const cost = new Date() - start
if (rOptions.protocol === 'https:') {
@ -163,6 +177,11 @@ module.exports = function createRequestHandler (createIntercepts, middlewares, e
log.error(`代理请求错误: ${url}, cost: ${cost} ms, error:`, e, ', rOptions:', jsonApi.stringify2(rOptions))
countSlow(isDnsIntercept, '代理请求错误: ' + e.message)
reject(e)
// 兼容程序:2
if (e.code === 'DEPTH_ZERO_SELF_SIGNED_CERT') {
compatible.setRequestRejectUnauthorized(rOptions, false)
}
})
proxyReq.on('aborted', () => {
const cost = new Date() - start

123
packages/mitmproxy/src/lib/proxy/tls/FakeServersCenter.js

@ -1,4 +1,5 @@
const https = require('https')
const http = require('http')
const tlsUtils = require('./tlsUtils')
const CertAndKeyContainer = require('./CertAndKeyContainer')
const forge = require('node-forge')
@ -6,6 +7,7 @@ const pki = forge.pki
// const colors = require('colors')
const tls = require('tls')
const log = require('../../../utils/util.log')
const compatible = require('../compatible/compatible')
function arraysHaveSameElements (arr1, arr2) {
if (arr1.length !== arr2.length) {
@ -43,15 +45,28 @@ module.exports = class FakeServersCenter {
return serverPromiseObj
}
getServerPromise (hostname, port) {
getServerPromise (hostname, port, ssl, manualCompatibleConfig) {
if (port === 443 || port === 80) {
ssl = port === 443
} else {
// 兼容程序:1
const compatibleConfig = compatible.getConnectCompatibleConfig(hostname, port, manualCompatibleConfig)
if (compatibleConfig && compatibleConfig.ssl != null) {
ssl = compatibleConfig.ssl
}
}
log.info(`getServerPromise, hostname: ${hostname}:${port}, ssl: ${ssl}, protocol: ${ssl ? 'https' : 'http'}`)
for (let i = 0; i < this.queue.length; i++) {
const serverPromiseObj = this.queue[i]
if (serverPromiseObj.port === port) {
if (serverPromiseObj.port === port && serverPromiseObj.ssl === ssl) {
const mappingHostNames = serverPromiseObj.mappingHostNames
for (let j = 0; j < mappingHostNames.length; j++) {
const DNSName = mappingHostNames[j]
if (tlsUtils.isMappingHostName(DNSName, hostname)) {
this.reRankServer(i)
log.info(`Load promise from cache, hostname: ${hostname}:${port}, ssl: ${ssl}, serverPromiseObj: {"ssl":${serverPromiseObj.ssl},"port":${serverPromiseObj.port},"mappingHostNames":${JSON.stringify(serverPromiseObj.mappingHostNames)}}`)
return serverPromiseObj.promise
}
}
@ -60,30 +75,38 @@ module.exports = class FakeServersCenter {
const serverPromiseObj = {
port,
ssl,
mappingHostNames: [hostname] // temporary hostname
}
const promise = new Promise((resolve, reject) => {
(async () => {
const certObj = await this.certAndKeyContainer.getCertPromise(hostname, port)
const cert = certObj.cert
const key = certObj.key
const certPem = pki.certificateToPem(cert)
const keyPem = pki.privateKeyToPem(key)
const fakeServer = new https.Server({
key: keyPem,
cert: certPem,
SNICallback: (hostname, done) => {
(async () => {
const certObj = await this.certAndKeyContainer.getCertPromise(hostname, port)
log.info(`sni callback: ${hostname}:${port}`)
done(null, tls.createSecureContext({
key: pki.privateKeyToPem(certObj.key),
cert: pki.certificateToPem(certObj.cert)
}))
})()
}
})
let fakeServer
let cert
let key
if (ssl) {
const certObj = await this.certAndKeyContainer.getCertPromise(hostname, port)
cert = certObj.cert
key = certObj.key
const certPem = pki.certificateToPem(cert)
const keyPem = pki.privateKeyToPem(key)
fakeServer = new https.Server({
key: keyPem,
cert: certPem,
SNICallback: (hostname, done) => {
(async () => {
const certObj = await this.certAndKeyContainer.getCertPromise(hostname, port)
log.info(`fakeServer SNICallback: ${hostname}:${port}`)
done(null, tls.createSecureContext({
key: pki.privateKeyToPem(certObj.key),
cert: pki.certificateToPem(certObj.cert)
}))
})()
}
})
} else {
fakeServer = new http.Server()
}
const serverObj = {
cert,
key,
@ -97,7 +120,6 @@ module.exports = class FakeServersCenter {
serverObj.port = address.port
})
fakeServer.on('request', (req, res) => {
const ssl = true
log.debug(`【fakeServer request - ${hostname}:${port}\r\n----- req -----\r\n`, req, '\r\n----- res -----\r\n', res)
this.requestHandler(req, res, ssl)
})
@ -116,7 +138,6 @@ module.exports = class FakeServersCenter {
resolve(serverObj)
})
fakeServer.on('upgrade', (req, socket, head) => {
const ssl = true
if (process.env.NODE_ENV === 'development') {
log.debug(`【fakeServer upgrade - ${hostname}:${port}\r\n----- req -----\r\n`, req, '\r\n----- socket -----\r\n', socket, '\r\n----- head -----\r\n', head)
} else {
@ -132,29 +153,47 @@ module.exports = class FakeServersCenter {
fakeServer.on('clientError', (err, socket) => {
// log.error(`【fakeServer clientError - ${hostname}:${port}】\r\n----- error -----\r\n`, err, '\r\n----- socket -----\r\n', socket)
log.error(`【fakeServer clientError - ${hostname}:${port}\r\n`, err)
// 兼容程序:1
if (port !== 443 && port !== 80) {
if (ssl === true && err.code.indexOf('ERR_SSL_') === 0) {
compatible.setConnectSsl(hostname, port, false)
log.error(`兼容程序:SSL异常,现设置为禁用ssl: ${hostname}:${port}, ssl = false`)
} else if (ssl === false && err.code === 'HPE_INVALID_METHOD') {
compatible.setConnectSsl(hostname, port, true)
log.error(`兼容程序:${err.code},现设置为启用ssl: ${hostname}:${port}, ssl = true`)
}
}
})
fakeServer.on('tlsClientError', (err, tlsSocket) => {
// log.error(`【fakeServer tlsClientError - ${hostname}:${port}】\r\n----- error -----\r\n`, err, '\r\n----- tlsSocket -----\r\n', tlsSocket)
log.error(`【fakeServer tlsClientError - ${hostname}:${port}\r\n`, err)
})
if (ssl) {
fakeServer.on('tlsClientError', (err, tlsSocket) => {
if (err.code === 'ECONNRESET' || err.code === 'ETIMEDOUT') {
return // 在tlsClientError事件中,以上异常不记录日志
}
// log.error(`【fakeServer tlsClientError - ${hostname}:${port}】\r\n----- error -----\r\n`, err, '\r\n----- tlsSocket -----\r\n', tlsSocket)
log.error(`【fakeServer tlsClientError - ${hostname}:${port}\r\n`, err)
})
}
// 其他监听事件,只打印debug日志
if (process.env.NODE_ENV === 'development') {
fakeServer.on('keylog', (line, tlsSocket) => {
log.debug(`【fakeServer keylog - ${hostname}:${port}\r\n----- line -----\r\n`, line, '\r\n----- tlsSocket -----\r\n', tlsSocket)
})
// fakeServer.on('newSession', (sessionId, sessionData, callback) => {
// log.debug('【fakeServer newSession - ${hostname}:${port}】\r\n----- sessionId -----\r\n', sessionId, '\r\n----- sessionData -----\r\n', sessionData, '\r\n----- callback -----\r\n', callback)
// })
// fakeServer.on('OCSPRequest', (certificate, issuer, callback) => {
// log.debug('【fakeServer OCSPRequest - ${hostname}:${port}】\r\n----- certificate -----\r\n', certificate, '\r\n----- issuer -----\r\n', issuer, '\r\n----- callback -----\r\n', callback)
// })
// fakeServer.on('resumeSession', (sessionId, callback) => {
// log.debug('【fakeServer resumeSession - ${hostname}:${port}】\r\n----- sessionId -----\r\n', sessionId, '\r\n----- callback -----\r\n', callback)
// })
fakeServer.on('secureConnection', (tlsSocket) => {
log.debug(`【fakeServer secureConnection - ${hostname}:${port}\r\n----- tlsSocket -----\r\n`, tlsSocket)
})
if (ssl) {
fakeServer.on('keylog', (line, tlsSocket) => {
log.debug(`【fakeServer keylog - ${hostname}:${port}\r\n----- line -----\r\n`, line, '\r\n----- tlsSocket -----\r\n', tlsSocket)
})
// fakeServer.on('newSession', (sessionId, sessionData, callback) => {
// log.debug(`【fakeServer newSession - ${hostname}:${port}】\r\n----- sessionId -----\r\n`, sessionId, '\r\n----- sessionData -----\r\n', sessionData, '\r\n----- callback -----\r\n', callback)
// })
// fakeServer.on('OCSPRequest', (certificate, issuer, callback) => {
// log.debug(`【fakeServer OCSPRequest - ${hostname}:${port}】\r\n----- certificate -----\r\n`, certificate, '\r\n----- issuer -----\r\n', issuer, '\r\n----- callback -----\r\n', callback)
// })
// fakeServer.on('resumeSession', (sessionId, callback) => {
// log.debug(`【fakeServer resumeSession - ${hostname}:${port}】\r\n----- sessionId -----\r\n`, sessionId, '\r\n----- callback -----\r\n', callback)
// })
fakeServer.on('secureConnection', (tlsSocket) => {
log.debug(`【fakeServer secureConnection - ${hostname}:${port}\r\n----- tlsSocket -----\r\n`, tlsSocket)
})
}
fakeServer.on('close', () => {
log.debug(`【fakeServer close - ${hostname}:${port}】no arguments...`)
})

4
packages/mitmproxy/src/options.js

@ -99,6 +99,10 @@ module.exports = (serverConfig) => {
speedTest: serverConfig.dns.speedTest
},
setting,
compatibleConfig: {
connect: serverConfig.compatible ? matchUtil.domainMapRegexply(serverConfig.compatible.connect) : {},
request: serverConfig.compatible ? matchUtil.domainMapRegexply(serverConfig.compatible.request) : {}
},
middlewares,
sslConnectInterceptor: (req, cltSocket, head) => {
const hostname = req.url.split(':')[0]

Loading…
Cancel
Save