github
/
dev-sidecar
mirror of https://github.com/docmirror/dev-sidecar
1
0
Fork 0
Code Issues Releases Wiki Activity

docs: 优选优化

pull/180/head
xiaojunnuo 2020-11-16 18:47:34 +08:00
parent ffa7a24835
commit e8c8bf4ca2
11 changed files with 269 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
packages/mitmproxy/src/lib/choice/DnsCounter.js Normal file
View File

@ -0,0 +1,3 @@
const { ChoiceCache } = require('./index')
module.exports = new ChoiceCache()

3
packages/mitmproxy/src/lib/choice/RequestCounter.js Normal file
View File

@ -0,0 +1,3 @@
const { ChoiceCache } = require('./index')
module.exports = new ChoiceCache()

107
packages/mitmproxy/src/lib/choice/index.js Normal file
View File

@ -0,0 +1,107 @@
const LRU = require('lru-cache')
const cacheSize = 1024
class ChoiceCache {
constructor () {
this.cache = new LRU(cacheSize)
}
get (key) {
return this.cache.get(key)
}
getOrCreate (key, backups) {
let item = this.cache.get(key)
if (item == null) {
item = new DynamicChoice(key)
item.setBackupList(backups)
this.cache.set(key, item)
}
return item
}
}
class DynamicChoice {
constructor (key) {
this.key = key
this.count = {}
this.createTime = new Date()
}
/**
* 设置新的backup列表
* @param backupList
*/
setBackupList (backupList) {
this.value = backupList.shift()
this.backup = backupList
for (const item of backupList) {
if (this.count[item]) {
continue
}
this.count[item] = { value: item, total: 0, error: 0, keepErrorCount: 0, successRate: 1 }
}
this.doCount(this.value, false)
}
doRank () {
// 将count里面根据权重排序
const list = []
for (const key in this.count) {
list.put(this.count[key])
}
list.sort(function (a, b) { return a.successRate - b.successRate })
const backup = list.map(item => item.value)
this.setBackupList(backup)
}
countStart (value) {
this.doCount(value, false)
}
/**
* 换下一个
* @param count
*/
changeNext (count) {
count.keepErrorCount = 0 // 清空连续失败
if (this.backup > 0) {
this.value = this.backup.shift()
} else {
this.value = null
}
}
/**
* 记录使用次数或错误次数
* @param value
* @param isError
*/
doCount (value, isError) {
let count = this.count[value]
if (count == null) {
count = this.count[value] = { value: value, total: 0, error: 0, keepErrorCount: 0, successRate: 1 }
}
if (isError) {
count.error++
count.keepErrorCount++
} else {
count.total++
}
count.successRate = 1 - (count.error / count.total)
if (isError && this.value === value) {
// 连续错误4次，切换下一个
if (count.keepErrorCount >= 4) {
this.changeNext(count)
}
// 成功率小于50%,切换下一个
if (count.successRate < 0.51) {
this.changeNext(count)
}
}
}
}
module.exports = {
DynamicChoice,
ChoiceCache
}

67
packages/mitmproxy/src/lib/dns/base.js
View File

@ -1,69 +1,26 @@
const LRU = require('lru-cache')
// const { isIP } = require('validator')
const log = require('../../utils/util.log')
const { DynamicChoice } = require('../choice/index')
const cacheSize = 1024
// eslint-disable-next-line no-unused-vars
// function _isIP (v) {
// return v && isIP(v)
// }
class IpCache {
class IpCache extends DynamicChoice {
constructor (hostname) {
this.hostname = hostname
this.count = {}
super(hostname)
this.lookupCount = 0
this.createTime = new Date()
}
/**
* 获取到新的ipList
* @param ipList
*/
setIpList (ipList) {
this.ip = ipList.shift()
this.ipList = ipList
setBackupList (ipList) {
super.setBackupList(ipList)
this.lookupCount++
this.doCount(this.ip, false)
}
/**
* 换下一个ip
* @param count
*/
changeNext (count) {
count.keepErrorCount = 0 // 清空连续失败
if (this.ipList > 0) {
this.ip = this.ipList.shift()
} else {
this.ip = null
}
}
/**
* 记录使用次数或错误次数
* @param ip
* @param isError
*/
doCount (ip, isError) {
let count = this.count[ip]
if (count == null) {
count = this.count[ip] = { total: 0, error: 0, keepErrorCount: 0, successRate: 0 }
}
if (isError) {
count.error++
count.keepErrorCount++
} else {
count.total++
}
count.successRate = 1 - (count.error / count.total)
if (isError && this.ip === ip) {
if (count.keepErrorCount >= 5) {
this.changeNext(count)
}
if (count.successRate < 0.51) {
this.changeNext(count)
}
}
}
}
@ -83,9 +40,9 @@ module.exports = class BaseDNS {
try {
let ipCache = this.cache.get(hostname)
if (ipCache) {
if (ipCache.ip != null) {
ipCache.doCount(ipCache.ip, false)
return ipCache.ip
if (ipCache.value != null) {
ipCache.doCount(ipCache.value, false)
return ipCache.value
}
} else {
ipCache = new IpCache(hostname)
@ -100,13 +57,13 @@ module.exports = class BaseDNS {
}
ipList.push(hostname) // 把原域名加入到统计里去
ipCache.setIpList(ipList)
ipCache.setBackupList(ipList)
log.info(`[DNS] ${hostname} -> ${ipCache.ip} (${new Date() - t} ms)`)
log.info(`[DNS] ${hostname} -> ${ipCache.value} (${new Date() - t} ms)`)
return ipCache.ip
return ipCache.value
} catch (error) {
log.error(`[DNS] cannot resolve hostname ${hostname} (${error})`)
log.error(`[DNS] cannot resolve hostname ${hostname} (${error})`, error)
return hostname
}
}

25
packages/mitmproxy/src/lib/interceptor/impl/abort.js
View File

@ -1,16 +1,13 @@
module.exports = function createIntercept (context) {
const { log } = context
return {
requestIntercept (interceptOpt, rOptions, req, res, ssl) {
log.info('abort:', rOptions.hostname, req.url)
res.writeHead(403)
res.write('DevSidecar 403: \n\n request abort, this request is matched by abort intercept.\n\n 因配置abort拦截器，本请求将取消')
res.end()
return true// 是否结束
},
is (interceptOpt) {
return !!interceptOpt.abort
}
module.exports = {
requestIntercept (context, interceptOpts, req, res, ssl, next) {
const { rOptions, log } = context
log.info('abort:', rOptions.hostname, req.url)
res.writeHead(403)
res.write('DevSidecar 403: \n\n request abort, this request is matched by abort intercept.\n\n 因配置abort拦截器，本请求将取消')
res.end()
return true// 是否结束
},
is (interceptOpt) {
return !!interceptOpt.abort
}
}

75
packages/mitmproxy/src/lib/interceptor/impl/proxy.js
View File

@ -1,31 +1,54 @@
const url = require('url')
module.exports = function createInterceptor (context) {
const { log } = context
return {
requestIntercept (interceptOpt, rOptions, req, res, ssl, next) {
let proxyTarget = interceptOpt.proxy + req.url
if (interceptOpt.replace) {
const regexp = new RegExp(interceptOpt.replace)
proxyTarget = req.url.replace(regexp, interceptOpt.proxy)
}
log.info('proxy', rOptions.path, rOptions.url)
// const backup = interceptOpt.backup
const proxy = proxyTarget.indexOf('http') === 0 ? proxyTarget : rOptions.protocol + '//' + proxyTarget
// eslint-disable-next-line node/no-deprecated-api
const URL = url.parse(proxy)
rOptions.protocol = URL.protocol
rOptions.hostname = URL.host
rOptions.host = URL.host
rOptions.headers.host = URL.host
rOptions.path = URL.path
if (URL.port == null) {
rOptions.port = rOptions.protocol === 'https:' ? 443 : 80
}
module.exports = {
requestIntercept (context, interceptOpt, req, res, ssl, next) {
const { rOptions, log, RequestConter } = context
log.info('proxy:', rOptions.hostname, req.url, proxyTarget)
},
is (interceptOpt) {
return !!interceptOpt.proxy
let proxyConf = interceptOpt.proxy
if (RequestConter && interceptOpt.backup && interceptOpt.backup.length > 0) {
// 优选逻辑
const backup = [proxyConf]
for (const bk of interceptOpt.backup) {
backup.push(bk)
}
const key = interceptOpt.key
const count = RequestConter.getOrCreate(key, backup)
if (count.value == null) {
count.doRank()
}
if (count.value == null) {
log.error('count value is null', count)
} else {
proxyConf = count.value
context.requestCount = {
key,
value: count.value,
count
}
}
}
let proxyTarget = proxyConf + req.url
if (interceptOpt.replace) {
const regexp = new RegExp(interceptOpt.replace)
proxyTarget = req.url.replace(regexp, proxyConf)
}
log.info('proxy', rOptions.path, rOptions.url)
// const backup = interceptOpt.backup
const proxy = proxyTarget.indexOf('http') === 0 ? proxyTarget : rOptions.protocol + '//' + proxyTarget
// eslint-disable-next-line node/no-deprecated-api
const URL = url.parse(proxy)
rOptions.protocol = URL.protocol
rOptions.hostname = URL.host
rOptions.host = URL.host
rOptions.headers.host = URL.host
rOptions.path = URL.path
if (URL.port == null) {
rOptions.port = rOptions.protocol === 'https:' ? 443 : 80
}
log.info('proxy:', rOptions.hostname, req.url, proxyTarget)
return true
},
is (interceptOpt) {
return !!interceptOpt.proxy
}
}

34
packages/mitmproxy/src/lib/interceptor/impl/redirect.js
View File

@ -1,21 +1,19 @@
module.exports = function createInterceptor (context) {
const { log } = context
return {
requestIntercept (interceptOpt, rOptions, req, res, ssl) {
const url = req.url
let redirect
if (typeof interceptOpt.redirect === 'string') {
redirect = rOptions.protocol + '//' + interceptOpt.redirect + url
} else {
redirect = interceptOpt.redirect(url)
}
log.info('请求重定向：', rOptions.hostname, url, redirect)
res.writeHead(302, { Location: redirect })
res.end()
return true// 是否结束
},
is (interceptOpt) {
return interceptOpt.redirect // 如果配置中有redirect，那么这个配置是需要redirect拦截的
module.exports = {
requestIntercept (context, interceptOpt, req, res, ssl, next) {
const { rOptions, log } = context
const url = req.url
let redirect
if (typeof interceptOpt.redirect === 'string') {
redirect = rOptions.protocol + '//' + interceptOpt.redirect + url
} else {
redirect = interceptOpt.redirect(url)
}
log.info('请求重定向：', rOptions.hostname, url, redirect)
res.writeHead(302, { Location: redirect })
res.end()
return true// 是否结束
},
is (interceptOpt) {
return interceptOpt.redirect // 如果配置中有redirect，那么这个配置是需要redirect拦截的
}
}

22
packages/mitmproxy/src/lib/interceptor/impl/script.js
View File

@ -1,8 +1,7 @@
module.exports = function createInterceptor (context) {
const { log } = context
return {
responseIntercept (interceptOpt, rOptions, req, res, proxyReq, proxyRes, ssl) {
const script = `
module.exports = {
responseIntercept (context, interceptOpt, req, res, proxyReq, proxyRes, ssl, next) {
const { rOptions, log } = context
const script = `
<script>
try{
${interceptOpt.script}
@ -11,13 +10,12 @@ module.exports = function createInterceptor (context) {
}
</script>
`
log.info('responseIntercept: body script', rOptions.hostname, rOptions.path)
return {
body: script
}
},
is (interceptOpt) {
return interceptOpt.script
log.info('responseIntercept: append script', rOptions.hostname, rOptions.path)
return {
head: script
}
},
is (interceptOpt) {
return interceptOpt.script
}
}

4
packages/mitmproxy/src/lib/interceptor/index.js
View File

@ -2,8 +2,6 @@ const proxy = require('./impl/proxy')
const redirect = require('./impl/redirect')
const abort = require('./impl/abort')
const script = require('./impl/script')
const log = require('../../utils/util.log')
const context = { log }
const modules = [proxy(context), redirect(context), abort(context), script(context)]
const modules = [proxy, redirect, abort, script]
module.exports = modules

69
packages/mitmproxy/src/lib/proxy/mitmproxy/createRequestHandler.js
View File

@ -5,6 +5,7 @@ const commonUtil = require('../common/util')
const DnsUtil = require('../../dns/index')
const log = require('../../../utils/util.log')
const HtmlMiddleware = require('../middleware/HtmlMiddleware')
const RequestCounter = require('../../choice/RequestCounter')
// create requestHandler function
module.exports = function createRequestHandler (createIntercepts, externalProxy, dnsConfig) {
// return
@ -19,7 +20,12 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
} else {
req.socket.setKeepAlive(true, 30000)
}
let interceptors = createIntercepts(rOptions)
const context = {
rOptions,
log,
RequestCounter
}
let interceptors = createIntercepts(context)
if (interceptors == null) {
interceptors = []
}
@ -34,8 +40,8 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
try {
if (reqIncpts && reqIncpts.length > 0) {
for (const reqIncpt of reqIncpts) {
const writableEnded = reqIncpt.requestIntercept(req, res, ssl)
if (writableEnded) {
const goNext = reqIncpt.requestIntercept(context, req, res, ssl, next)
if (goNext) {
next()
return
}
@ -99,6 +105,11 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
dns.count(hostname, ip, true)
log.error('记录ip失败次数,用于优选ip：', hostname, ip)
}
const counter = context.requestCount
if (counter != null) {
counter.count.doCount(counter.value, true)
log.error('记录prxoy失败次数：', counter.value)
}
log.error('代理请求超时', rOptions.protocol, rOptions.hostname, rOptions.path, (end - start) + 'ms')
proxyReq.end()
proxyReq.destroy()
@ -114,12 +125,33 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
dns.count(hostname, ip, true)
log.error('记录ip失败次数,用于优选ip：', hostname, ip)
}
const counter = context.requestCount
if (counter != null) {
counter.count.doCount(counter.value, true)
log.error('记录prxoy失败次数：', counter.value)
}
log.error('代理请求错误', e.code, e.message, rOptions.hostname, rOptions.path, (end - start) + 'ms')
reject(e)
})
proxyReq.on('aborted', () => {
log.error('代理请求被取消', rOptions.hostname, rOptions.path)
const end = new Date().getTime()
const cost = end - start
log.error('代理请求被取消', rOptions.hostname, rOptions.path, cost + 'ms')
if (cost > 8000) {
if (isDnsIntercept) {
const { dns, ip, hostname } = isDnsIntercept
dns.count(hostname, ip, true)
log.error('记录ip失败次数,用于优选ip：', hostname, ip)
}
const counter = context.requestCount
if (counter != null) {
counter.count.doCount(counter.value, true)
log.error('记录prxoy失败次数：', counter.value)
}
}
if (res.writableEnded) {
return
}
@ -161,6 +193,11 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
// // console.log('BODY: ')
// })
proxyRes.on('error', (error) => {
const counter = context.requestCount
if (counter != null) {
counter.count.doCount(counter.value, true)
log.error('记录prxoy失败次数：', counter.value)
}
log.error('proxy res error', error)
})
@ -173,7 +210,7 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
let head = ''
let body = ''
for (const resIncpt of resIncpts) {
const append = resIncpt.responseIntercept(req, res, proxyReq, proxyRes, ssl)
const append = resIncpt.responseIntercept(context, req, res, proxyReq, proxyRes, ssl)
if (append && append.head) {
head += append.head
}
@ -209,20 +246,14 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
res.writeHead(proxyRes.statusCode)
proxyRes.pipe(res)
}
})().then(
(flag) => {
// do nothing
// console.log('res', flag)
},
(e) => {
if (!res.writableEnded) {
const status = e.status || 500
res.writeHead(status)
res.write(`DevSidecar Warning:\n\n ${e.toString()}`)
res.end()
log.error('request error', e.message)
}
})().catch(e => {
if (!res.writableEnded) {
const status = e.status || 500
res.writeHead(status)
res.write(`DevSidecar Warning:\n\n ${e.toString()}`)
res.end()
log.error('request error', e.message)
}
)
})
}
}

12
packages/mitmproxy/src/options.js
View File

@ -63,7 +63,8 @@ module.exports = (config) => {
}
return !!matchHostname(intercepts, hostname) // 配置了拦截的域名，将会被代理
},
createIntercepts: (rOptions) => {
createIntercepts: (context) => {
const rOptions = context.rOptions
const hostname = rOptions.hostname
const interceptOpts = matchHostname(intercepts, hostname)
if (!interceptOpts) { // 该域名没有配置拦截器，直接过
@ -73,6 +74,7 @@ module.exports = (config) => {
const matchIntercepts = []
for (const regexp in interceptOpts) { // 遍历拦截配置
const interceptOpt = interceptOpts[regexp]
interceptOpt.key = regexp
if (regexp !== true) {
if (!isMatched(rOptions.path, regexp)) {
continue
@ -84,13 +86,13 @@ module.exports = (config) => {
const interceptor = {}
if (impl.requestIntercept) {
// req拦截器
interceptor.requestIntercept = (req, res, ssl) => {
impl.requestIntercept(interceptOpt, rOptions, req, res, ssl)
interceptor.requestIntercept = (context, req, res, ssl, next) => {
return impl.requestIntercept(context, interceptOpt, req, res, ssl, next)
}
} else if (impl.responseIntercept) {
// res拦截器
interceptor.responseIntercept = (req, res, proxyReq, proxyRes, ssl) => {
impl.responseIntercept(interceptOpt, rOptions, req, res, proxyReq, proxyRes, ssl)
interceptor.responseIntercept = (context, req, res, proxyReq, proxyRes, ssl, next) => {
return impl.responseIntercept(context, interceptOpt, req, res, proxyReq, proxyRes, ssl, next)
}
}
matchIntercepts.push(interceptor)