docs: 优选优化

packages/mitmproxy/src/lib/choice/DnsCounter.js

@@ -0,0 +1,3 @@
+const { ChoiceCache } = require('./index')
+
+module.exports = new ChoiceCache()

packages/mitmproxy/src/lib/choice/RequestCounter.js

@@ -0,0 +1,3 @@
+const { ChoiceCache } = require('./index')
+
+module.exports = new ChoiceCache()

packages/mitmproxy/src/lib/choice/index.js

@@ -0,0 +1,107 @@
+const LRU = require('lru-cache')
+const cacheSize = 1024
+class ChoiceCache {
+  constructor () {
+    this.cache = new LRU(cacheSize)
+  }
+
+  get (key) {
+    return this.cache.get(key)
+  }
+
+  getOrCreate (key, backups) {
+    let item = this.cache.get(key)
+    if (item == null) {
+      item = new DynamicChoice(key)
+      item.setBackupList(backups)
+      this.cache.set(key, item)
+    }
+    return item
+  }
+}
+
+class DynamicChoice {
+  constructor (key) {
+    this.key = key
+    this.count = {}
+    this.createTime = new Date()
+  }
+
+  /**
+     * 设置新的backup列表
+     * @param backupList
+     */
+  setBackupList (backupList) {
+    this.value = backupList.shift()
+    this.backup = backupList
+    for (const item of backupList) {
+      if (this.count[item]) {
+        continue
+      }
+      this.count[item] = { value: item, total: 0, error: 0, keepErrorCount: 0, successRate: 1 }
+    }
+    this.doCount(this.value, false)
+  }
+
+  doRank () {
+    // 将count里面根据权重排序
+    const list = []
+    for (const key in this.count) {
+      list.put(this.count[key])
+    }
+    list.sort(function (a, b) { return a.successRate - b.successRate })
+    const backup = list.map(item => item.value)
+    this.setBackupList(backup)
+  }
+
+  countStart (value) {
+    this.doCount(value, false)
+  }
+
+  /**
+     * 换下一个
+     * @param count
+     */
+  changeNext (count) {
+    count.keepErrorCount = 0 // 清空连续失败
+    if (this.backup > 0) {
+      this.value = this.backup.shift()
+    } else {
+      this.value = null
+    }
+  }
+
+  /**
+     * 记录使用次数或错误次数
+     * @param value
+     * @param isError
+     */
+  doCount (value, isError) {
+    let count = this.count[value]
+    if (count == null) {
+      count = this.count[value] = { value: value, total: 0, error: 0, keepErrorCount: 0, successRate: 1 }
+    }
+    if (isError) {
+      count.error++
+      count.keepErrorCount++
+    } else {
+      count.total++
+    }
+    count.successRate = 1 - (count.error / count.total)
+    if (isError && this.value === value) {
+      // 连续错误4次，切换下一个
+      if (count.keepErrorCount >= 4) {
+        this.changeNext(count)
+      }
+      // 成功率小于50%,切换下一个
+      if (count.successRate < 0.51) {
+        this.changeNext(count)
+      }
+    }
+  }
+}
+
+module.exports = {
+  DynamicChoice,
+  ChoiceCache
+}

packages/mitmproxy/src/lib/dns/base.js

@@ -1,69 +1,26 @@
 const LRU = require('lru-cache')
 // const { isIP } = require('validator')
 const log = require('../../utils/util.log')
+const { DynamicChoice } = require('../choice/index')
 const cacheSize = 1024
 // eslint-disable-next-line no-unused-vars
 // function _isIP (v) {
 //   return v && isIP(v)
 // }
 
-class IpCache {
+class IpCache extends DynamicChoice {
   constructor (hostname) {
-    this.hostname = hostname
+    super(hostname)
-    this.count = {}
     this.lookupCount = 0
-    this.createTime = new Date()
   }
 
   /**
    * 获取到新的ipList
    * @param ipList
    */
-  setIpList (ipList) {
+  setBackupList (ipList) {
-    this.ip = ipList.shift()
+    super.setBackupList(ipList)
-    this.ipList = ipList
     this.lookupCount++
-    this.doCount(this.ip, false)
-  }
-
-  /**
-   * 换下一个ip
-   * @param count
-   */
-  changeNext (count) {
-    count.keepErrorCount = 0 // 清空连续失败
-    if (this.ipList > 0) {
-      this.ip = this.ipList.shift()
-    } else {
-      this.ip = null
-    }
-  }
-
-  /**
-   * 记录使用次数或错误次数
-   * @param ip
-   * @param isError
-   */
-  doCount (ip, isError) {
-    let count = this.count[ip]
-    if (count == null) {
-      count = this.count[ip] = { total: 0, error: 0, keepErrorCount: 0, successRate: 0 }
-    }
-    if (isError) {
-      count.error++
-      count.keepErrorCount++
-    } else {
-      count.total++
-    }
-    count.successRate = 1 - (count.error / count.total)
-    if (isError && this.ip === ip) {
-      if (count.keepErrorCount >= 5) {
-        this.changeNext(count)
-      }
-      if (count.successRate < 0.51) {
-        this.changeNext(count)
-      }
-    }
   }
 }
 
@@ -83,9 +40,9 @@ module.exports = class BaseDNS {
     try {
       let ipCache = this.cache.get(hostname)
       if (ipCache) {
-        if (ipCache.ip != null) {
+        if (ipCache.value != null) {
-          ipCache.doCount(ipCache.ip, false)
+          ipCache.doCount(ipCache.value, false)
-          return ipCache.ip
+          return ipCache.value
         }
       } else {
         ipCache = new IpCache(hostname)
@@ -100,13 +57,13 @@ module.exports = class BaseDNS {
       }
       ipList.push(hostname) // 把原域名加入到统计里去
 
-      ipCache.setIpList(ipList)
+      ipCache.setBackupList(ipList)
 
-      log.info(`[DNS] ${hostname} -> ${ipCache.ip} (${new Date() - t} ms)`)
+      log.info(`[DNS] ${hostname} -> ${ipCache.value} (${new Date() - t} ms)`)
 
-      return ipCache.ip
+      return ipCache.value
     } catch (error) {
-      log.error(`[DNS] cannot resolve hostname ${hostname} (${error})`)
+      log.error(`[DNS] cannot resolve hostname ${hostname} (${error})`, error)
       return hostname
     }
   }

packages/mitmproxy/src/lib/interceptor/impl/abort.js

@@ -1,8 +1,6 @@
-
+module.exports = {
-module.exports = function createIntercept (context) {
+  requestIntercept (context, interceptOpts, req, res, ssl, next) {
-  const { log } = context
+    const { rOptions, log } = context
-  return {
-    requestIntercept (interceptOpt, rOptions, req, res, ssl) {
     log.info('abort:', rOptions.hostname, req.url)
     res.writeHead(403)
     res.write('DevSidecar 403: \n\n request abort, this request is matched by abort intercept.\n\n 因配置abort拦截器，本请求将取消')
@@ -12,5 +10,4 @@ module.exports = function createIntercept (context) {
   is (interceptOpt) {
     return !!interceptOpt.abort
   }
-  }
 }

packages/mitmproxy/src/lib/interceptor/impl/proxy.js

@@ -1,12 +1,36 @@
 const url = require('url')
-module.exports = function createInterceptor (context) {
+module.exports = {
-  const { log } = context
+  requestIntercept (context, interceptOpt, req, res, ssl, next) {
-  return {
+    const { rOptions, log, RequestConter } = context
-    requestIntercept (interceptOpt, rOptions, req, res, ssl, next) {
+
-      let proxyTarget = interceptOpt.proxy + req.url
+    let proxyConf = interceptOpt.proxy
+    if (RequestConter && interceptOpt.backup && interceptOpt.backup.length > 0) {
+      // 优选逻辑
+      const backup = [proxyConf]
+      for (const bk of interceptOpt.backup) {
+        backup.push(bk)
+      }
+      const key = interceptOpt.key
+      const count = RequestConter.getOrCreate(key, backup)
+      if (count.value == null) {
+        count.doRank()
+      }
+      if (count.value == null) {
+        log.error('count value is null', count)
+      } else {
+        proxyConf = count.value
+        context.requestCount = {
+          key,
+          value: count.value,
+          count
+        }
+      }
+    }
+
+    let proxyTarget = proxyConf + req.url
     if (interceptOpt.replace) {
       const regexp = new RegExp(interceptOpt.replace)
-        proxyTarget = req.url.replace(regexp, interceptOpt.proxy)
+      proxyTarget = req.url.replace(regexp, proxyConf)
     }
     log.info('proxy', rOptions.path, rOptions.url)
     // const backup = interceptOpt.backup
@@ -21,11 +45,10 @@ module.exports = function createInterceptor (context) {
     if (URL.port == null) {
       rOptions.port = rOptions.protocol === 'https:' ? 443 : 80
     }
-
     log.info('proxy:', rOptions.hostname, req.url, proxyTarget)
+    return true
   },
   is (interceptOpt) {
     return !!interceptOpt.proxy
   }
-  }
 }

packages/mitmproxy/src/lib/interceptor/impl/redirect.js

@@ -1,7 +1,6 @@
-module.exports = function createInterceptor (context) {
+module.exports = {
-  const { log } = context
+  requestIntercept (context, interceptOpt, req, res, ssl, next) {
-  return {
+    const { rOptions, log } = context
-    requestIntercept (interceptOpt, rOptions, req, res, ssl) {
     const url = req.url
     let redirect
     if (typeof interceptOpt.redirect === 'string') {
@@ -17,5 +16,4 @@ module.exports = function createInterceptor (context) {
   is (interceptOpt) {
     return interceptOpt.redirect // 如果配置中有redirect，那么这个配置是需要redirect拦截的
   }
-  }
 }

packages/mitmproxy/src/lib/interceptor/impl/script.js

@@ -1,7 +1,6 @@
-module.exports = function createInterceptor (context) {
+module.exports = {
-  const { log } = context
+  responseIntercept (context, interceptOpt, req, res, proxyReq, proxyRes, ssl, next) {
-  return {
+    const { rOptions, log } = context
-    responseIntercept (interceptOpt, rOptions, req, res, proxyReq, proxyRes, ssl) {
     const script = `
       <script>
       try{
@@ -11,13 +10,12 @@ module.exports = function createInterceptor (context) {
       }
       </script>
       `
-      log.info('responseIntercept: body script', rOptions.hostname, rOptions.path)
+    log.info('responseIntercept: append script', rOptions.hostname, rOptions.path)
     return {
-        body: script
+      head: script
     }
   },
   is (interceptOpt) {
     return interceptOpt.script
   }
-  }
 }

packages/mitmproxy/src/lib/interceptor/index.js

@@ -2,8 +2,6 @@ const proxy = require('./impl/proxy')
 const redirect = require('./impl/redirect')
 const abort = require('./impl/abort')
 const script = require('./impl/script')
-const log = require('../../utils/util.log')
+const modules = [proxy, redirect, abort, script]
-const context = { log }
-const modules = [proxy(context), redirect(context), abort(context), script(context)]
 
 module.exports = modules

packages/mitmproxy/src/lib/proxy/mitmproxy/createRequestHandler.js

@@ -5,6 +5,7 @@ const commonUtil = require('../common/util')
 const DnsUtil = require('../../dns/index')
 const log = require('../../../utils/util.log')
 const HtmlMiddleware = require('../middleware/HtmlMiddleware')
+const RequestCounter = require('../../choice/RequestCounter')
 // create requestHandler function
 module.exports = function createRequestHandler (createIntercepts, externalProxy, dnsConfig) {
   // return
@@ -19,7 +20,12 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
     } else {
       req.socket.setKeepAlive(true, 30000)
     }
-    let interceptors = createIntercepts(rOptions)
+    const context = {
+      rOptions,
+      log,
+      RequestCounter
+    }
+    let interceptors = createIntercepts(context)
     if (interceptors == null) {
       interceptors = []
     }
@@ -34,8 +40,8 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
         try {
           if (reqIncpts && reqIncpts.length > 0) {
             for (const reqIncpt of reqIncpts) {
-              const writableEnded = reqIncpt.requestIntercept(req, res, ssl)
+              const goNext = reqIncpt.requestIntercept(context, req, res, ssl, next)
-              if (writableEnded) {
+              if (goNext) {
                 next()
                 return
               }
@@ -99,6 +105,11 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
               dns.count(hostname, ip, true)
               log.error('记录ip失败次数,用于优选ip：', hostname, ip)
             }
+            const counter = context.requestCount
+            if (counter != null) {
+              counter.count.doCount(counter.value, true)
+              log.error('记录prxoy失败次数：', counter.value)
+            }
             log.error('代理请求超时', rOptions.protocol, rOptions.hostname, rOptions.path, (end - start) + 'ms')
             proxyReq.end()
             proxyReq.destroy()
@@ -114,12 +125,33 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
               dns.count(hostname, ip, true)
               log.error('记录ip失败次数,用于优选ip：', hostname, ip)
             }
+            const counter = context.requestCount
+            if (counter != null) {
+              counter.count.doCount(counter.value, true)
+              log.error('记录prxoy失败次数：', counter.value)
+            }
             log.error('代理请求错误', e.code, e.message, rOptions.hostname, rOptions.path, (end - start) + 'ms')
             reject(e)
           })
 
           proxyReq.on('aborted', () => {
-            log.error('代理请求被取消', rOptions.hostname, rOptions.path)
+            const end = new Date().getTime()
+            const cost = end - start
+            log.error('代理请求被取消', rOptions.hostname, rOptions.path, cost + 'ms')
+
+            if (cost > 8000) {
+              if (isDnsIntercept) {
+                const { dns, ip, hostname } = isDnsIntercept
+                dns.count(hostname, ip, true)
+                log.error('记录ip失败次数,用于优选ip：', hostname, ip)
+              }
+              const counter = context.requestCount
+              if (counter != null) {
+                counter.count.doCount(counter.value, true)
+                log.error('记录prxoy失败次数：', counter.value)
+              }
+            }
+
             if (res.writableEnded) {
               return
             }
@@ -161,6 +193,11 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
       //   // console.log('BODY: ')
       // })
       proxyRes.on('error', (error) => {
+        const counter = context.requestCount
+        if (counter != null) {
+          counter.count.doCount(counter.value, true)
+          log.error('记录prxoy失败次数：', counter.value)
+        }
         log.error('proxy res error', error)
       })
 
@@ -173,7 +210,7 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
             let head = ''
             let body = ''
             for (const resIncpt of resIncpts) {
-              const append = resIncpt.responseIntercept(req, res, proxyReq, proxyRes, ssl)
+              const append = resIncpt.responseIntercept(context, req, res, proxyReq, proxyRes, ssl)
               if (append && append.head) {
                 head += append.head
               }
@@ -209,12 +246,7 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
         res.writeHead(proxyRes.statusCode)
         proxyRes.pipe(res)
       }
-    })().then(
+    })().catch(e => {
-      (flag) => {
-        // do nothing
-        // console.log('res', flag)
-      },
-      (e) => {
       if (!res.writableEnded) {
         const status = e.status || 500
         res.writeHead(status)
@@ -222,7 +254,6 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
         res.end()
         log.error('request error', e.message)
       }
-      }
+    })
-    )
   }
 }

packages/mitmproxy/src/options.js

@@ -63,7 +63,8 @@ module.exports = (config) => {
       }
       return !!matchHostname(intercepts, hostname) // 配置了拦截的域名，将会被代理
     },
-    createIntercepts: (rOptions) => {
+    createIntercepts: (context) => {
+      const rOptions = context.rOptions
       const hostname = rOptions.hostname
       const interceptOpts = matchHostname(intercepts, hostname)
       if (!interceptOpts) { // 该域名没有配置拦截器，直接过
@@ -73,6 +74,7 @@ module.exports = (config) => {
       const matchIntercepts = []
       for (const regexp in interceptOpts) { // 遍历拦截配置
         const interceptOpt = interceptOpts[regexp]
+        interceptOpt.key = regexp
         if (regexp !== true) {
           if (!isMatched(rOptions.path, regexp)) {
             continue
@@ -84,13 +86,13 @@ module.exports = (config) => {
             const interceptor = {}
             if (impl.requestIntercept) {
               // req拦截器
-              interceptor.requestIntercept = (req, res, ssl) => {
+              interceptor.requestIntercept = (context, req, res, ssl, next) => {
-                impl.requestIntercept(interceptOpt, rOptions, req, res, ssl)
+                return impl.requestIntercept(context, interceptOpt, req, res, ssl, next)
               }
             } else if (impl.responseIntercept) {
               // res拦截器
-              interceptor.responseIntercept = (req, res, proxyReq, proxyRes, ssl) => {
+              interceptor.responseIntercept = (context, req, res, proxyReq, proxyRes, ssl, next) => {
-                impl.responseIntercept(interceptOpt, rOptions, req, res, proxyReq, proxyRes, ssl)
+                return impl.responseIntercept(context, interceptOpt, req, res, proxyReq, proxyRes, ssl, next)
               }
             }
             matchIntercepts.push(interceptor)