feat: speedTest

packages/core/src/config/index.js

@@ -152,11 +152,31 @@ module.exports = {
           server: 'https://dns.alidns.com/dns-query',
           cacheSize: 1000
         },
+        ipaddress: {
+          type: 'ipaddress',
+          server: 'ipaddress',
+          cacheSize: 1000
+        },
         usa: {
           type: 'https',
           server: 'https://1.1.1.1/dns-query',
           cacheSize: 1000
         },
+        quad9: {
+          type: 'https',
+          server: 'https://9.9.9.9/dns-query',
+          cacheSize: 1000
+        }
+        // google: {
+        //   type: 'https',
+        //   server: 'https://8.8.8.8/dns-query',
+        //   cacheSize: 1000
+        // },
+        // dnsSB: {
+        //   type: 'https',
+        //   server: 'https://doh.dns.sb/dns-query',
+        //   cacheSize: 1000
+        // }
       },
       mapping: {
         // 'assets.fastgit.org': 'usa',
@@ -167,9 +187,14 @@ module.exports = {
         '*.githubusercontent.com': 'usa',
         '*.githubassets.com': 'usa',
         // "解决push的时候需要输入密码的问题",
+        'github.com': 'usa',
         '*github.com': 'usa',
         '*.vuepress.vuejs.org': 'usa',
         'gh.docmirror.top': 'usa'
+      },
+      speedTest: {
+        hostnameList: ['github.com'],
+        dnsProviders: ['usa', 'quad9']
       }
     }
   },

packages/core/src/modules/plugin/overwall/config.js

@@ -1,6 +1,6 @@
 module.exports = {
   name: '梯子',
-  enabled: false,
+  enabled: false, // 默认关闭梯子
   server: {
   },
   serverDefault: {
@@ -10,7 +10,8 @@ module.exports = {
     }
   },
   targets: {
-    '*facebook.com': true
+    '*facebook.com': true,
+    'github.com': true
   },
   pac: {
     enabled: true,

packages/gui/src/view/components/setup-ca.vue

@@ -53,7 +53,6 @@ export default {
   },
   async created () {
     const platform = await this.$api.info.getSystemPlatform()
-    console.log('11', platform)
     this.systemPlatform = platform
   },
   computed: {

packages/gui/src/view/pages/server.vue

@@ -69,6 +69,7 @@
                 <a-select :disabled="item.value ===false" v-model="item.value">
                   <a-select-option value="usa">USA DNS</a-select-option>
                   <a-select-option value="aliyun">Aliyun DNS</a-select-option>
+                  <a-select-option value="ipaddress">IpAddress</a-select-option>
                 </a-select>
               </a-col>
               <a-col :span="3">

packages/mitmproxy/package.json

@@ -12,6 +12,7 @@
   "scripts": {},
   "dependencies": {
     "agentkeepalive": "^2.1.1",
+    "axios": "^0.21.1",
     "child_process": "^1.0.2",
     "colors": "^1.1.2",
     "commander": "^2.9.0",

packages/mitmproxy/src/lib/dns/index.js

@@ -1,10 +1,15 @@
 const DNSOverTLS = require('./tls.js')
 const DNSOverHTTPS = require('./https.js')
+const DNSOverIpAddress = require('./ipaddress.js')
 module.exports = {
   initDNS (dnsProviders) {
     const dnsMap = {}
     for (const provider in dnsProviders) {
       const conf = dnsProviders[provider]
+      if (conf.type === 'ipaddress') {
+        dnsMap[provider] = new DNSOverIpAddress(conf.server)
+        continue
+      }
       dnsMap[provider] = conf.type === 'https' ? new DNSOverHTTPS(conf.server) : new DNSOverTLS(conf.server)
     }
     return dnsMap

packages/mitmproxy/src/lib/dns/ipaddress.js

@@ -0,0 +1,40 @@
+const dnstls = require('dns-over-tls')
+const BaseDNS = require('./base')
+const axios = require('axios')
+const log = require('../../utils/util.log')
+const fs = require('fs')
+const path = require('path')
+module.exports = class DNSOverIpAddress extends BaseDNS {
+  async _lookup (hostname) {
+    const url = `https://${hostname}.ipaddress.com`
+
+    // const res = fs.readFileSync(path.resolve(__dirname, './data.txt')).toString()
+    const res = await axios.get(url)
+    if (res.status !== 200 && res.status !== 201) {
+      log.info(`[dns] get ${hostname} ipaddress: error:${res}`)
+      return
+    }
+    const ret = res.data
+
+    const regexp = /<tr><th>IP Address<\/th><td><ul class="comma-separated"><li>([^<]*)<\/li><\/ul><\/td><\/tr>/gm
+    const matched = regexp.exec(ret)
+    let ip = null
+
+    if (matched && matched.length >= 1) {
+      ip = matched[1]
+      log.info(`[dns] get ${hostname} ipaddress:${ip}`)
+      return [ip]
+    }
+    log.info(`[dns] get ${hostname} ipaddress: error`)
+    return null
+
+    // const { answers } = await dnstls.query(hostname)
+    //
+    // const answer = answers.find(answer => answer.type === 'A' && answer.class === 'IN')
+    //
+    // log.info('dns lookup：', hostname, answer)
+    // if (answer) {
+    //   return answer.data
+    // }
+  }
+}

packages/mitmproxy/src/lib/interceptor/impl/proxy.js

@@ -33,7 +33,8 @@ module.exports = {
       const regexp = new RegExp(interceptOpt.replace)
       proxyTarget = req.url.replace(regexp, proxyConf)
     }
-    // eslint-disable-next-line no-template-curly-in-string
+    // eslint-disable-next-line
+    // no-template-curly-in-string
     proxyTarget = proxyTarget.replace('${host}', rOptions.hostname)
 
     // const backup = interceptOpt.backup

packages/mitmproxy/src/lib/proxy/middleware/InsertScriptMiddleware.js

@@ -48,7 +48,6 @@ function injectScriptIntoHtml (tags, chunk, script) {
     if (index < 0) {
       continue
     }
-    console.log('insert script:', tag)
     const scriptBuf = Buffer.from(script)
     const chunkNew = Buffer.alloc(chunk.length + scriptBuf.length)
     chunk.copy(chunkNew, 0, 0, index)

packages/mitmproxy/src/lib/proxy/mitmproxy/createConnectHandler.js

@@ -6,6 +6,8 @@ const DnsUtil = require('../../dns/index')
 const localIP = '127.0.0.1'
 const defaultDns = require('dns')
 
+const speedTest = require('../../speed/index.js')
+
 function isSslConnect (sslConnectInterceptors, req, cltSocket, head) {
   for (const intercept of sslConnectInterceptors) {
     const ret = intercept(req, cltSocket, head)
@@ -32,6 +34,7 @@ module.exports = function createConnectHandler (sslConnectInterceptor, middlewar
     const hostname = srvUrl.hostname
     if (isSslConnect(sslConnectInterceptors, req, cltSocket, head)) {
       fakeServerCenter.getServerPromise(hostname, srvUrl.port).then((serverObj) => {
+        log.info('--- fakeServer connect', hostname)
         connect(req, cltSocket, head, localIP, serverObj.port)
       }, (e) => {
         log.error('getServerPromise', e)
@@ -57,9 +60,19 @@ function connect (req, cltSocket, head, hostname, port, dnsConfig) {
       const dns = DnsUtil.hasDnsLookup(dnsConfig, hostname)
       if (dns) {
         options.lookup = (hostname, options, callback) => {
+          const tester = speedTest.getSpeedTester(hostname)
+          if (tester) {
+            const ip = tester.pickFastAliveIp()
+            if (ip) {
+              log.info(`-----${hostname} use alive ip:${ip}-----`)
+              callback(null, ip, 4)
+              return
+            }
+          }
           dns.lookup(hostname).then(ip => {
             isDnsIntercept = { dns, hostname, ip }
             if (ip !== hostname) {
+              log.info(`-----${hostname} use ip:${ip}-----`)
               callback(null, ip, 4)
             } else {
               defaultDns.lookup(hostname, options, callback)

packages/mitmproxy/src/lib/proxy/mitmproxy/createRequestHandler.js

@@ -7,7 +7,7 @@ const log = require('../../../utils/util.log')
 const RequestCounter = require('../../choice/RequestCounter')
 const InsertScriptMiddleware = require('../middleware/InsertScriptMiddleware')
 const OverWallMiddleware = require('../middleware/overwall')
-
+const speedTest = require('../../speed/index.js')
 const defaultDns = require('dns')
 const MAX_SLOW_TIME = 8000 // 超过此时间 则认为太慢了
 // create requestHandler function
@@ -106,10 +106,19 @@ module.exports = function createRequestHandler (createIntercepts, middlewares, e
             const dns = DnsUtil.hasDnsLookup(dnsConfig, rOptions.hostname)
             if (dns) {
               rOptions.lookup = (hostname, options, callback) => {
+                const tester = speedTest.getSpeedTester(hostname)
+                if (tester) {
+                  const ip = tester.pickFastAliveIp()
+                  if (ip) {
+                    log.info(`-----${hostname} use alive ip:${ip}-----`)
+                    callback(null, ip, 4)
+                    return
+                  }
+                }
                 dns.lookup(hostname).then(ip => {
                   isDnsIntercept = { dns, hostname, ip }
                   if (ip !== hostname) {
-                    log.info(`request url :${url},use ip :${ip}`)
+                    log.info(`----request url :${url},use ip :${ip}----`)
                     callback(null, ip, 4)
                   } else {
                     log.info(`request url :${url},use hostname :${hostname}`)
@@ -266,7 +275,7 @@ module.exports = function createRequestHandler (createIntercepts, middlewares, e
     })().catch(e => {
       if (!res.writableEnded) {
         const status = e.status || 500
-        res.writeHead(status)
+        res.writeHead(status, { 'Content-Type': 'text/html;charset=UTF8' })
         res.write(`DevSidecar Warning:\n\n ${e.toString()}`)
         res.end()
         log.error('request error', e.message)

packages/mitmproxy/src/lib/proxy/mitmproxy/index.js

@@ -1,11 +1,15 @@
 const tlsUtils = require('../tls/tlsUtils')
 const http = require('http')
+const https = require('https')
 const config = require('../common/config')
 const log = require('../../../utils/util.log')
 const createRequestHandler = require('./createRequestHandler')
 const createConnectHandler = require('./createConnectHandler')
 const createFakeServerCenter = require('./createFakeServerCenter')
 const createUpgradeHandler = require('./createUpgradeHandler')
+const DnsUtil = require('../../dns/index')
+const defaultDns = require('dns')
+const speedTest = require('../../speed/index.js')
 module.exports = {
   createProxy ({
     port = config.defaultPort,
@@ -35,7 +39,40 @@ module.exports = {
       log.info(`CA private key saved in: ${caKeyPath}`)
     }
 
+    // function lookup (hostname, options, callback) {
+    //   const dns = DnsUtil.hasDnsLookup(dnsConfig, hostname)
+    //   if (dns) {
+    //     dns.lookup(hostname).then(ip => {
+    //       // isDnsIntercept = { dns, hostname, ip }
+    //       if (ip !== hostname) {
+    //         log.info(`-----${hostname} use ip:${ip}-----`)
+    //         callback(null, ip, 4)
+    //       } else {
+    //         defaultDns.lookup(hostname, options, callback)
+    //       }
+    //     })
+    //   } else {
+    //     defaultDns.lookup(hostname, options, callback)
+    //   }
+    // }
+    //
+    // https.globalAgent.lookup = lookup
+
     port = ~~port
+
+    const speedTestConfig = dnsConfig.speedTest
+    const dnsMap = dnsConfig.providers
+    if (speedTestConfig) {
+      const dnsProviders = speedTestConfig.dnsProviders
+      const map = {}
+      for (const dnsProvider of dnsProviders) {
+        if (dnsMap[dnsProvider]) {
+          map[dnsProvider] = dnsMap[dnsProvider]
+        }
+      }
+      speedTest.initSpeedTestPool({ hostnameList: speedTestConfig.hostnameList, dnsMap: map })
+    }
+
     const requestHandler = createRequestHandler(
       createIntercepts,
       middlewares,

packages/mitmproxy/src/lib/speed/SpeedTester.js

@@ -0,0 +1,144 @@
+const net = require('net')
+const config = require('./config.js')
+const log = require('../../utils/util.log.js')
+const DISABLE_TIMEOUT = 60 * 60 * 1000 // 1个小时不访问，取消获取
+
+class SpeedTester {
+  constructor ({ hostname }) {
+    this.dnsMap = config.getConfig().dnsMap
+    this.hostname = hostname
+    this.lastReadTime = Date.now()
+    this.ready = false
+    this.alive = []
+    this.backupList = []
+    this.keepCheckId = false
+
+    this.loadingIps = false
+    this.loadingTest = false
+
+    this.test()
+  }
+
+  pickFastAliveIp () {
+    this.touch()
+    if (this.alive.length === 0) {
+      this.test()
+      return null
+    }
+    return this.alive[0].host
+  }
+
+  touch () {
+    this.lastReadTime = Date.now()
+    if (!this.keepCheckId) {
+      this.startChecker()
+    }
+  }
+
+  startChecker () {
+    if (this.keepCheckId) {
+      clearInterval(this.keepCheckId)
+    }
+    this.keepCheckId = setInterval(() => {
+      if (Date.now() - DISABLE_TIMEOUT > this.lastReadTime) {
+        // 超过很长时间没有访问，取消测试
+        clearInterval(this.keepCheckId)
+        return
+      }
+      if (this.alive.length > 0) {
+        this.testBackups()
+        return
+      }
+      this.test()
+    }, 60 * 1000)
+  }
+
+  async getIpListFromDns (dnsMap) {
+    const ips = {}
+    const promiseList = []
+    for (const key in dnsMap) {
+      const one = this.getFromOneDns(dnsMap[key]).then(ipList => {
+        if (ipList) {
+          for (const ip of ipList) {
+            ips[ip] = 1
+          }
+        }
+      })
+      promiseList.push(one)
+    }
+    await Promise.all(promiseList)
+    const items = []
+    for (const ip in ips) {
+      items.push({ host: ip, port: 443 })
+    }
+    return items
+  }
+
+  async getFromOneDns (dns) {
+    return await dns._lookup(this.hostname)
+  }
+
+  async test () {
+    this.backupList = await this.getIpListFromDns(this.dnsMap)
+    log.info('[speed]', this.hostname, ' ips:', this.backupList)
+    this.testBackups()
+  }
+
+  async testBackups () {
+    const testAll = []
+    const aliveList = []
+    for (const item of this.backupList) {
+      testAll.push(this.doTest(item, aliveList))
+    }
+    await Promise.all(testAll)
+    this.alive = aliveList
+    this.ready = true
+  }
+
+  async doTest (item, aliveList) {
+    try {
+      const ret = await this.testOne(item)
+      aliveList.push({ ...ret, ...item })
+      aliveList.sort((a, b) => a.time - b.time)
+    } catch (e) {
+      log.error('[speed] test error', this.hostname, item.host, e.message)
+    }
+  }
+
+  testOne (item) {
+    const timeout = 5000
+    const { host, port } = item
+    const startTime = Date.now()
+    let isOver = false
+    return new Promise((resolve, reject) => {
+      let timeoutId = null
+      const client = net.createConnection({ host, port }, () => {
+        // 'connect' 监听器
+        const connectionTime = Date.now()
+        isOver = true
+        clearTimeout(timeoutId)
+        resolve({ status: 'success', time: connectionTime - startTime })
+        client.end()
+      })
+      client.on('end', () => {
+      })
+      client.on('error', (error) => {
+        log.error('[speed]test error', this.hostname, host, error.message)
+        isOver = true
+        clearTimeout(timeoutId)
+        reject(error)
+      })
+
+      timeoutId = setTimeout(() => {
+        if (isOver) {
+          return
+        }
+        log.error('[speed] test timeout', this.hostname, host)
+        reject(new Error('timeout'))
+        client.end()
+      }, timeout)
+    })
+  }
+}
+
+module.exports = SpeedTester

packages/mitmproxy/src/lib/speed/config.js

@@ -0,0 +1,8 @@
+const config = {
+
+}
+module.exports = {
+  getConfig () {
+    return config
+  }
+}

packages/mitmproxy/src/lib/speed/index.js

@@ -0,0 +1,28 @@
+const SpeedTester = require('./SpeedTester.js')
+const _ = require('lodash')
+const config = require('./config')
+
+const SpeedTestPool = {
+
+}
+function initSpeedTestPool ({ hostnameList, dnsMap }) {
+  config.getConfig().dnsMap = dnsMap
+  _.forEach(hostnameList, (hostname) => {
+    SpeedTestPool[hostname] = new SpeedTester({ hostname })
+  })
+
+  console.log('[speed] dnsMap', dnsMap)
+}
+
+module.exports = {
+  SpeedTester,
+  initSpeedTestPool,
+  getSpeedTester (hostname) {
+    let instance = SpeedTestPool[hostname]
+    if (instance == null) {
+      instance = new SpeedTester({ hostname })
+      SpeedTestPool[hostname] = instance
+    }
+    return instance
+  }
+}

packages/mitmproxy/src/options.js

@@ -21,7 +21,8 @@ module.exports = (config) => {
     port: serverConfig.port,
     dnsConfig: {
       providers: dnsUtil.initDNS(serverConfig.dns.providers),
-      mapping: dnsMapping
+      mapping: dnsMapping,
+      speedTest: config.dns.speedTest
     },
     setting,
     middlewares,

packages/mitmproxy/test/dnsSpeed.js

@@ -0,0 +1,33 @@
+const SpeedTester = require('../src/lib/speed/SpeedTester.js')
+const SpeedTest = require('../src/lib/speed/index.js')
+const dns = require('../src/lib/dns/index.js')
+
+const dnsMap = dns.initDNS({
+  ipaddress: {
+    type: 'ipaddress',
+    server: 'ipaddress',
+    cacheSize: 1000
+  },
+  usa: {
+    type: 'https',
+    server: 'https://1.1.1.1/dns-query',
+    cacheSize: 1000
+  }
+  // google: {
+  //   type: 'https',
+  //   server: 'https://8.8.8.8/dns-query',
+  //   cacheSize: 1000
+  // },
+  // dnsSB: {
+  //   type: 'https',
+  //   server: 'https://doh.dns.sb/dns-query',
+  //   cacheSize: 1000
+  // }
+})
+
+SpeedTest.initSpeedTestPool({ hostnameList: {}, dnsMap })
+
+const tester = new SpeedTester({ hostname: 'github.com' })
+tester.test().then(ret => {
+  console.log(tester.alive)
+})

packages/mitmproxy/test/dnsTest.mjs

@@ -10,13 +10,34 @@ const dnsProviders = dns.initDNS({
     type: 'https',
     server: 'https://1.1.1.1/dns-query',
     cacheSize: 1000
+  },
+  ipaddress: {
+    type: 'ipaddress',
+    server: 'ipaddress',
+    cacheSize: 1000
+  },
+  quad9: {
+    type: 'https',
+    server: 'https://9.9.9.9/dns-query',
+    cacheSize: 1000
   }
 })
 
+// const test = '111<tr><th>IP Address</th><td><ul class="comma-separated"><li>140.82.113.4</li></ul></td></tr>2222'
+// // <tr><th>IP Address</th><td><ul class="comma-separated"><li>140.82.113.4</li></ul></td></tr>
+// // <tr><th>IP Address</th><td><ul class="comma-separated"><li>(.*)</li></ul></td></tr>
+// const regexp = /<tr><th>IP Address<\/th><td><ul class="comma-separated"><li>(.*)<\/li><\/ul><\/td><\/tr>/
+// const matched = regexp.exec(test)
+// console.log('data:', matched)
+
 const hostname0 = 'github.com'
-dnsProviders.usa.lookup(hostname0)
+// console.log('first')
-dnsProviders.usa.lookup(hostname0)
+// dnsProviders.usa.lookup(hostname0)
-dnsProviders.usa.lookup(hostname0)
+console.log('test')
+dnsProviders.quad9.lookup(hostname0)
+// dnsProviders.usa.lookup(hostname0)
+// dnsProviders.ipaddress.lookup(hostname0)
+// dnsProviders.ipaddress.lookup(hostname0)
 
 // const hostname = 'api.github.com'
 // dnsProviders.usa.lookup(hostname)

packages/mitmproxy/test/proxyTest.js

@@ -1,12 +1,17 @@
-const https = require('https')
+const http = require('http')
 
 var options = {
   headers: {
     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'
+  },
+  lookup (hostname, options, callback) {
+    const ip = '106.52.191.148'
+    console.log('lookup')
+    callback(null, ip, 4)
   }
 }
 
-var request = https.get('https://api.github.com/', options, function (response) {
+var request = http.get('http://test.target/', options, function (response) {
   response.on('data', function (data) {
     process.stdout.write(data)
   })

packages/mitmproxy/yarn.lock

@@ -1501,6 +1501,13 @@ aws4@^1.8.0:
   resolved "https://registry.npm.taobao.org/aws4/download/aws4-1.11.0.tgz?cache=0&sync_timestamp=1604101166484&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Faws4%2Fdownload%2Faws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59"
   integrity sha1-1h9G2DslGSUOJ4Ta9bCUeai0HFk=
 
+axios@^0.21.1:
+  version "0.21.1"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.1.tgz#22563481962f4d6bde9a76d516ef0e5d3c09b2b8"
+  integrity sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==
+  dependencies:
+    follow-redirects "^1.10.0"
+
 babel-eslint@^10.1.0:
   version "10.1.0"
   resolved "https://registry.npm.taobao.org/babel-eslint/download/babel-eslint-10.1.0.tgz?cache=0&sync_timestamp=1599054223324&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fbabel-eslint%2Fdownload%2Fbabel-eslint-10.1.0.tgz#6968e568a910b78fb3779cdd8b6ac2f479943232"
@@ -2984,6 +2991,11 @@ flush-write-stream@^1.0.0:
     inherits "^2.0.3"
     readable-stream "^2.3.6"
 
+follow-redirects@^1.10.0:
+  version "1.13.3"
+  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.13.3.tgz#e5598ad50174c1bc4e872301e82ac2cd97f90267"
+  integrity sha512-DUgl6+HDzB0iEptNQEXLx/KhTmDb8tZUHSeLqpnjpknR70H0nC2t9N73BK6fN4hOvJ84pKlIQVQ4k5FFlBedKA==
+
 for-in@^1.0.2:
   version "1.0.2"
   resolved "https://registry.npm.taobao.org/for-in/download/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80"