feat: speedTest

pull/67/head
xiaojunnuo 2021-03-24 13:59:19 +08:00
parent 6bc097fc68
commit 219323a0d6
20 changed files with 397 additions and 14 deletions

View File

@ -152,11 +152,31 @@ module.exports = {
server: 'https://dns.alidns.com/dns-query',
cacheSize: 1000
},
ipaddress: {
type: 'ipaddress',
server: 'ipaddress',
cacheSize: 1000
},
usa: {
type: 'https',
server: 'https://1.1.1.1/dns-query',
cacheSize: 1000
},
quad9: {
type: 'https',
server: 'https://9.9.9.9/dns-query',
cacheSize: 1000
}
// google: {
// type: 'https',
// server: 'https://8.8.8.8/dns-query',
// cacheSize: 1000
// },
// dnsSB: {
// type: 'https',
// server: 'https://doh.dns.sb/dns-query',
// cacheSize: 1000
// }
},
mapping: {
// 'assets.fastgit.org': 'usa',
@ -167,9 +187,14 @@ module.exports = {
'*.githubusercontent.com': 'usa',
'*.githubassets.com': 'usa',
// "解决push的时候需要输入密码的问题",
'github.com': 'usa',
'*github.com': 'usa',
'*.vuepress.vuejs.org': 'usa',
'gh.docmirror.top': 'usa'
},
speedTest: {
hostnameList: ['github.com'],
dnsProviders: ['usa', 'quad9']
}
}
},

View File

@ -1,6 +1,6 @@
module.exports = {
name: '梯子',
enabled: false,
enabled: false, // 默认关闭梯子
server: {
},
serverDefault: {
@ -10,7 +10,8 @@ module.exports = {
}
},
targets: {
'*facebook.com': true
'*facebook.com': true,
'github.com': true
},
pac: {
enabled: true,

View File

@ -53,7 +53,6 @@ export default {
},
async created () {
const platform = await this.$api.info.getSystemPlatform()
console.log('11', platform)
this.systemPlatform = platform
},
computed: {

View File

@ -69,6 +69,7 @@
<a-select :disabled="item.value ===false" v-model="item.value">
<a-select-option value="usa">USA DNS</a-select-option>
<a-select-option value="aliyun">Aliyun DNS</a-select-option>
<a-select-option value="ipaddress">IpAddress</a-select-option>
</a-select>
</a-col>
<a-col :span="3">

View File

@ -12,6 +12,7 @@
"scripts": {},
"dependencies": {
"agentkeepalive": "^2.1.1",
"axios": "^0.21.1",
"child_process": "^1.0.2",
"colors": "^1.1.2",
"commander": "^2.9.0",

View File

@ -1,10 +1,15 @@
const DNSOverTLS = require('./tls.js')
const DNSOverHTTPS = require('./https.js')
const DNSOverIpAddress = require('./ipaddress.js')
module.exports = {
initDNS (dnsProviders) {
const dnsMap = {}
for (const provider in dnsProviders) {
const conf = dnsProviders[provider]
if (conf.type === 'ipaddress') {
dnsMap[provider] = new DNSOverIpAddress(conf.server)
continue
}
dnsMap[provider] = conf.type === 'https' ? new DNSOverHTTPS(conf.server) : new DNSOverTLS(conf.server)
}
return dnsMap

View File

@ -0,0 +1,40 @@
const dnstls = require('dns-over-tls')
const BaseDNS = require('./base')
const axios = require('axios')
const log = require('../../utils/util.log')
const fs = require('fs')
const path = require('path')
module.exports = class DNSOverIpAddress extends BaseDNS {
async _lookup (hostname) {
const url = `https://${hostname}.ipaddress.com`
// const res = fs.readFileSync(path.resolve(__dirname, './data.txt')).toString()
const res = await axios.get(url)
if (res.status !== 200 && res.status !== 201) {
log.info(`[dns] get ${hostname} ipaddress: error:${res}`)
return
}
const ret = res.data
const regexp = /<tr><th>IP Address<\/th><td><ul class="comma-separated"><li>([^<]*)<\/li><\/ul><\/td><\/tr>/gm
const matched = regexp.exec(ret)
let ip = null
if (matched && matched.length >= 1) {
ip = matched[1]
log.info(`[dns] get ${hostname} ipaddress:${ip}`)
return [ip]
}
log.info(`[dns] get ${hostname} ipaddress: error`)
return null
// const { answers } = await dnstls.query(hostname)
//
// const answer = answers.find(answer => answer.type === 'A' && answer.class === 'IN')
//
// log.info('dns lookup', hostname, answer)
// if (answer) {
// return answer.data
// }
}
}

View File

@ -33,7 +33,8 @@ module.exports = {
const regexp = new RegExp(interceptOpt.replace)
proxyTarget = req.url.replace(regexp, proxyConf)
}
// eslint-disable-next-line no-template-curly-in-string
// eslint-disable-next-line
// no-template-curly-in-string
proxyTarget = proxyTarget.replace('${host}', rOptions.hostname)
// const backup = interceptOpt.backup

View File

@ -48,7 +48,6 @@ function injectScriptIntoHtml (tags, chunk, script) {
if (index < 0) {
continue
}
console.log('insert script:', tag)
const scriptBuf = Buffer.from(script)
const chunkNew = Buffer.alloc(chunk.length + scriptBuf.length)
chunk.copy(chunkNew, 0, 0, index)

View File

@ -6,6 +6,8 @@ const DnsUtil = require('../../dns/index')
const localIP = '127.0.0.1'
const defaultDns = require('dns')
const speedTest = require('../../speed/index.js')
function isSslConnect (sslConnectInterceptors, req, cltSocket, head) {
for (const intercept of sslConnectInterceptors) {
const ret = intercept(req, cltSocket, head)
@ -32,6 +34,7 @@ module.exports = function createConnectHandler (sslConnectInterceptor, middlewar
const hostname = srvUrl.hostname
if (isSslConnect(sslConnectInterceptors, req, cltSocket, head)) {
fakeServerCenter.getServerPromise(hostname, srvUrl.port).then((serverObj) => {
log.info('--- fakeServer connect', hostname)
connect(req, cltSocket, head, localIP, serverObj.port)
}, (e) => {
log.error('getServerPromise', e)
@ -57,9 +60,19 @@ function connect (req, cltSocket, head, hostname, port, dnsConfig) {
const dns = DnsUtil.hasDnsLookup(dnsConfig, hostname)
if (dns) {
options.lookup = (hostname, options, callback) => {
const tester = speedTest.getSpeedTester(hostname)
if (tester) {
const ip = tester.pickFastAliveIp()
if (ip) {
log.info(`-----${hostname} use alive ip:${ip}-----`)
callback(null, ip, 4)
return
}
}
dns.lookup(hostname).then(ip => {
isDnsIntercept = { dns, hostname, ip }
if (ip !== hostname) {
log.info(`-----${hostname} use ip:${ip}-----`)
callback(null, ip, 4)
} else {
defaultDns.lookup(hostname, options, callback)

View File

@ -7,7 +7,7 @@ const log = require('../../../utils/util.log')
const RequestCounter = require('../../choice/RequestCounter')
const InsertScriptMiddleware = require('../middleware/InsertScriptMiddleware')
const OverWallMiddleware = require('../middleware/overwall')
const speedTest = require('../../speed/index.js')
const defaultDns = require('dns')
const MAX_SLOW_TIME = 8000 // 超过此时间 则认为太慢了
// create requestHandler function
@ -106,10 +106,19 @@ module.exports = function createRequestHandler (createIntercepts, middlewares, e
const dns = DnsUtil.hasDnsLookup(dnsConfig, rOptions.hostname)
if (dns) {
rOptions.lookup = (hostname, options, callback) => {
const tester = speedTest.getSpeedTester(hostname)
if (tester) {
const ip = tester.pickFastAliveIp()
if (ip) {
log.info(`-----${hostname} use alive ip:${ip}-----`)
callback(null, ip, 4)
return
}
}
dns.lookup(hostname).then(ip => {
isDnsIntercept = { dns, hostname, ip }
if (ip !== hostname) {
log.info(`request url :${url},use ip :${ip}`)
log.info(`----request url :${url},use ip :${ip}----`)
callback(null, ip, 4)
} else {
log.info(`request url :${url},use hostname :${hostname}`)
@ -266,7 +275,7 @@ module.exports = function createRequestHandler (createIntercepts, middlewares, e
})().catch(e => {
if (!res.writableEnded) {
const status = e.status || 500
res.writeHead(status)
res.writeHead(status, { 'Content-Type': 'text/html;charset=UTF8' })
res.write(`DevSidecar Warning:\n\n ${e.toString()}`)
res.end()
log.error('request error', e.message)

View File

@ -1,11 +1,15 @@
const tlsUtils = require('../tls/tlsUtils')
const http = require('http')
const https = require('https')
const config = require('../common/config')
const log = require('../../../utils/util.log')
const createRequestHandler = require('./createRequestHandler')
const createConnectHandler = require('./createConnectHandler')
const createFakeServerCenter = require('./createFakeServerCenter')
const createUpgradeHandler = require('./createUpgradeHandler')
const DnsUtil = require('../../dns/index')
const defaultDns = require('dns')
const speedTest = require('../../speed/index.js')
module.exports = {
createProxy ({
port = config.defaultPort,
@ -35,7 +39,40 @@ module.exports = {
log.info(`CA private key saved in: ${caKeyPath}`)
}
// function lookup (hostname, options, callback) {
// const dns = DnsUtil.hasDnsLookup(dnsConfig, hostname)
// if (dns) {
// dns.lookup(hostname).then(ip => {
// // isDnsIntercept = { dns, hostname, ip }
// if (ip !== hostname) {
// log.info(`-----${hostname} use ip:${ip}-----`)
// callback(null, ip, 4)
// } else {
// defaultDns.lookup(hostname, options, callback)
// }
// })
// } else {
// defaultDns.lookup(hostname, options, callback)
// }
// }
//
// https.globalAgent.lookup = lookup
port = ~~port
const speedTestConfig = dnsConfig.speedTest
const dnsMap = dnsConfig.providers
if (speedTestConfig) {
const dnsProviders = speedTestConfig.dnsProviders
const map = {}
for (const dnsProvider of dnsProviders) {
if (dnsMap[dnsProvider]) {
map[dnsProvider] = dnsMap[dnsProvider]
}
}
speedTest.initSpeedTestPool({ hostnameList: speedTestConfig.hostnameList, dnsMap: map })
}
const requestHandler = createRequestHandler(
createIntercepts,
middlewares,

View File

@ -0,0 +1,144 @@
const net = require('net')
const config = require('./config.js')
const log = require('../../utils/util.log.js')
const DISABLE_TIMEOUT = 60 * 60 * 1000 // 1个小时不访问取消获取
class SpeedTester {
constructor ({ hostname }) {
this.dnsMap = config.getConfig().dnsMap
this.hostname = hostname
this.lastReadTime = Date.now()
this.ready = false
this.alive = []
this.backupList = []
this.keepCheckId = false
this.loadingIps = false
this.loadingTest = false
this.test()
}
pickFastAliveIp () {
this.touch()
if (this.alive.length === 0) {
this.test()
return null
}
return this.alive[0].host
}
touch () {
this.lastReadTime = Date.now()
if (!this.keepCheckId) {
this.startChecker()
}
}
startChecker () {
if (this.keepCheckId) {
clearInterval(this.keepCheckId)
}
this.keepCheckId = setInterval(() => {
if (Date.now() - DISABLE_TIMEOUT > this.lastReadTime) {
// 超过很长时间没有访问,取消测试
clearInterval(this.keepCheckId)
return
}
if (this.alive.length > 0) {
this.testBackups()
return
}
this.test()
}, 60 * 1000)
}
async getIpListFromDns (dnsMap) {
const ips = {}
const promiseList = []
for (const key in dnsMap) {
const one = this.getFromOneDns(dnsMap[key]).then(ipList => {
if (ipList) {
for (const ip of ipList) {
ips[ip] = 1
}
}
})
promiseList.push(one)
}
await Promise.all(promiseList)
const items = []
for (const ip in ips) {
items.push({ host: ip, port: 443 })
}
return items
}
async getFromOneDns (dns) {
return await dns._lookup(this.hostname)
}
async test () {
this.backupList = await this.getIpListFromDns(this.dnsMap)
log.info('[speed]', this.hostname, ' ips:', this.backupList)
this.testBackups()
}
async testBackups () {
const testAll = []
const aliveList = []
for (const item of this.backupList) {
testAll.push(this.doTest(item, aliveList))
}
await Promise.all(testAll)
this.alive = aliveList
this.ready = true
}
async doTest (item, aliveList) {
try {
const ret = await this.testOne(item)
aliveList.push({ ...ret, ...item })
aliveList.sort((a, b) => a.time - b.time)
} catch (e) {
log.error('[speed] test error', this.hostname, item.host, e.message)
}
}
testOne (item) {
const timeout = 5000
const { host, port } = item
const startTime = Date.now()
let isOver = false
return new Promise((resolve, reject) => {
let timeoutId = null
const client = net.createConnection({ host, port }, () => {
// 'connect' 监听器
const connectionTime = Date.now()
isOver = true
clearTimeout(timeoutId)
resolve({ status: 'success', time: connectionTime - startTime })
client.end()
})
client.on('end', () => {
})
client.on('error', (error) => {
log.error('[speed]test error', this.hostname, host, error.message)
isOver = true
clearTimeout(timeoutId)
reject(error)
})
timeoutId = setTimeout(() => {
if (isOver) {
return
}
log.error('[speed] test timeout', this.hostname, host)
reject(new Error('timeout'))
client.end()
}, timeout)
})
}
}
module.exports = SpeedTester

View File

@ -0,0 +1,8 @@
const config = {
}
module.exports = {
getConfig () {
return config
}
}

View File

@ -0,0 +1,28 @@
const SpeedTester = require('./SpeedTester.js')
const _ = require('lodash')
const config = require('./config')
const SpeedTestPool = {
}
function initSpeedTestPool ({ hostnameList, dnsMap }) {
config.getConfig().dnsMap = dnsMap
_.forEach(hostnameList, (hostname) => {
SpeedTestPool[hostname] = new SpeedTester({ hostname })
})
console.log('[speed] dnsMap', dnsMap)
}
module.exports = {
SpeedTester,
initSpeedTestPool,
getSpeedTester (hostname) {
let instance = SpeedTestPool[hostname]
if (instance == null) {
instance = new SpeedTester({ hostname })
SpeedTestPool[hostname] = instance
}
return instance
}
}

View File

@ -21,7 +21,8 @@ module.exports = (config) => {
port: serverConfig.port,
dnsConfig: {
providers: dnsUtil.initDNS(serverConfig.dns.providers),
mapping: dnsMapping
mapping: dnsMapping,
speedTest: config.dns.speedTest
},
setting,
middlewares,

View File

@ -0,0 +1,33 @@
const SpeedTester = require('../src/lib/speed/SpeedTester.js')
const SpeedTest = require('../src/lib/speed/index.js')
const dns = require('../src/lib/dns/index.js')
const dnsMap = dns.initDNS({
ipaddress: {
type: 'ipaddress',
server: 'ipaddress',
cacheSize: 1000
},
usa: {
type: 'https',
server: 'https://1.1.1.1/dns-query',
cacheSize: 1000
}
// google: {
// type: 'https',
// server: 'https://8.8.8.8/dns-query',
// cacheSize: 1000
// },
// dnsSB: {
// type: 'https',
// server: 'https://doh.dns.sb/dns-query',
// cacheSize: 1000
// }
})
SpeedTest.initSpeedTestPool({ hostnameList: {}, dnsMap })
const tester = new SpeedTester({ hostname: 'github.com' })
tester.test().then(ret => {
console.log(tester.alive)
})

View File

@ -10,13 +10,34 @@ const dnsProviders = dns.initDNS({
type: 'https',
server: 'https://1.1.1.1/dns-query',
cacheSize: 1000
},
ipaddress: {
type: 'ipaddress',
server: 'ipaddress',
cacheSize: 1000
},
quad9: {
type: 'https',
server: 'https://9.9.9.9/dns-query',
cacheSize: 1000
}
})
// const test = '111<tr><th>IP Address</th><td><ul class="comma-separated"><li>140.82.113.4</li></ul></td></tr>2222'
// // <tr><th>IP Address</th><td><ul class="comma-separated"><li>140.82.113.4</li></ul></td></tr>
// // <tr><th>IP Address</th><td><ul class="comma-separated"><li>(.*)</li></ul></td></tr>
// const regexp = /<tr><th>IP Address<\/th><td><ul class="comma-separated"><li>(.*)<\/li><\/ul><\/td><\/tr>/
// const matched = regexp.exec(test)
// console.log('data:', matched)
const hostname0 = 'github.com'
dnsProviders.usa.lookup(hostname0)
dnsProviders.usa.lookup(hostname0)
dnsProviders.usa.lookup(hostname0)
// console.log('first')
// dnsProviders.usa.lookup(hostname0)
console.log('test')
dnsProviders.quad9.lookup(hostname0)
// dnsProviders.usa.lookup(hostname0)
// dnsProviders.ipaddress.lookup(hostname0)
// dnsProviders.ipaddress.lookup(hostname0)
// const hostname = 'api.github.com'
// dnsProviders.usa.lookup(hostname)

View File

@ -1,12 +1,17 @@
const https = require('https')
const http = require('http')
var options = {
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'
},
lookup (hostname, options, callback) {
const ip = '106.52.191.148'
console.log('lookup')
callback(null, ip, 4)
}
}
var request = https.get('https://api.github.com/', options, function (response) {
var request = http.get('http://test.target/', options, function (response) {
response.on('data', function (data) {
process.stdout.write(data)
})

View File

@ -1501,6 +1501,13 @@ aws4@^1.8.0:
resolved "https://registry.npm.taobao.org/aws4/download/aws4-1.11.0.tgz?cache=0&sync_timestamp=1604101166484&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Faws4%2Fdownload%2Faws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59"
integrity sha1-1h9G2DslGSUOJ4Ta9bCUeai0HFk=
axios@^0.21.1:
version "0.21.1"
resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.1.tgz#22563481962f4d6bde9a76d516ef0e5d3c09b2b8"
integrity sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==
dependencies:
follow-redirects "^1.10.0"
babel-eslint@^10.1.0:
version "10.1.0"
resolved "https://registry.npm.taobao.org/babel-eslint/download/babel-eslint-10.1.0.tgz?cache=0&sync_timestamp=1599054223324&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fbabel-eslint%2Fdownload%2Fbabel-eslint-10.1.0.tgz#6968e568a910b78fb3779cdd8b6ac2f479943232"
@ -2984,6 +2991,11 @@ flush-write-stream@^1.0.0:
inherits "^2.0.3"
readable-stream "^2.3.6"
follow-redirects@^1.10.0:
version "1.13.3"
resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.13.3.tgz#e5598ad50174c1bc4e872301e82ac2cd97f90267"
integrity sha512-DUgl6+HDzB0iEptNQEXLx/KhTmDb8tZUHSeLqpnjpknR70H0nC2t9N73BK6fN4hOvJ84pKlIQVQ4k5FFlBedKA==
for-in@^1.0.2:
version "1.0.2"
resolved "https://registry.npm.taobao.org/for-in/download/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80"