From 1e68ae5a8a6a83d7fbb9f46e3d5e2e98e6c552ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E8=89=AF?= <841369634@qq.com> Date: Wed, 5 Mar 2025 21:28:27 +0800 Subject: [PATCH] =?UTF-8?q?optimize:=20=E4=BC=98=E5=8C=96DNS=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E8=A7=A3=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/mitmproxy/src/lib/dns/base.js | 16 +++-- packages/mitmproxy/src/lib/dns/https.js | 2 +- packages/mitmproxy/src/lib/dns/index.js | 64 +++++++++++-------- packages/mitmproxy/src/lib/dns/lookup.js | 4 -- packages/mitmproxy/src/lib/dns/preset.js | 4 +- packages/mitmproxy/src/lib/dns/tcp.js | 2 +- packages/mitmproxy/src/lib/dns/tls.js | 2 +- packages/mitmproxy/src/lib/dns/udp.js | 2 +- .../proxy/mitmproxy/createConnectHandler.js | 4 +- .../proxy/mitmproxy/createRequestHandler.js | 6 +- .../src/lib/proxy/mitmproxy/dnsLookup.js | 6 +- packages/mitmproxy/test/dnsTest.mjs | 57 ++++++++++------- 12 files changed, 93 insertions(+), 76 deletions(-) delete mode 100644 packages/mitmproxy/src/lib/dns/lookup.js diff --git a/packages/mitmproxy/src/lib/dns/base.js b/packages/mitmproxy/src/lib/dns/base.js index a8718df..fb44844 100644 --- a/packages/mitmproxy/src/lib/dns/base.js +++ b/packages/mitmproxy/src/lib/dns/base.js @@ -33,8 +33,9 @@ class IpCache extends DynamicChoice { } module.exports = class BaseDNS { - constructor (dnsName, cacheSize, preSetIpList) { + constructor (dnsName, dnsType, cacheSize, preSetIpList) { this.dnsName = dnsName + this.dnsType = dnsType this.preSetIpList = preSetIpList this.cache = new LRUCache({ maxSize: (cacheSize > 0 ? cacheSize : defaultCacheSize), @@ -73,11 +74,11 @@ module.exports = class BaseDNS { ipList.push(hostname) // 把原域名加入到统计里去 ipCache.setBackupList(ipList) - log.info(`[DNS '${this.dnsName}']: ${hostname} ➜ ${ipCache.value} (${new Date() - t} ms), ipList: ${JSON.stringify(ipList)}, ipCache:`, JSON.stringify(ipCache)) + log.info(`[DNS-over-${this.dnsType} '${this.dnsName}'] ${hostname} ➜ ${ipCache.value} (${new Date() - t} ms), ipList: ${JSON.stringify(ipList)}, ipCache:`, JSON.stringify(ipCache)) return ipCache.value } catch (error) { - log.error(`[DNS '${this.dnsName}'] cannot resolve hostname ${hostname}, error:`, error) + log.error(`[DNS-over-${this.dnsType} '${this.dnsName}'] cannot resolve hostname ${hostname}, error:`, error) return hostname } } @@ -94,6 +95,7 @@ module.exports = class BaseDNS { if (hostnamePreSetIpList.length > 0) { hostnamePreSetIpList.isPreSet = true + log.info(`[DNS-over-${this.dnsType} '${this.dnsName}'] 获取到该域名的预设IP列表: ${hostname} - ${JSON.stringify(hostnamePreSetIpList)}`) return hostnamePreSetIpList } } @@ -108,18 +110,18 @@ module.exports = class BaseDNS { const cost = Date.now() - start if (response == null || response.answers == null || response.answers.length == null || response.answers.length === 0) { // 说明没有获取到ip - log.warn(`DNS '${this.dnsName}' 没有该域名的IP地址: ${hostname}, cost: ${cost} ms, response:`, response) + log.warn(`[DNS-over-${this.dnsType} '${this.dnsName}'] 没有该域名的IP地址: ${hostname}, cost: ${cost} ms, response:`, response) return [] } const ret = response.answers.filter(item => item.type === 'A').map(item => item.data) if (ret.length === 0) { - log.info(`DNS '${this.dnsName}' 没有该域名的IPv4地址: ${hostname}, cost: ${cost} ms`) + log.info(`[DNS-over-${this.dnsType} '${this.dnsName}'] 没有该域名的IPv4地址: ${hostname}, cost: ${cost} ms`) } else { - log.info(`DNS '${this.dnsName}' 获取到该域名的IPv4地址: ${hostname} - ${JSON.stringify(ret)}, cost: ${cost} ms`) + log.info(`[DNS-over-${this.dnsType} '${this.dnsName}'] 获取到该域名的IPv4地址: ${hostname} - ${JSON.stringify(ret)}, cost: ${cost} ms`) } return ret } catch (e) { - log.error(`DNS query error: ${hostname}, dns: ${this.dnsName}${this.dnsServer ? (`, dnsServer: ${this.dnsServer}`) : ''}, cost: ${Date.now() - start} ms, error:`, e) + log.error(`[DNS-over-${this.dnsType} '${this.dnsName}'] DNS query error, hostname: ${hostname}${this.dnsServer ? `, dnsServer: ${this.dnsServer}` : ''}, cost: ${Date.now() - start} ms, error:`, e) return [] } } diff --git a/packages/mitmproxy/src/lib/dns/https.js b/packages/mitmproxy/src/lib/dns/https.js index 95c6606..3ab889c 100644 --- a/packages/mitmproxy/src/lib/dns/https.js +++ b/packages/mitmproxy/src/lib/dns/https.js @@ -6,7 +6,7 @@ const dohQueryAsync = promisify(doh.query) module.exports = class DNSOverHTTPS extends BaseDNS { constructor (dnsName, cacheSize, preSetIpList, dnsServer) { - super(dnsName, cacheSize, preSetIpList) + super(dnsName, 'HTTPS', cacheSize, preSetIpList) this.dnsServer = dnsServer } diff --git a/packages/mitmproxy/src/lib/dns/index.js b/packages/mitmproxy/src/lib/dns/index.js index 5244f26..ee3e40c 100644 --- a/packages/mitmproxy/src/lib/dns/index.js +++ b/packages/mitmproxy/src/lib/dns/index.js @@ -13,6 +13,7 @@ module.exports = { for (const provider in dnsProviders) { const conf = dnsProviders[provider] + // 获取DNS服务器 let server = conf.server || conf.host if (server != null) { server = server.replace(/\s+/, '') @@ -22,44 +23,55 @@ module.exports = { } // 获取DNS类型 - if (conf.type == null) { + let type = conf.type + if (type == null) { if (server.startsWith('https://') || server.startsWith('http://')) { - conf.type = 'https' + type = 'https' } else if (server.startsWith('tls://')) { - conf.type = 'tls' + type = 'tls' } else if (server.startsWith('tcp://')) { - conf.type = 'tcp' + type = 'tcp' } else if (server.includes('://') && !server.startsWith('udp://')) { throw new Error(`Unknown type DNS: ${server}, provider: ${provider}`) } else { - conf.type = 'udp' + type = 'udp' } } else { - conf.type = conf.type.toLowerCase() + type = type.replace(/\s+/, '').toLowerCase() } - if (conf.type === 'https') { + // 创建DNS对象 + if (type === 'https' || type === 'doh' || type === 'dns-over-https') { + if (!server.includes('/')) { + server = `https://${server}/dns-query` + } + + // 基于 https dnsMap[provider] = new DNSOverHTTPS(provider, conf.cacheSize, preSetIpList, server) - } else if (conf.type === 'tls') { - if (server.startsWith('tls://')) { - server = server.substring(6) - } - dnsMap[provider] = new DNSOverTLS(provider, conf.cacheSize, preSetIpList, server, conf.port, conf.servername) - } else if (conf.type === 'tcp') { - if (server.startsWith('tcp://')) { - server = server.substring(6) - } - dnsMap[provider] = new DNSOverTCP(provider, conf.cacheSize, preSetIpList, server, conf.port) - } else { // udp - if (server.startsWith('udp://')) { - server = server.substring(6) - } - dnsMap[provider] = new DNSOverUDP(provider, conf.cacheSize, preSetIpList, server, conf.port) - } + } else { + // 获取DNS端口 + let port = conf.port - // 设置DNS名称到name属性中 - dnsMap[provider].name = provider - dnsMap[provider].type = conf.type + // 处理带协议的DNS服务地址 + if (server.includes('://')) { + server = server.split('://')[1] + } + // 处理带端口的DNS服务地址 + if (port == null && server.includes(':')) { + [server, port] = server.split(':') + } + + if (type === 'tls' || type === 'dot' || type === 'dns-over-tls') { + // 基于 tls + dnsMap[provider] = new DNSOverTLS(provider, conf.cacheSize, preSetIpList, server, port, conf.servername) + } else if (type === 'tcp' || type === 'dns-over-tcp') { + // 基于 tcp + dnsMap[provider] = new DNSOverTCP(provider, conf.cacheSize, preSetIpList, server, port) + } else { + // 基于 udp + dnsMap[provider] = new DNSOverUDP(provider, conf.cacheSize, preSetIpList, server, port) + } + } } // 创建预设IP的DNS diff --git a/packages/mitmproxy/src/lib/dns/lookup.js b/packages/mitmproxy/src/lib/dns/lookup.js deleted file mode 100644 index 108d106..0000000 --- a/packages/mitmproxy/src/lib/dns/lookup.js +++ /dev/null @@ -1,4 +0,0 @@ -module.exports = { - lookup () { - }, -} diff --git a/packages/mitmproxy/src/lib/dns/preset.js b/packages/mitmproxy/src/lib/dns/preset.js index 4d39ec0..b992990 100644 --- a/packages/mitmproxy/src/lib/dns/preset.js +++ b/packages/mitmproxy/src/lib/dns/preset.js @@ -2,9 +2,7 @@ const BaseDNS = require('./base') module.exports = class DNSOverPreSetIpList extends BaseDNS { constructor (preSetIpList) { - super('PreSet', null, preSetIpList) - this.name = 'PreSet' - this.type = 'PreSet' + super('PreSet', 'PreSet', null, preSetIpList) } async _lookup (_hostname) { diff --git a/packages/mitmproxy/src/lib/dns/tcp.js b/packages/mitmproxy/src/lib/dns/tcp.js index 98357ee..779e75e 100644 --- a/packages/mitmproxy/src/lib/dns/tcp.js +++ b/packages/mitmproxy/src/lib/dns/tcp.js @@ -8,7 +8,7 @@ const defaultPort = 53 // UDP类型的DNS服务默认端口号 module.exports = class DNSOverTCP extends BaseDNS { constructor (dnsName, cacheSize, preSetIpList, dnsServer, dnsServerPort) { - super(dnsName, cacheSize, preSetIpList) + super(dnsName, 'TCP', cacheSize, preSetIpList) this.dnsServer = dnsServer this.dnsServerPort = Number.parseInt(dnsServerPort) || defaultPort } diff --git a/packages/mitmproxy/src/lib/dns/tls.js b/packages/mitmproxy/src/lib/dns/tls.js index ba0a55f..b4b5d99 100644 --- a/packages/mitmproxy/src/lib/dns/tls.js +++ b/packages/mitmproxy/src/lib/dns/tls.js @@ -5,7 +5,7 @@ const defaultPort = 853 module.exports = class DNSOverTLS extends BaseDNS { constructor (dnsName, cacheSize, preSetIpList, dnsServer, dnsServerPort, dnsServerName) { - super(dnsName, cacheSize, preSetIpList) + super(dnsName, 'TLS', cacheSize, preSetIpList) this.dnsServer = dnsServer this.dnsServerPort = Number.parseInt(dnsServerPort) || defaultPort this.dnsServerName = dnsServerName diff --git a/packages/mitmproxy/src/lib/dns/udp.js b/packages/mitmproxy/src/lib/dns/udp.js index d751ac5..10464c3 100644 --- a/packages/mitmproxy/src/lib/dns/udp.js +++ b/packages/mitmproxy/src/lib/dns/udp.js @@ -9,7 +9,7 @@ const defaultPort = 53 // UDP类型的DNS服务默认端口号 module.exports = class DNSOverUDP extends BaseDNS { constructor (dnsName, cacheSize, preSetIpList, dnsServer, dnsServerPort) { - super(dnsName, cacheSize, preSetIpList) + super(dnsName, 'UDP', cacheSize, preSetIpList) this.dnsServer = dnsServer this.dnsServerPort = Number.parseInt(dnsServerPort) || defaultPort } diff --git a/packages/mitmproxy/src/lib/proxy/mitmproxy/createConnectHandler.js b/packages/mitmproxy/src/lib/proxy/mitmproxy/createConnectHandler.js index 49fa2fe..0ac8f4b 100644 --- a/packages/mitmproxy/src/lib/proxy/mitmproxy/createConnectHandler.js +++ b/packages/mitmproxy/src/lib/proxy/mitmproxy/createConnectHandler.js @@ -143,7 +143,7 @@ function connect (req, cltSocket, head, hostname, port, dnsConfig = null, isDire if (isDnsIntercept && isDnsIntercept.dns && isDnsIntercept.ip !== isDnsIntercept.hostname) { const { dns, ip, hostname } = isDnsIntercept dns.count(hostname, ip, true) - log.error(`记录ip失败次数,用于优选ip! hostname: ${hostname}, ip: ${ip}, reason: ${errorMsg}, dns: ${dns.name}`) + log.error(`记录ip失败次数,用于优选ip! hostname: ${hostname}, ip: ${ip}, reason: ${errorMsg}, dns: ${dns.dnsName}`) } }) proxySocket.on('error', (e) => { @@ -157,7 +157,7 @@ function connect (req, cltSocket, head, hostname, port, dnsConfig = null, isDire if (isDnsIntercept && isDnsIntercept.dns && isDnsIntercept.ip !== isDnsIntercept.hostname) { const { dns, ip, hostname } = isDnsIntercept dns.count(hostname, ip, true) - log.error(`记录ip失败次数,用于优选ip! hostname: ${hostname}, ip: ${ip}, reason: ${errorMsg}, dns: ${dns.name}`) + log.error(`记录ip失败次数,用于优选ip! hostname: ${hostname}, ip: ${ip}, reason: ${errorMsg}, dns: ${dns.dnsName}`) } }) diff --git a/packages/mitmproxy/src/lib/proxy/mitmproxy/createRequestHandler.js b/packages/mitmproxy/src/lib/proxy/mitmproxy/createRequestHandler.js index 516591a..b3c27f9 100644 --- a/packages/mitmproxy/src/lib/proxy/mitmproxy/createRequestHandler.js +++ b/packages/mitmproxy/src/lib/proxy/mitmproxy/createRequestHandler.js @@ -84,7 +84,7 @@ module.exports = function createRequestHandler (createIntercepts, middlewares, e if (isDnsIntercept && isDnsIntercept.dns && isDnsIntercept.ip !== isDnsIntercept.hostname) { const { dns, ip, hostname } = isDnsIntercept dns.count(hostname, ip, true) - log.error(`记录ip失败次数,用于优选ip! hostname: ${hostname}, ip: ${ip}, reason: ${reason}, dns: ${dns.name}`) + log.error(`记录ip失败次数,用于优选ip! hostname: ${hostname}, ip: ${ip}, reason: ${reason}, dns: ${dns.dnsName}`) } const counter = context.requestCount if (counter != null) { @@ -123,8 +123,8 @@ module.exports = function createRequestHandler (createIntercepts, middlewares, e } if (dns) { rOptions.lookup = dnsLookup.createLookupFunc(res, dns, 'request url', url, isDnsIntercept) - log.debug(`域名 ${rOptions.hostname} DNS: ${dns.name}`) - res.setHeader('DS-DNS', dns.name) + log.debug(`域名 ${rOptions.hostname} DNS: ${dns.dnsName}`) + res.setHeader('DS-DNS', dns.dnsName) } else { log.info(`域名 ${rOptions.hostname} 在DNS中未配置`) } diff --git a/packages/mitmproxy/src/lib/proxy/mitmproxy/dnsLookup.js b/packages/mitmproxy/src/lib/proxy/mitmproxy/dnsLookup.js index 466d543..6668ac0 100644 --- a/packages/mitmproxy/src/lib/proxy/mitmproxy/dnsLookup.js +++ b/packages/mitmproxy/src/lib/proxy/mitmproxy/dnsLookup.js @@ -43,15 +43,15 @@ module.exports = { } } if (isTestFailedIp === false) { - log.info(`----- ${action}: ${hostname}, use ip from dns '${dns.name}': ${ip}${target} -----`) + log.info(`----- ${action}: ${hostname}, use ip from dns '${dns.dnsName}': ${ip}${target} -----`) if (res) { - res.setHeader('DS-DNS-Lookup', `DNS: ${ip} ${dns.name === '预设IP' ? 'PreSet' : dns.name}`) + res.setHeader('DS-DNS-Lookup', `DNS: ${ip} ${dns.dnsName === '预设IP' ? 'PreSet' : dns.dnsName}`) } callback(null, ip, 4) return } else { // 使用默认dns - log.info(`----- ${action}: ${hostname}, use hostname by default DNS: ${hostname}, skip test failed ip from dns '${dns.name}: ${ip}'${target}, options:`, options) + log.info(`----- ${action}: ${hostname}, use hostname by default DNS: ${hostname}, skip test failed ip from dns '${dns.dnsName}: ${ip}'${target}, options:`, options) } } else { // 使用默认dns diff --git a/packages/mitmproxy/test/dnsTest.mjs b/packages/mitmproxy/test/dnsTest.mjs index dc0232f..cb50beb 100644 --- a/packages/mitmproxy/test/dnsTest.mjs +++ b/packages/mitmproxy/test/dnsTest.mjs @@ -1,7 +1,13 @@ import assert from 'node:assert' import dns from '../src/lib/dns/index.js' +import matchUtil from '../src/utils/util.match.js' const presetIp = '100.100.100.100' +const preSetIpList = matchUtil.domainMapRegexply({ + 'xxx.com': [ + presetIp + ] +}) const dnsProviders = dns.initDNS({ // https @@ -19,6 +25,11 @@ const dnsProviders = dns.initDNS({ server: 'https://dns.alidns.com/dns-query', cacheSize: 1000, }, + aliyun2: { + type: 'https', + server: 'dns.alidns.com', // 会自动补上 `https://` 和 `/dns-query` + cacheSize: 1000, + }, safe360: { server: 'https://doh.360.cn/dns-query', cacheSize: 1000, @@ -45,8 +56,7 @@ const dnsProviders = dns.initDNS({ cacheSize: 1000, }, aliyunTLS: { - type: 'tls', - server: '223.5.5.5', + server: 'tls://223.5.5.5:853', cacheSize: 1000, }, aliyunTLS2: { @@ -62,6 +72,7 @@ const dnsProviders = dns.initDNS({ googleTCP: { type: 'tcp', server: '8.8.8.8', + port: 53, cacheSize: 1000, }, aliyunTCP: { @@ -79,13 +90,7 @@ const dnsProviders = dns.initDNS({ server: 'udp://223.5.5.5', cacheSize: 1000, }, -}, { - origin: { - 'xxx.com': [ - presetIp - ] - } -}) +}, preSetIpList) const presetHostname = 'xxx.com' @@ -111,27 +116,31 @@ ip = await dnsProviders.cloudflare.lookup(presetHostname) assert.strictEqual(ip, presetIp) // test preset console.log('\n\n') -assert.strictEqual(dnsProviders.cloudflare.type, 'https') +assert.strictEqual(dnsProviders.cloudflare.dnsType, 'HTTPS') // ip = await dnsProviders.cloudflare.lookup(hostname1) // console.log('===> test cloudflare:', ip, '\n\n') -assert.strictEqual(dnsProviders.quad9.type, 'https') +assert.strictEqual(dnsProviders.quad9.dnsType, 'HTTPS') // ip = await dnsProviders.quad9.lookup(hostname1) // console.log('===> test quad9:', ip, '\n\n') -assert.strictEqual(dnsProviders.aliyun.type, 'https') +assert.strictEqual(dnsProviders.aliyun.dnsType, 'HTTPS') // ip = await dnsProviders.aliyun.lookup(hostname1) // console.log('===> test aliyun:', ip, '\n\n') -assert.strictEqual(dnsProviders.safe360.type, 'https') +assert.strictEqual(dnsProviders.aliyun2.dnsType, 'HTTPS') +// ip = await dnsProviders.aliyun2.lookup(hostname1) +// console.log('===> test aliyun2:', ip, '\n\n') + +assert.strictEqual(dnsProviders.safe360.dnsType, 'HTTPS') // ip = await dnsProviders.safe360.lookup(hostname1) // console.log('===> test safe360:', ip, '\n\n') -assert.strictEqual(dnsProviders.rubyfish.type, 'https') +assert.strictEqual(dnsProviders.rubyfish.dnsType, 'HTTPS') // ip = await dnsProviders.rubyfish.lookup(hostname1) // console.log('===> test rubyfish:', ip, '\n\n') -assert.strictEqual(dnsProviders.py233.type, 'https') +assert.strictEqual(dnsProviders.py233.dnsType, 'HTTPS') // ip = await dnsProviders.py233.lookup(hostname1) // console.log('===> test py233:', ip, '\n\n') @@ -141,23 +150,23 @@ ip = await dnsProviders.cloudflareTLS.lookup(presetHostname) assert.strictEqual(ip, presetIp) // test preset console.log('\n\n') -assert.strictEqual(dnsProviders.cloudflareTLS.type, 'tls') +assert.strictEqual(dnsProviders.cloudflareTLS.dnsType, 'TLS') // ip = await dnsProviders.cloudflareTLS.lookup(hostname1) // console.log('===> test cloudflareTLS:', ip, '\n\n') -assert.strictEqual(dnsProviders.quad9TLS.type, 'tls') +assert.strictEqual(dnsProviders.quad9TLS.dnsType, 'TLS') // ip = await dnsProviders.quad9TLS.lookup(hostname1) // console.log('===> test quad9TLS:', ip, '\n\n') -assert.strictEqual(dnsProviders.aliyunTLS.type, 'tls') +assert.strictEqual(dnsProviders.aliyunTLS.dnsType, 'TLS') // ip = await dnsProviders.aliyunTLS.lookup(hostname1) // console.log('===> test aliyunTLS:', ip, '\n\n') -assert.strictEqual(dnsProviders.aliyunTLS2.type, 'tls') +assert.strictEqual(dnsProviders.aliyunTLS2.dnsType, 'TLS') // ip = await dnsProviders.aliyunTLS2.lookup(hostname1) // console.log('===> test aliyunTLS2:', ip, '\n\n') -assert.strictEqual(dnsProviders.safe360TLS.type, 'tls') +assert.strictEqual(dnsProviders.safe360TLS.dnsType, 'TLS') // ip = await dnsProviders.safe360TLS.lookup(hostname1) // console.log('===> test safe360TLS:', ip, '\n\n') @@ -167,11 +176,11 @@ ip = await dnsProviders.googleTCP.lookup(presetHostname) assert.strictEqual(ip, presetIp) // test preset console.log('\n\n') -assert.strictEqual(dnsProviders.googleTCP.type, 'tcp') +assert.strictEqual(dnsProviders.googleTCP.dnsType, 'TCP') // ip = await dnsProviders.googleTCP.lookup(hostname1) // console.log('===> test googleTCP:', ip, '\n\n') -assert.strictEqual(dnsProviders.aliyunTCP.type, 'tcp') +assert.strictEqual(dnsProviders.aliyunTCP.dnsType, 'TCP') // ip = await dnsProviders.aliyunTCP.lookup(hostname1) // console.log('===> test aliyunTCP:', ip, '\n\n') @@ -181,10 +190,10 @@ ip = await dnsProviders.googleUDP.lookup(presetHostname) assert.strictEqual(ip, presetIp) // test preset console.log('\n\n') -assert.strictEqual(dnsProviders.googleUDP.type, 'udp') +assert.strictEqual(dnsProviders.googleUDP.dnsType, 'UDP') // ip = await dnsProviders.googleUDP.lookup(hostname1) // console.log('===> test googleUDP:', ip, '\n\n') -assert.strictEqual(dnsProviders.aliyunUDP.type, 'udp') +assert.strictEqual(dnsProviders.aliyunUDP.dnsType, 'UDP') // ip = await dnsProviders.aliyunUDP.lookup(hostname1) // console.log('===> test aliyunUDP:', ip, '\n\n')