github
/
dev-sidecar
mirror of https://github.com/docmirror/dev-sidecar
1
0
Fork 0
Code Issues Releases Wiki Activity

feat: 脚本支持，gitclone加速链接复制

pull/180/head
xiaojunnuo 2020-11-17 18:08:26 +08:00
parent 193c381a10
commit 02fead91f7
22 changed files with 186 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
README.md
View File

@ -13,7 +13,8 @@
可解决npm install 时某些安装包下载不下来的问题
### 2、 dns优选
根据网络状况智能解析域名ip地址，获取最佳网络速度
根据网络状况智能解析最佳域名ip地址，获取最佳网络速度
第一次访问会比较慢，等多次访问之后，慢慢的选到比较快的ip，之后就很快了
比如：
1. 解决git push 偶尔失败需要输入账号密码的问题（
fatal: TaskCanceledException encountered / fatal: HttpRequestException encountered）
@ -76,10 +77,14 @@ recaptcha 图片验证码加速
4. 某些库用cnpm也下载不下来的话，可以试试打开dev-sidecar的npm加速
### 其他加速
1. git clone 加速
新增快捷方式：
![](./doc/clone.png)
方式2：
> 使用方式用实际的名称替换{}的内容，即可加速clone
> https://hub.fastgit.org/{username}/{reponame}.git
> clone 出来的 remote "origin" 为fastgit的地址，需要手动改回来
> 你也可以直接使用他们的clone加速工具 [fgit-go](https://github.com/FastGitORG/fgit-go)
2. github.com的镜像网站(注意：不能登录)
>1. [hub.fastgit.org](https://hub.fastgit.org/)
>2. [github.com.cnpmjs.org](https://github.com.cnpmjs.org/) 这个很容易超限
@ -117,11 +122,9 @@ const intercepts = {
}
```
### DNS优选
某些域名（比如api.github.com）会被解析到新加坡的ip上，新加坡的服务器在上午挺好，到了晚上就卡死，基本不可用。
所以将这些域名解析到美国服务器上就可以正常访问
另外，配置了dns mapping的域名，将会从dns获取到的ip列表中选择相对快一点的服务器进行访问
### DNS优选配置
某些域名解析出来的ip会无法访问，（比如api.github.com会被解析到新加坡的ip上，新加坡的服务器在上午挺好，到了晚上就卡死，基本不可用）
通过从dns上获取ip列表，切换不同的ip进行尝试，最终会挑选到一个最快的ip
```js
dns: {
@ -143,6 +146,7 @@ const intercepts = {
本项目参考如下开源项目
* [node-mitmproxy](https://github.com/wuchangming/node-mitmproxy)
* [ReplaceGoogleCDN](https://github.com/justjavac/ReplaceGoogleCDN)
* [github增强油猴脚本](https://greasyfork.org/zh-CN/scripts/412245-github-%E5%A2%9E%E5%BC%BA-%E9%AB%98%E9%80%9F%E4%B8%8B%E8%BD%BD)
本项目加速资源由如下组织提供
* [fastgit](https://fastgit.org/)

BIN
doc/clone.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 51 KiB

15
packages/core/src/config/index.js
View File

@ -20,14 +20,17 @@ module.exports = {
'/.*/.*/blame/': {
redirect: 'hub.fastgit.org'
},
'/.+/[^\\/]+$': {
script: [
'jquery',
'github'
]
},
'/.*': {
proxy: 'github.com',
backup: [
'github.docmirror.cn'
'github.docmirror.cn/_proxy'
]
},
'/.*/[^\\/]*$': {
script: 'console.log("123123132")'
}
},
'raw.githubusercontent.com': {
@ -122,7 +125,9 @@ module.exports = {
'*.githubassets.com': 'usa',
// "解决push的时候需要输入密码的问题",
'github.com': 'usa',
'*.vuepress.vuejs.org': 'usa'
'*.vuepress.vuejs.org': 'usa',
'github.docmirror.cn': 'usa',
'gh.docmirror.top': 'usa'
}
}
},

2
packages/core/src/utils/util.log.js
View File

@ -5,7 +5,7 @@ const getDefaultConfigBasePath = function () {
return path.resolve(userHome, './.dev-sidecar')
}
log4js.configure({
appenders: { std: { type: 'stdout' }, file: { type: 'file', pattern: '.yyyy-MM-dd', daysToKeep: 3, filename: getDefaultConfigBasePath() + '/logs/server.log' } },
appenders: { std: { type: 'stdout' }, file: { type: 'file', pattern: 'yyyy-MM-dd', daysToKeep: 3, filename: getDefaultConfigBasePath() + '/logs/server.log' } },
categories: { default: { appenders: ['file', 'std'], level: 'info' } }
})
const logger = log4js.getLogger('server')

2
packages/gui/src/utils/util.log.js
View File

@ -5,7 +5,7 @@ const getDefaultConfigBasePath = function () {
return path.resolve(userHome, './.dev-sidecar')
}
log4js.configure({
appenders: { std: { type: 'stdout' }, file: { type: 'file', pattern: '.yyyy-MM-dd', daysToKeep: 3, filename: getDefaultConfigBasePath() + '/logs/server.log' } },
appenders: { std: { type: 'stdout' }, file: { type: 'file', pattern: 'yyyy-MM-dd', daysToKeep: 3, filename: getDefaultConfigBasePath() + '/logs/server.log' } },
categories: { default: { appenders: ['file', 'std'], level: 'info' } }
})
const logger = log4js.getLogger('server')

1
packages/mitmproxy/package.json
View File

@ -15,6 +15,7 @@
"child_process": "^1.0.2",
"colors": "^1.1.2",
"commander": "^2.9.0",
"crypto-js": "^4.0.0",
"debug": "^4.1.1",
"dns-over-http": "^0.2.0",
"dns-over-tls": "^0.0.8",

12
packages/mitmproxy/src/lib/choice/index.js
View File

@ -38,7 +38,7 @@ class DynamicChoice {
if (this.count[item]) {
continue
}
this.count[item] = { value: item, total: 0, error: 0, keepErrorCount: 0, successRate: 1 }
this.count[item] = { value: item, total: 0, error: 0, keepErrorCount: 0, successRate: 1.0 }
}
this.doCount(this.value, false)
}
@ -47,10 +47,14 @@ class DynamicChoice {
// 将count里面根据权重排序
const list = []
for (const key in this.count) {
list.put(this.count[key])
list.push(this.count[key])
}
list.sort(function (a, b) { return a.successRate - b.successRate })
list.sort((a, b) => {
return b.successRate - a.successRate
})
console.log('do rank', list)
const backup = list.map(item => item.value)
this.setBackupList(backup)
}
@ -87,7 +91,7 @@ class DynamicChoice {
} else {
count.total++
}
count.successRate = 1 - (count.error / count.total)
count.successRate = 1.0 - (count.error / count.total)
if (isError && this.value === value) {
// 连续错误4次，切换下一个
if (count.keepErrorCount >= 4) {

7
packages/mitmproxy/src/lib/interceptor/impl/proxy.js
View File

@ -1,23 +1,24 @@
const url = require('url')
module.exports = {
requestIntercept (context, interceptOpt, req, res, ssl, next) {
const { rOptions, log, RequestConter } = context
const { rOptions, log, RequestCounter } = context
let proxyConf = interceptOpt.proxy
if (RequestConter && interceptOpt.backup && interceptOpt.backup.length > 0) {
if (RequestCounter && interceptOpt.backup && interceptOpt.backup.length > 0) {
// 优选逻辑
const backup = [proxyConf]
for (const bk of interceptOpt.backup) {
backup.push(bk)
}
const key = interceptOpt.key
const count = RequestConter.getOrCreate(key, backup)
const count = RequestCounter.getOrCreate(key, backup)
if (count.value == null) {
count.doRank()
}
if (count.value == null) {
log.error('count value is null', count)
} else {
count.doCount(count.value)
proxyConf = count.value
context.requestCount = {
key,

36
packages/mitmproxy/src/lib/interceptor/impl/script.js
View File

@ -1,18 +1,36 @@
const contextPath = '/____ds_script____/'
const monkey = require('../../monkey')
const CryptoJs = require('crypto-js')
function getScript (key, script) {
const scriptUrl = contextPath + key
const hash = CryptoJs.SHA256(script).toString(CryptoJs.enc.Base64)
return `
<script crossorigin="anonymous" defer="defer" type="application/javascript"
integrity="sha256-${hash}"
src="${scriptUrl}"></script>
`
}
module.exports = {
responseIntercept (context, interceptOpt, req, res, proxyReq, proxyRes, ssl, next) {
const { rOptions, log } = context
const script = `
<script>
try{
${interceptOpt.script}
}catch (err){
console.error('脚本执行出错：',err)
let keys = interceptOpt.script
if (typeof keys === 'string') {
keys = [keys]
}
let tags = getScript('global', monkey.get().global.script)
for (const key of keys) {
const script = monkey.get()[key]
if (script == null) {
continue
}
</script>
`
const scriptTag = getScript(key, script.script)
tags += '\r\n' + scriptTag
}
log.info('responseIntercept: append script', rOptions.hostname, rOptions.path)
return {
head: script
head: tags
}
},
is (interceptOpt) {

30
packages/mitmproxy/src/lib/monkey/index.js
View File

@ -1,16 +1,17 @@
const fs = require('fs')
const path = require('path')
let scripts = []
let scripts
function buildScript (sc, content) {
const grant = sc.grant
const pre = '(function () { \r\n'
const pre = 'window.addEventListener("load",' +
' ()=> { \r\n'
let grantSc = ''
for (const item of grant) {
grantSc += 'const ' + item + ' = Monkey_Grants[\'' + item + '\']\r\n'
grantSc += 'const ' + item + ' = window.__ds_global__[\'' + item + '\']\r\n'
}
const tail = content + '\r\n' +
'})()'
const tail = ';' + content + '\r\n' +
'})'
return pre + grantSc + tail
}
@ -25,7 +26,6 @@ function loadScript (content) {
const arr = content.split(annoFlag)
const start = 0
console.log('arr', arr.length)
const confStr = arr[start]
const confItemArr = confStr.split('\n')
const sc = {
@ -54,14 +54,24 @@ function loadScript (content) {
return sc
}
module.exports = {
function readFile (script) {
return fs.readFileSync(path.join(__dirname, './scripts/' + script)).toString()
}
const api = {
get () {
if (scripts == null) {
api.load()
}
return scripts
},
load () {
const github = loadScript(fs.readFileSync(path.join(__dirname, './scripts/github.script')).toString())
scripts = []
scripts.push(github)
scripts = {}
scripts.github = loadScript(readFile('github.script'))
scripts.jquery = { script: readFile('jquery.min.js') }
scripts.global = { script: readFile('global.script') }
return scripts
}
}
module.exports = api

1
packages/mitmproxy/src/lib/monkey/scripts/github.script
View File

@ -20,6 +20,7 @@
// ==/UserScript==
(function () {
console.log('github script loaded')
var download_url = [
'https://gh.con.sh',
'https://gh.api.99988866.xyz',

9
packages/mitmproxy/src/lib/monkey/scripts/global.script Normal file
View File

@ -0,0 +1,9 @@
window.__ds_global__={
GM_registerMenuCommand: () => {},
GM_unregisterMenuCommand: () => {},
GM_openInTab: () => {},
GM_getValue: () => {},
GM_setValue: () => {},
GM_notification: () => {}
}
console.log('ds_global loaded')

2
packages/mitmproxy/src/lib/monkey/scripts/jquery.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

21
packages/mitmproxy/src/lib/proxy/middleware/HtmlMiddleware.js
View File

@ -35,8 +35,7 @@ function chunkReplace (_this, chunk, enc, callback, append) {
if (append && append.body) {
chunkString = injectScriptIntoBodyHtml(chunkString, append.body)
}
// eslint-disable-next-line node/no-deprecated-api
_this.push(new Buffer(chunkString))
_this.push(Buffer.from(chunkString))
callback()
}
@ -62,9 +61,23 @@ module.exports = {
const newkey = key
if (isHtml && key === 'content-length') {
// do nothing
} else {
res.setHeader(newkey, proxyRes.headers[key])
return
}
if (isHtml && key === 'content-security-policy') {
// content-security-policy
let policy = proxyRes.headers[key]
const reg = /script-src ([^:]*);/i
const matched = policy.match(reg)
if (matched) {
if (matched[1].indexOf('self') < 0) {
policy = policy.replace('script-src', 'script-src \'self\' ')
}
}
res.setHeader(newkey, policy)
return
}
res.setHeader(newkey, proxyRes.headers[key])
}
})

23
packages/mitmproxy/src/lib/proxy/middleware/ScriptMiddleware.js Normal file
View File

@ -0,0 +1,23 @@
const contextPath = '/____ds_script____/'
const monkey = require('../../monkey')
module.exports = {
requestIntercept (context, req, res, ssl, next) {
const { rOptions, log } = context
const urlPath = rOptions.path
const filename = urlPath.replace(contextPath, '')
const script = monkey.get()[filename]
log.info('ds_script', filename, script != null)
res.writeHead(200)
res.write(script.script)
res.end()
return true
},
is (rOptions) {
if (rOptions.path.indexOf(contextPath) !== 0) {
return false
}
return true
}
}

3
packages/mitmproxy/src/lib/proxy/mitmproxy/createConnectHandler.js
View File

@ -61,6 +61,9 @@ function connect (req, cltSocket, head, hostname, port, dnsConfig) {
cltSocket.pipe(proxySocket)
})
cltSocket.on('error', (e) => {
log.error('cltSocket error', e.message)
})
proxySocket.on('timeout', () => {
const end = new Date().getTime()
log.info('代理socket timeout：', hostname, port, (end - start) + 'ms')

68
packages/mitmproxy/src/lib/proxy/mitmproxy/createRequestHandler.js
View File

@ -6,6 +6,7 @@ const DnsUtil = require('../../dns/index')
const log = require('../../../utils/util.log')
const HtmlMiddleware = require('../middleware/HtmlMiddleware')
const RequestCounter = require('../../choice/RequestCounter')
const ScriptMiddleware = require('../middleware/ScriptMiddleware')
// create requestHandler function
module.exports = function createRequestHandler (createIntercepts, externalProxy, dnsConfig) {
// return
@ -29,7 +30,7 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
if (interceptors == null) {
interceptors = []
}
const reqIncpts = interceptors.filter(item => { return item.requestIntercept != null })
let reqIncpts = interceptors.filter(item => { return item.requestIntercept != null })
const resIncpts = interceptors.filter(item => { return item.responseIntercept != null })
const requestInterceptorPromise = () => {
@ -38,6 +39,13 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
resolve()
}
try {
if (ScriptMiddleware.is(rOptions)) {
if (reqIncpts == null) {
reqIncpts = []
}
reqIncpts.unshift(ScriptMiddleware)
}
if (reqIncpts && reqIncpts.length > 0) {
for (const reqIncpt of reqIncpts) {
const goNext = reqIncpt.requestIntercept(context, req, res, ssl, next)
@ -56,6 +64,19 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
})
}
function countSlow (isDnsIntercept) {
if (isDnsIntercept) {
const { dns, ip, hostname } = isDnsIntercept
dns.count(hostname, ip, true)
log.error('记录ip失败次数,用于优选ip：', hostname, ip)
}
const counter = context.requestCount
if (counter != null) {
counter.count.doCount(counter.value, true)
log.error('记录prxoy失败次数：', counter.value)
}
}
const proxyRequestPromise = async () => {
rOptions.host = rOptions.hostname || rOptions.host || 'localhost'
return new Promise((resolve, reject) => {
@ -92,24 +113,19 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
}
proxyReq = (rOptions.protocol === 'https:' ? https : http).request(rOptions, (proxyRes) => {
const end = new Date().getTime()
const cost = end - start
if (rOptions.protocol === 'https:') {
log.info('代理请求返回:', url, (end - start) + 'ms')
log.info('代理请求返回:', url, cost + 'ms')
}
if (cost > 8000) {
countSlow(isDnsIntercept)
}
resolve(proxyRes)
})
proxyReq.on('timeout', () => {
const end = new Date().getTime()
if (isDnsIntercept) {
const { dns, ip, hostname } = isDnsIntercept
dns.count(hostname, ip, true)
log.error('记录ip失败次数,用于优选ip：', hostname, ip)
}
const counter = context.requestCount
if (counter != null) {
counter.count.doCount(counter.value, true)
log.error('记录prxoy失败次数：', counter.value)
}
countSlow(isDnsIntercept)
log.error('代理请求超时', rOptions.protocol, rOptions.hostname, rOptions.path, (end - start) + 'ms')
proxyReq.end()
proxyReq.destroy()
@ -120,16 +136,7 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
proxyReq.on('error', (e) => {
const end = new Date().getTime()
if (isDnsIntercept) {
const { dns, ip, hostname } = isDnsIntercept
dns.count(hostname, ip, true)
log.error('记录ip失败次数,用于优选ip：', hostname, ip)
}
const counter = context.requestCount
if (counter != null) {
counter.count.doCount(counter.value, true)
log.error('记录prxoy失败次数：', counter.value)
}
countSlow(isDnsIntercept)
log.error('代理请求错误', e.code, e.message, rOptions.hostname, rOptions.path, (end - start) + 'ms')
reject(e)
})
@ -140,16 +147,7 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
log.error('代理请求被取消', rOptions.hostname, rOptions.path, cost + 'ms')
if (cost > 8000) {
if (isDnsIntercept) {
const { dns, ip, hostname } = isDnsIntercept
dns.count(hostname, ip, true)
log.error('记录ip失败次数,用于优选ip：', hostname, ip)
}
const counter = context.requestCount
if (counter != null) {
counter.count.doCount(counter.value, true)
log.error('记录prxoy失败次数：', counter.value)
}
countSlow(isDnsIntercept)
}
if (res.writableEnded) {
@ -193,11 +191,7 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
// // console.log('BODY: ')
// })
proxyRes.on('error', (error) => {
const counter = context.requestCount
if (counter != null) {
counter.count.doCount(counter.value, true)
log.error('记录prxoy失败次数：', counter.value)
}
countSlow()
log.error('proxy res error', error)
})

11
packages/mitmproxy/src/lib/utils/logger.js
View File

@ -1,11 +0,0 @@
const debug = require('debug')
module.exports = function getLogger (name) {
return {
debug: debug(`dev-sidecar:${name}:debug`),
info: debug(`dev-sidecar:${name}:info`),
error: debug(`dev-sidecar:${name}:error`)
}
}
debug.enable('dev-sidecar:*')

2
packages/mitmproxy/src/utils/util.log.js
View File

@ -1,7 +1,7 @@
const log4js = require('log4js')
const proxyConfig = require('../lib/proxy/common/config')
log4js.configure({
appenders: { std: { type: 'stdout' }, file: { type: 'file', pattern: '.yyyy-MM-dd', daysToKeep: 3, filename: proxyConfig.getDefaultCABasePath() + '/logs/server.log' } },
appenders: { std: { type: 'stdout' }, file: { type: 'file', pattern: 'yyyy-MM-dd', daysToKeep: 3, filename: proxyConfig.getDefaultCABasePath() + '/logs/server.log' } },
categories: { default: { appenders: ['file', 'std'], level: 'info' } }
})
const logger = log4js.getLogger('server')

9
packages/mitmproxy/test/dnsTest.mjs
View File

@ -20,4 +20,11 @@ const dnsProviders = dns.initDNS({
// dnsProviders.usa.lookup(hostname)
const hostname1 = 'api.github.com'
dnsProviders.aliyun.lookup(hostname1)
dnsProviders.usa.lookup(hostname1)
const hostname2 = 'hk.docmirror.cn'
dnsProviders.usa.lookup(hostname2)
const hostname3 = 'github.docmirror.cn'
dnsProviders.usa.lookup(hostname3)
const hostname4 = 'gh.docmirror.top'
dnsProviders.usa.lookup(hostname4)

4
packages/mitmproxy/test/sha256Test.js Normal file
View File

@ -0,0 +1,4 @@
const CryptoJs = require('crypto-js')
const ret = CryptoJs.SHA256('111111111111')
console.log(ret.toString(CryptoJs.enc.Base64))
console.log(1 / 2)

5
packages/mitmproxy/yarn.lock
View File

@ -2159,6 +2159,11 @@ crypto-browserify@^3.11.0:
randombytes "^2.0.0"
randomfill "^1.0.3"
crypto-js@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/crypto-js/-/crypto-js-4.0.0.tgz#2904ab2677a9d042856a2ea2ef80de92e4a36dcc"
integrity sha512-bzHZN8Pn+gS7DQA6n+iUmBfl0hO5DJq++QP3U6uTucDtk/0iGpXd/Gg7CGR0p8tJhofJyaKoWBuJI4eAO00BBg==
cyclist@^1.0.1:
version "1.0.1"
resolved "https://registry.npm.taobao.org/cyclist/download/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9"