feat: 脚本支持，gitclone加速链接复制

README.md

@@ -13,7 +13,8 @@
 可解决npm install 时某些安装包下载不下来的问题
 
 ### 2、 dns优选
-根据网络状况智能解析域名ip地址，获取最佳网络速度 
+根据网络状况智能解析最佳域名ip地址，获取最佳网络速度     
+第一次访问会比较慢，等多次访问之后，慢慢的选到比较快的ip，之后就很快了      
 比如：   
 1. 解决git push 偶尔失败需要输入账号密码的问题（
 fatal: TaskCanceledException encountered  /  fatal: HttpRequestException encountered）
@@ -76,10 +77,14 @@ recaptcha 图片验证码加速
  4. 某些库用cnpm也下载不下来的话，可以试试打开dev-sidecar的npm加速
 ### 其他加速
  1. git clone 加速   
+ 新增快捷方式：
+ ![](./doc/clone.png)  
+ 方式2：
   > 使用方式用实际的名称替换{}的内容，即可加速clone  
   > https://hub.fastgit.org/{username}/{reponame}.git     
   > clone 出来的 remote "origin" 为fastgit的地址，需要手动改回来  
   > 你也可以直接使用他们的clone加速工具 [fgit-go](https://github.com/FastGitORG/fgit-go)
+
  2. github.com的镜像网站(注意：不能登录)   
    >1. [hub.fastgit.org](https://hub.fastgit.org/) 
    >2. [github.com.cnpmjs.org](https://github.com.cnpmjs.org/) 这个很容易超限
@@ -117,11 +122,9 @@ const intercepts = {
 }
 ```
 
-### DNS优选
+### DNS优选配置
-某些域名（比如api.github.com）会被解析到新加坡的ip上，新加坡的服务器在上午挺好，到了晚上就卡死，基本不可用。     
+某些域名解析出来的ip会无法访问，（比如api.github.com会被解析到新加坡的ip上，新加坡的服务器在上午挺好，到了晚上就卡死，基本不可用）        
-所以将这些域名解析到美国服务器上就可以正常访问
+通过从dns上获取ip列表，切换不同的ip进行尝试，最终会挑选到一个最快的ip
-
-另外，配置了dns mapping的域名，将会从dns获取到的ip列表中选择相对快一点的服务器进行访问
 
 ```js
  dns: {
@@ -143,6 +146,7 @@ const intercepts = {
 本项目参考如下开源项目
 * [node-mitmproxy](https://github.com/wuchangming/node-mitmproxy)   
 * [ReplaceGoogleCDN](https://github.com/justjavac/ReplaceGoogleCDN)
+* [github增强油猴脚本](https://greasyfork.org/zh-CN/scripts/412245-github-%E5%A2%9E%E5%BC%BA-%E9%AB%98%E9%80%9F%E4%B8%8B%E8%BD%BD)
 
 本项目加速资源由如下组织提供
 * [fastgit](https://fastgit.org/)

packages/core/src/config/index.js

@@ -20,14 +20,17 @@ module.exports = {
         '/.*/.*/blame/': {
           redirect: 'hub.fastgit.org'
         },
+        '/.+/[^\\/]+$': {
+          script: [
+            'jquery',
+            'github'
+          ]
+        },
         '/.*': {
           proxy: 'github.com',
           backup: [
-            'github.docmirror.cn'
+            'github.docmirror.cn/_proxy'
           ]
-        },
-        '/.*/[^\\/]*$': {
-          script: 'console.log("123123132")'
         }
       },
       'raw.githubusercontent.com': {
@@ -122,7 +125,9 @@ module.exports = {
         '*.githubassets.com': 'usa',
         // "解决push的时候需要输入密码的问题",
         'github.com': 'usa',
-        '*.vuepress.vuejs.org': 'usa'
+        '*.vuepress.vuejs.org': 'usa',
+        'github.docmirror.cn': 'usa',
+        'gh.docmirror.top': 'usa'
       }
     }
   },

packages/core/src/utils/util.log.js

@@ -5,7 +5,7 @@ const getDefaultConfigBasePath = function () {
   return path.resolve(userHome, './.dev-sidecar')
 }
 log4js.configure({
-  appenders: { std: { type: 'stdout' }, file: { type: 'file', pattern: '.yyyy-MM-dd', daysToKeep: 3, filename: getDefaultConfigBasePath() + '/logs/server.log' } },
+  appenders: { std: { type: 'stdout' }, file: { type: 'file', pattern: 'yyyy-MM-dd', daysToKeep: 3, filename: getDefaultConfigBasePath() + '/logs/server.log' } },
   categories: { default: { appenders: ['file', 'std'], level: 'info' } }
 })
 const logger = log4js.getLogger('server')

packages/gui/src/utils/util.log.js

@@ -5,7 +5,7 @@ const getDefaultConfigBasePath = function () {
   return path.resolve(userHome, './.dev-sidecar')
 }
 log4js.configure({
-  appenders: { std: { type: 'stdout' }, file: { type: 'file', pattern: '.yyyy-MM-dd', daysToKeep: 3, filename: getDefaultConfigBasePath() + '/logs/server.log' } },
+  appenders: { std: { type: 'stdout' }, file: { type: 'file', pattern: 'yyyy-MM-dd', daysToKeep: 3, filename: getDefaultConfigBasePath() + '/logs/server.log' } },
   categories: { default: { appenders: ['file', 'std'], level: 'info' } }
 })
 const logger = log4js.getLogger('server')

packages/mitmproxy/package.json

@@ -15,6 +15,7 @@
     "child_process": "^1.0.2",
     "colors": "^1.1.2",
     "commander": "^2.9.0",
+    "crypto-js": "^4.0.0",
     "debug": "^4.1.1",
     "dns-over-http": "^0.2.0",
     "dns-over-tls": "^0.0.8",

packages/mitmproxy/src/lib/choice/index.js

@@ -38,7 +38,7 @@ class DynamicChoice {
       if (this.count[item]) {
         continue
       }
-      this.count[item] = { value: item, total: 0, error: 0, keepErrorCount: 0, successRate: 1 }
+      this.count[item] = { value: item, total: 0, error: 0, keepErrorCount: 0, successRate: 1.0 }
     }
     this.doCount(this.value, false)
   }
@@ -47,10 +47,14 @@ class DynamicChoice {
     // 将count里面根据权重排序
     const list = []
     for (const key in this.count) {
-      list.put(this.count[key])
+      list.push(this.count[key])
     }
-    list.sort(function (a, b) { return a.successRate - b.successRate })
+    list.sort((a, b) => {
+      return b.successRate - a.successRate
+    })
+    console.log('do rank', list)
     const backup = list.map(item => item.value)
+
     this.setBackupList(backup)
   }
 
@@ -87,7 +91,7 @@ class DynamicChoice {
     } else {
       count.total++
     }
-    count.successRate = 1 - (count.error / count.total)
+    count.successRate = 1.0 - (count.error / count.total)
     if (isError && this.value === value) {
       // 连续错误4次，切换下一个
       if (count.keepErrorCount >= 4) {

packages/mitmproxy/src/lib/interceptor/impl/proxy.js

@@ -1,23 +1,24 @@
 const url = require('url')
 module.exports = {
   requestIntercept (context, interceptOpt, req, res, ssl, next) {
-    const { rOptions, log, RequestConter } = context
+    const { rOptions, log, RequestCounter } = context
 
     let proxyConf = interceptOpt.proxy
-    if (RequestConter && interceptOpt.backup && interceptOpt.backup.length > 0) {
+    if (RequestCounter && interceptOpt.backup && interceptOpt.backup.length > 0) {
       // 优选逻辑
       const backup = [proxyConf]
       for (const bk of interceptOpt.backup) {
         backup.push(bk)
       }
       const key = interceptOpt.key
-      const count = RequestConter.getOrCreate(key, backup)
+      const count = RequestCounter.getOrCreate(key, backup)
       if (count.value == null) {
         count.doRank()
       }
       if (count.value == null) {
         log.error('count value is null', count)
       } else {
+        count.doCount(count.value)
         proxyConf = count.value
         context.requestCount = {
           key,

packages/mitmproxy/src/lib/interceptor/impl/script.js

@@ -1,18 +1,36 @@
+const contextPath = '/____ds_script____/'
+const monkey = require('../../monkey')
+const CryptoJs = require('crypto-js')
+function getScript (key, script) {
+  const scriptUrl = contextPath + key
+
+  const hash = CryptoJs.SHA256(script).toString(CryptoJs.enc.Base64)
+  return `
+      <script crossorigin="anonymous" defer="defer"   type="application/javascript" 
+      integrity="sha256-${hash}" 
+      src="${scriptUrl}"></script>
+      `
+}
+
 module.exports = {
   responseIntercept (context, interceptOpt, req, res, proxyReq, proxyRes, ssl, next) {
     const { rOptions, log } = context
-    const script = `
+    let keys = interceptOpt.script
-      <script>
+    if (typeof keys === 'string') {
-      try{
+      keys = [keys]
-          ${interceptOpt.script}
+    }
-      }catch (err){
+    let tags = getScript('global', monkey.get().global.script)
-          console.error('脚本执行出错：',err)
+    for (const key of keys) {
+      const script = monkey.get()[key]
+      if (script == null) {
+        continue
+      }
+      const scriptTag = getScript(key, script.script)
+      tags += '\r\n' + scriptTag
     }
-      </script>
-      `
     log.info('responseIntercept: append script', rOptions.hostname, rOptions.path)
     return {
-      head: script
+      head: tags
     }
   },
   is (interceptOpt) {

packages/mitmproxy/src/lib/monkey/index.js

@@ -1,16 +1,17 @@
 const fs = require('fs')
 const path = require('path')
-let scripts = []
+let scripts
 
 function buildScript (sc, content) {
   const grant = sc.grant
-  const pre = '(function () { \r\n'
+  const pre = 'window.addEventListener("load",' +
+      ' ()=> { \r\n'
   let grantSc = ''
   for (const item of grant) {
-    grantSc += 'const ' + item + ' = Monkey_Grants[\'' + item + '\']\r\n'
+    grantSc += 'const ' + item + ' = window.__ds_global__[\'' + item + '\']\r\n'
   }
-  const tail = content + '\r\n' +
+  const tail = ';' + content + '\r\n' +
-      '})()'
+      '})'
   return pre + grantSc + tail
 }
 
@@ -25,7 +26,6 @@ function loadScript (content) {
   const arr = content.split(annoFlag)
   const start = 0
 
-  console.log('arr', arr.length)
   const confStr = arr[start]
   const confItemArr = confStr.split('\n')
   const sc = {
@@ -54,14 +54,24 @@ function loadScript (content) {
   return sc
 }
 
-module.exports = {
+function readFile (script) {
+  return fs.readFileSync(path.join(__dirname, './scripts/' + script)).toString()
+}
+
+const api = {
   get () {
+    if (scripts == null) {
+      api.load()
+    }
     return scripts
   },
   load () {
-    const github = loadScript(fs.readFileSync(path.join(__dirname, './scripts/github.script')).toString())
+    scripts = {}
-    scripts = []
+    scripts.github = loadScript(readFile('github.script'))
-    scripts.push(github)
+    scripts.jquery = { script: readFile('jquery.min.js') }
+    scripts.global = { script: readFile('global.script') }
     return scripts
   }
 }
+
+module.exports = api

packages/mitmproxy/src/lib/monkey/scripts/github.script

@@ -20,6 +20,7 @@
 // ==/UserScript==
 
 (function () {
+  console.log('github script  loaded')
   var download_url = [
     'https://gh.con.sh',
     'https://gh.api.99988866.xyz',

packages/mitmproxy/src/lib/monkey/scripts/global.script

@@ -0,0 +1,9 @@
+window.__ds_global__={
+    GM_registerMenuCommand: () => {},
+    GM_unregisterMenuCommand: () => {},
+    GM_openInTab: () => {},
+    GM_getValue: () => {},
+    GM_setValue: () => {},
+    GM_notification: () => {}
+}
+console.log('ds_global loaded')

packages/mitmproxy/src/lib/proxy/middleware/HtmlMiddleware.js

@@ -35,8 +35,7 @@ function chunkReplace (_this, chunk, enc, callback, append) {
   if (append && append.body) {
     chunkString = injectScriptIntoBodyHtml(chunkString, append.body)
   }
-  // eslint-disable-next-line node/no-deprecated-api
+  _this.push(Buffer.from(chunkString))
-  _this.push(new Buffer(chunkString))
   callback()
 }
 
@@ -62,9 +61,23 @@ module.exports = {
           const newkey = key
           if (isHtml && key === 'content-length') {
             // do nothing
-          } else {
+            return
-            res.setHeader(newkey, proxyRes.headers[key])
           }
+          if (isHtml && key === 'content-security-policy') {
+            // content-security-policy
+            let policy = proxyRes.headers[key]
+            const reg = /script-src ([^:]*);/i
+            const matched = policy.match(reg)
+            if (matched) {
+              if (matched[1].indexOf('self') < 0) {
+                policy = policy.replace('script-src', 'script-src \'self\' ')
+              }
+            }
+            res.setHeader(newkey, policy)
+            return
+          }
+
+          res.setHeader(newkey, proxyRes.headers[key])
         }
       })
 

packages/mitmproxy/src/lib/proxy/middleware/ScriptMiddleware.js

@@ -0,0 +1,23 @@
+const contextPath = '/____ds_script____/'
+const monkey = require('../../monkey')
+module.exports = {
+  requestIntercept (context, req, res, ssl, next) {
+    const { rOptions, log } = context
+    const urlPath = rOptions.path
+    const filename = urlPath.replace(contextPath, '')
+
+    const script = monkey.get()[filename]
+
+    log.info('ds_script', filename, script != null)
+    res.writeHead(200)
+    res.write(script.script)
+    res.end()
+    return true
+  },
+  is (rOptions) {
+    if (rOptions.path.indexOf(contextPath) !== 0) {
+      return false
+    }
+    return true
+  }
+}

packages/mitmproxy/src/lib/proxy/mitmproxy/createConnectHandler.js

@@ -61,6 +61,9 @@ function connect (req, cltSocket, head, hostname, port, dnsConfig) {
       cltSocket.pipe(proxySocket)
     })
 
+    cltSocket.on('error', (e) => {
+      log.error('cltSocket error', e.message)
+    })
     proxySocket.on('timeout', () => {
       const end = new Date().getTime()
       log.info('代理socket timeout：', hostname, port, (end - start) + 'ms')

packages/mitmproxy/src/lib/proxy/mitmproxy/createRequestHandler.js

@@ -6,6 +6,7 @@ const DnsUtil = require('../../dns/index')
 const log = require('../../../utils/util.log')
 const HtmlMiddleware = require('../middleware/HtmlMiddleware')
 const RequestCounter = require('../../choice/RequestCounter')
+const ScriptMiddleware = require('../middleware/ScriptMiddleware')
 // create requestHandler function
 module.exports = function createRequestHandler (createIntercepts, externalProxy, dnsConfig) {
   // return
@@ -29,7 +30,7 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
     if (interceptors == null) {
       interceptors = []
     }
-    const reqIncpts = interceptors.filter(item => { return item.requestIntercept != null })
+    let reqIncpts = interceptors.filter(item => { return item.requestIntercept != null })
     const resIncpts = interceptors.filter(item => { return item.responseIntercept != null })
 
     const requestInterceptorPromise = () => {
@@ -38,6 +39,13 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
           resolve()
         }
         try {
+          if (ScriptMiddleware.is(rOptions)) {
+            if (reqIncpts == null) {
+              reqIncpts = []
+            }
+            reqIncpts.unshift(ScriptMiddleware)
+          }
+
           if (reqIncpts && reqIncpts.length > 0) {
             for (const reqIncpt of reqIncpts) {
               const goNext = reqIncpt.requestIntercept(context, req, res, ssl, next)
@@ -56,6 +64,19 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
       })
     }
 
+    function countSlow (isDnsIntercept) {
+      if (isDnsIntercept) {
+        const { dns, ip, hostname } = isDnsIntercept
+        dns.count(hostname, ip, true)
+        log.error('记录ip失败次数,用于优选ip：', hostname, ip)
+      }
+      const counter = context.requestCount
+      if (counter != null) {
+        counter.count.doCount(counter.value, true)
+        log.error('记录prxoy失败次数：', counter.value)
+      }
+    }
+
     const proxyRequestPromise = async () => {
       rOptions.host = rOptions.hostname || rOptions.host || 'localhost'
       return new Promise((resolve, reject) => {
@@ -92,24 +113,19 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
           }
           proxyReq = (rOptions.protocol === 'https:' ? https : http).request(rOptions, (proxyRes) => {
             const end = new Date().getTime()
+            const cost = end - start
             if (rOptions.protocol === 'https:') {
-              log.info('代理请求返回:', url, (end - start) + 'ms')
+              log.info('代理请求返回:', url, cost + 'ms')
+            }
+            if (cost > 8000) {
+              countSlow(isDnsIntercept)
             }
             resolve(proxyRes)
           })
 
           proxyReq.on('timeout', () => {
             const end = new Date().getTime()
-            if (isDnsIntercept) {
+            countSlow(isDnsIntercept)
-              const { dns, ip, hostname } = isDnsIntercept
-              dns.count(hostname, ip, true)
-              log.error('记录ip失败次数,用于优选ip：', hostname, ip)
-            }
-            const counter = context.requestCount
-            if (counter != null) {
-              counter.count.doCount(counter.value, true)
-              log.error('记录prxoy失败次数：', counter.value)
-            }
             log.error('代理请求超时', rOptions.protocol, rOptions.hostname, rOptions.path, (end - start) + 'ms')
             proxyReq.end()
             proxyReq.destroy()
@@ -120,16 +136,7 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
 
           proxyReq.on('error', (e) => {
             const end = new Date().getTime()
-            if (isDnsIntercept) {
+            countSlow(isDnsIntercept)
-              const { dns, ip, hostname } = isDnsIntercept
-              dns.count(hostname, ip, true)
-              log.error('记录ip失败次数,用于优选ip：', hostname, ip)
-            }
-            const counter = context.requestCount
-            if (counter != null) {
-              counter.count.doCount(counter.value, true)
-              log.error('记录prxoy失败次数：', counter.value)
-            }
             log.error('代理请求错误', e.code, e.message, rOptions.hostname, rOptions.path, (end - start) + 'ms')
             reject(e)
           })
@@ -140,16 +147,7 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
             log.error('代理请求被取消', rOptions.hostname, rOptions.path, cost + 'ms')
 
             if (cost > 8000) {
-              if (isDnsIntercept) {
+              countSlow(isDnsIntercept)
-                const { dns, ip, hostname } = isDnsIntercept
-                dns.count(hostname, ip, true)
-                log.error('记录ip失败次数,用于优选ip：', hostname, ip)
-              }
-              const counter = context.requestCount
-              if (counter != null) {
-                counter.count.doCount(counter.value, true)
-                log.error('记录prxoy失败次数：', counter.value)
-              }
             }
 
             if (res.writableEnded) {
@@ -193,11 +191,7 @@ module.exports = function createRequestHandler (createIntercepts, externalProxy,
       //   // console.log('BODY: ')
       // })
       proxyRes.on('error', (error) => {
-        const counter = context.requestCount
+        countSlow()
-        if (counter != null) {
-          counter.count.doCount(counter.value, true)
-          log.error('记录prxoy失败次数：', counter.value)
-        }
         log.error('proxy res error', error)
       })
 

packages/mitmproxy/src/lib/utils/logger.js

@@ -1,11 +0,0 @@
-const debug = require('debug')
-
-module.exports = function getLogger (name) {
-  return {
-    debug: debug(`dev-sidecar:${name}:debug`),
-    info: debug(`dev-sidecar:${name}:info`),
-    error: debug(`dev-sidecar:${name}:error`)
-  }
-}
-
-debug.enable('dev-sidecar:*')

packages/mitmproxy/src/utils/util.log.js

@@ -1,7 +1,7 @@
 const log4js = require('log4js')
 const proxyConfig = require('../lib/proxy/common/config')
 log4js.configure({
-  appenders: { std: { type: 'stdout' }, file: { type: 'file', pattern: '.yyyy-MM-dd', daysToKeep: 3, filename: proxyConfig.getDefaultCABasePath() + '/logs/server.log' } },
+  appenders: { std: { type: 'stdout' }, file: { type: 'file', pattern: 'yyyy-MM-dd', daysToKeep: 3, filename: proxyConfig.getDefaultCABasePath() + '/logs/server.log' } },
   categories: { default: { appenders: ['file', 'std'], level: 'info' } }
 })
 const logger = log4js.getLogger('server')

packages/mitmproxy/test/dnsTest.mjs

@@ -20,4 +20,11 @@ const dnsProviders = dns.initDNS({
 // dnsProviders.usa.lookup(hostname)
 
 const hostname1 = 'api.github.com'
-dnsProviders.aliyun.lookup(hostname1)
+dnsProviders.usa.lookup(hostname1)
+
+const hostname2 = 'hk.docmirror.cn'
+dnsProviders.usa.lookup(hostname2)
+const hostname3 = 'github.docmirror.cn'
+dnsProviders.usa.lookup(hostname3)
+const hostname4 = 'gh.docmirror.top'
+dnsProviders.usa.lookup(hostname4)

packages/mitmproxy/test/sha256Test.js

@@ -0,0 +1,4 @@
+const CryptoJs = require('crypto-js')
+const ret = CryptoJs.SHA256('111111111111')
+console.log(ret.toString(CryptoJs.enc.Base64))
+console.log(1 / 2)

packages/mitmproxy/yarn.lock

@@ -2159,6 +2159,11 @@ crypto-browserify@^3.11.0:
     randombytes "^2.0.0"
     randomfill "^1.0.3"
 
+crypto-js@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/crypto-js/-/crypto-js-4.0.0.tgz#2904ab2677a9d042856a2ea2ef80de92e4a36dcc"
+  integrity sha512-bzHZN8Pn+gS7DQA6n+iUmBfl0hO5DJq++QP3U6uTucDtk/0iGpXd/Gg7CGR0p8tJhofJyaKoWBuJI4eAO00BBg==
+
 cyclist@^1.0.1:
   version "1.0.1"
   resolved "https://registry.npm.taobao.org/cyclist/download/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9"