feature: 新增 `pac.txt`（即 `GFW` 列表）自动更新功能 (#352)

2024-09-11 17:00:02 +08:00 · 2024-09-11 17:00:02 +08:00 · 2d8243692c
parent 116e7c778d
commit 2d8243692c
5 changed files with 149 additions and 16 deletions
--- a/packages/core/src/modules/plugin/overwall/config.js
+++ b/packages/core/src/modules/plugin/overwall/config.js
@ -42,7 +42,8 @@ module.exports = {
  },
  pac: {
    enabled: true,
-    // update: [ 'https://gitlab.com/gfwlist/gfwlist/raw/master/gfwlist.txt' ],
+    autoUpdate: true,
+    pacFileUpdateUrl: 'https://gitlab.com/gfwlist/gfwlist/raw/master/gfwlist.txt',
    pacFileAbsolutePath: null, // 自定义 pac.txt 文件位置，可以是本地文件路径
    pacFilePath: './extra/pac/pac.txt' // 内置 pac.txt 文件路径
  }
--- a/packages/gui/src/view/pages/plugin/node.vue
+++ b/packages/gui/src/view/pages/plugin/node.vue
@ -62,10 +62,10 @@
          <div class="form-help">某些库需要自己设置镜像变量，才能下载，比如：electron</div>
          <a-row :gutter="10" style="margin-top: 5px" v-for="(item,index) of npmVariables" :key='index'>
            <a-col :span="10">
-              <a-input v-model="item.key" :title="item.key" :readonly="true"></a-input>
+              <a-input v-model="item.key" :title="item.key" readOnly></a-input>
            </a-col>
            <a-col :span="10">
-              <a-input v-model="item.value" :title="item.value" :readonly="true"></a-input>
+              <a-input v-model="item.value" :title="item.value" readOnly></a-input>
            </a-col>
            <a-col :span="4">
              <a-icon v-if="item.exists && item.hadSet" title="已设置" style="color:green" type="check"/>
--- a/packages/gui/src/view/pages/plugin/overwall.vue
+++ b/packages/gui/src/view/pages/plugin/overwall.vue
@ -19,12 +19,29 @@
            声明：仅供技术学习与探讨！
          </div>
        </a-form-item>
+        <hr/>
        <a-form-item label="PAC" :label-col="labelCol" :wrapper-col="wrapperCol">
          <a-checkbox v-model="config.plugin.overwall.pac.enabled">
            启用PAC
          </a-checkbox>
          <div class="form-help">PAC内收录了常见的被封杀的域名，当里面某些域名你不想被拦截时，可以关闭PAC</div>
        </a-form-item>
+        <a-form-item label="自动更新PAC" :label-col="labelCol" :wrapper-col="wrapperCol">
+          <a-checkbox v-model="config.plugin.overwall.pac.autoUpdate">
+            是否自动更新PAC
+          </a-checkbox>
+          <div class="form-help">
+            开启自动更新后，启动代理服务时，将会异步从下面的远程地址下载PAC文件到本地。<br/>
+            注：只要下载成功后，即使关闭自动更新功能，也会优先读取最近下载的PAC文件！
+          </div>
+        </a-form-item>
+        <a-form-item label="远程PAC文件地址" :label-col="labelCol" :wrapper-col="wrapperCol">
+          <a-input v-model="config.plugin.overwall.pac.pacFileUpdateUrl"/>
+          <div class="form-help">
+            远程PAC文件内容可以是 base64 编码格式，也可以是未经过编码的
+          </div>
+        </a-form-item>
+        <hr/>
        <a-form-item label="自定义域名" :label-col="labelCol" :wrapper-col="wrapperCol">
          <div>
            <a-row :gutter="10" style="">
--- a/packages/mitmproxy/src/lib/proxy/middleware/overwall.js
+++ b/packages/mitmproxy/src/lib/proxy/middleware/overwall.js
@ -1,8 +1,12 @@
 const url = require('url')
+const request = require('request')
 const lodash = require('lodash')
 const pac = require('./source/pac')
 const matchUtil = require('../../../utils/util.match')
 const log = require('../../../utils/util.log')
+const path = require('path')
+const fs = require('fs')
+const { Buffer } = require('buffer')
 let pacClient = null

 function matched (hostname, overWallTargetMap) {
@ -23,7 +27,93 @@ function matched (hostname, overWallTargetMap) {
  }
 }

-module.exports = function createOverWallIntercept (overWallConfig) {
+function getUserBasePath () {
+  const userHome = process.env.USERPROFILE || process.env.HOME || '/'
+  return path.resolve(userHome, './.dev-sidecar')
+}
+
+// 下载的 pac.txt 文件保存路径
+function getTmpPacFilePath () {
+  return path.join(getUserBasePath(), '/pac.txt')
+}
+
+function loadPacLastModifiedTime (pacTxt) {
+  const matched = pacTxt.match(/(?<=! Last Modified: )[^\n]+/g)
+  if (matched && matched.length > 0) {
+    try {
+      return new Date(matched[0])
+    } catch (ignore) {
+      return null
+    }
+  }
+}
+
+// 保存 pac 内容到 `~/pac.txt` 文件中
+function savePacFile (pacTxt) {
+  const pacFilePath = getTmpPacFilePath()
+  fs.writeFileSync(pacFilePath, pacTxt)
+  log.info('保存 pac.txt 文件成功:', pacFilePath)
+
+  // 尝试解析和修改 pac.txt 文件时间
+  const lastModifiedTime = loadPacLastModifiedTime(pacTxt)
+  if (lastModifiedTime) {
+    fs.stat(pacFilePath, (err, stats) => {
+      if (err) {
+        log.error('修改 pac.txt 文件时间失败:', err)
+        return
+      }
+
+      // 修改文件的访问时间和修改时间为当前时间
+      fs.utimes(pacFilePath, lastModifiedTime, lastModifiedTime, (utimesErr) => {
+        if (utimesErr) {
+          log.error('修改 pac.txt 文件时间失败:', utimesErr)
+        } else {
+          log.info(`${pacFilePath} 文件时间已被修改其最近更新时间 '${lastModifiedTime}'`)
+        }
+      })
+    })
+  }
+
+  return pacFilePath
+}
+
+// 异步下载 pac.txt ，避免影响代理服务的启动速度
+async function downloadPacAsync (pacConfig) {
+  const remotePacFileUrl = pacConfig.pacFileUpdateUrl
+  log.info('开始下载远程 pac.txt 文件:', remotePacFileUrl)
+  request(remotePacFileUrl, (error, response, body) => {
+    if (error) {
+      log.error('下载远程 pac.txt 文件失败, error:', error, ', response:', response, ', body:', body)
+      return
+    }
+    if (response && response.statusCode === 200) {
+      if (body == null || body.length < 100) {
+        log.warn('下载远程 pac.txt 文件成功，但内容为空或内容太短，判断为无效的 pax.txt 文件:', remotePacFileUrl, ', body:', body)
+        return
+      } else {
+        log.info('下载远程 pac.txt 文件成功:', remotePacFileUrl)
+      }
+
+      // 尝试解析Base64（注：https://gitlab.com/gfwlist/gfwlist/raw/master/gfwlist.txt 下载下来的是Base64格式）
+      let pacTxt = body
+      try {
+        pacTxt = Buffer.from(pacTxt, 'base64').toString('utf8')
+      } catch (e) {
+        if (pacTxt.indexOf('||') < 0) { // TODO: 待优化，需要判断下载的 pac.txt 文件内容是否正确，目前暂时先简单判断一下
+          log.error(`远程 pac.txt 文件内容即不是base64格式，也不是要求的格式，url: ${remotePacFileUrl}，body: ${body}`)
+          return
+        }
+      }
+
+      // 保存到本地
+      savePacFile(pacTxt)
+    } else {
+      log.error('下载远程 pac.txt 文件失败, response:', response, ', body:', body)
+    }
+  })
+}
+
+function createOverwallMiddleware (overWallConfig) {
  if (!overWallConfig || overWallConfig.enabled !== true) {
    return null
  }
@ -109,3 +199,9 @@ module.exports = function createOverWallIntercept (overWallConfig) {
    }
  }
 }
+
+module.exports = {
+  getTmpPacFilePath,
+  downloadPacAsync,
+  createOverwallMiddleware
+}
--- a/packages/mitmproxy/src/options.js
+++ b/packages/mitmproxy/src/options.js
@ -3,9 +3,10 @@ const dnsUtil = require('./lib/dns')
 const log = require('./utils/util.log')
 const matchUtil = require('./utils/util.match')
 const path = require('path')
+const fs = require('fs')
 const scriptInterceptor = require('./lib/interceptor/impl/res/script')

-const createOverwallMiddleware = require('./lib/proxy/middleware/overwall')
+const { getTmpPacFilePath, downloadPacAsync, createOverwallMiddleware } = require('./lib/proxy/middleware/overwall')

 // 处理拦截配置
 function buildIntercepts (intercepts) {
@ -15,12 +16,11 @@ function buildIntercepts (intercepts) {
  return intercepts
 }

-module.exports = (config) => {
-  const intercepts = matchUtil.domainMapRegexply(buildIntercepts(config.intercepts))
-  const whiteList = matchUtil.domainMapRegexply(config.whiteList)
+module.exports = (serverConfig) => {
+  const intercepts = matchUtil.domainMapRegexply(buildIntercepts(serverConfig.intercepts))
+  const whiteList = matchUtil.domainMapRegexply(serverConfig.whiteList)

-  const dnsMapping = config.dns.mapping
-  const serverConfig = config
+  const dnsMapping = serverConfig.dns.mapping
  const setting = serverConfig.setting

  if (!setting.script.dirAbsolutePath) {
@ -30,16 +30,35 @@ module.exports = (config) => {
    setting.verifySsl = true
  }

-  const overwallConfig = serverConfig.plugin.overwall
-  if (!overwallConfig.pac.pacFileAbsolutePath) {
-    overwallConfig.pac.pacFileAbsolutePath = path.join(setting.rootDir, overwallConfig.pac.pacFilePath)
+  const overWallConfig = serverConfig.plugin.overwall
+  if (overWallConfig.pac && overWallConfig.pac.enabled) {
+    const pacConfig = overWallConfig.pac
+
+    // 自动更新 pac.txt
+    if (!pacConfig.pacFileAbsolutePath && pacConfig.autoUpdate) {
+      // 异步下载远程 pac.txt 文件，并保存到本地；下载成功后，需要重启代理服务才会生效
+      downloadPacAsync(pacConfig)
+    }
+
+    // 优先使用本地已下载的 pac.txt 文件
+    if (!pacConfig.pacFileAbsolutePath && fs.existsSync(getTmpPacFilePath())) {
+      pacConfig.pacFileAbsolutePath = getTmpPacFilePath()
+      log.info('读取已下载的 pac.txt 文件:', pacConfig.pacFileAbsolutePath)
+    }
+
+    if (!pacConfig.pacFileAbsolutePath) {
+      pacConfig.pacFileAbsolutePath = path.join(setting.rootDir, pacConfig.pacFilePath)
+      if (pacConfig.autoUpdate) {
+        log.warn('远程 pac.txt 文件下载失败或还在下载中，现使用内置 pac.txt 文件:', pacConfig.pacFileAbsolutePath)
+      }
+    }
  }

  // 插件列表
  const middlewares = []

  // 梯子插件：如果启用了，则添加到插件列表中
-  const overwallMiddleware = createOverwallMiddleware(overwallConfig)
+  const overwallMiddleware = createOverwallMiddleware(overWallConfig)
  if (overwallMiddleware) {
    middlewares.push(overwallMiddleware)
  }
@ -48,9 +67,9 @@ module.exports = (config) => {
    host: serverConfig.host,
    port: serverConfig.port,
    dnsConfig: {
-      providers: dnsUtil.initDNS(serverConfig.dns.providers, matchUtil.domainMapRegexply(config.preSetIpList)),
+      providers: dnsUtil.initDNS(serverConfig.dns.providers, matchUtil.domainMapRegexply(serverConfig.preSetIpList)),
      mapping: matchUtil.domainMapRegexply(dnsMapping),
-      speedTest: config.dns.speedTest
+      speedTest: serverConfig.dns.speedTest
    },
    setting,
    sniConfig: serverConfig.sniList,