diff --git a/packages/core/src/config.js b/packages/core/src/config.js index 00f66f1..c544f47 100644 --- a/packages/core/src/config.js +++ b/packages/core/src/config.js @@ -17,12 +17,12 @@ function get () { const getDefaultConfigBasePath = function () { return get().server.setting.userBasePath } -function _getRemoteSavePath (prefix = '', version = '') { +function _getRemoteSavePath (prefix = '') { const dir = getDefaultConfigBasePath() if (!fs.existsSync(dir)) { fs.mkdirSync(dir) } - return path.join(dir, prefix + 'remote_config.json' + version) + return path.join(dir, prefix + 'remote_config.json5') } function _getConfigPath () { const dir = getDefaultConfigBasePath() @@ -64,7 +64,7 @@ const configApi = { return } if (response && response.statusCode === 200) { - const originalRemoteSavePath = _getRemoteSavePath('original_', '5') + const originalRemoteSavePath = _getRemoteSavePath('original_') fs.writeFileSync(originalRemoteSavePath, body) log.info('保存原来的远程配置文件成功:', originalRemoteSavePath) diff --git a/packages/core/src/config/index.js b/packages/core/src/config/index.js index a4b2d44..fd6f804 100644 --- a/packages/core/src/config/index.js +++ b/packages/core/src/config/index.js @@ -52,14 +52,14 @@ module.exports = { intercepts: { 'github.com': { '/.*/.*/releases/download/': { - redirect: 'download.fastgit.org', + redirect: 'gh.api.99988866.xyz/https://github.com', desc: 'release文件加速下载跳转地址' }, '/.*/.*/archive/': { - redirect: 'download.fastgit.org' + redirect: 'gh.api.99988866.xyz/https://github.com' }, '/.*/.*/blame/': { - redirect: 'hub.fastgit.org' + redirect: 'gh.api.99988866.xyz/https://github.com' }, '^/[^/]+/[^/]+(/releases(/.*)?)?$': { script: [ @@ -69,15 +69,17 @@ module.exports = { }, '/.*': { proxy: 'github.com', - // proxy: 'gh.docmirror.top/_proxy', desc: '目前禁掉sni就可以直接访问,如果后续github.com的ip被封锁,只能再走proxy模式', sni: 'baidu.com' + }, + '/fluidicon.png': { + cacheDays: 365, + desc: 'Github那只猫的图片,缓存1年' + }, + '^/[^/]+/[^/]+/pull/\\d+/open_with_menu.*$': { + cacheDays: 1, + desc: 'PR详情页:标题右边那个Code按钮的HTML代理请求地址,感觉上应该可以缓存。暂时先设置为缓存1天' } - // '/.*/.*/raw11/': { - // replace: '(.+)\\/raw\\/(.+)', - // proxy: 'raw.fastgit.org$1/$2', - // sni: 'baidu.com' - // } }, 'github-releases.githubusercontent.com': { '.*': { @@ -95,6 +97,10 @@ module.exports = { '.*': { proxy: 'camo.githubusercontent.com', sni: 'baidu.com' + }, + '^[a-zA-Z0-9/]+(\\?.*)?$': { + cacheDays: 365, + desc: '图片,缓存1年' } }, 'collector.github.com': { @@ -116,12 +122,30 @@ module.exports = { '.*': { proxy: 'user-images.githubusercontent.com', sni: 'baidu.com' + }, + '^/.*\\.png(\\?.*)?$': { + cacheDays: 365, + desc: '用户在PR或issue等内容中上传的图片,缓存1年。注:每张图片都有唯一的ID,不会重复,可以安心缓存' + } + }, + 'private-user-images.githubusercontent.com': { + '.*': { + proxy: 'private-user-images.githubusercontent.com', + sni: 'baidu.com' + }, + '^/.*\\.png(\\?.*)?$': { + cacheDays: 365, + desc: '用户在PR或issue等内容中上传的图片,缓存1年。注:每张图片都有唯一的ID,不会重复,可以安心缓存' } }, 'avatars.githubusercontent.com': { '.*': { proxy: 'avatars.githubusercontent.com', sni: 'baidu.com' + }, + '^/u/\\d+.*$': { + cacheDays: 365, + desc: '用户头像,缓存1年' } }, 'api.github.com': { diff --git a/packages/core/src/modules/server/index.js b/packages/core/src/modules/server/index.js index d514ef6..18cc11e 100644 --- a/packages/core/src/modules/server/index.js +++ b/packages/core/src/modules/server/index.js @@ -67,6 +67,11 @@ const serverApi = { } } serverConfig.plugin = allConfig.plugin + + if (allConfig.proxy && allConfig.proxy.enabled) { + serverConfig.proxy = allConfig.proxy + } + // fireStatus('ing') // 启动中 const basePath = serverConfig.setting.userBasePath const runningConfigPath = path.join(basePath, '/running.json') @@ -125,13 +130,13 @@ const serverApi = { // fireStatus('ing')// 关闭中 server.close((err) => { if (err) { - log.info('close error', err, ',', err.code, ',', err.message, ',', err.errno) + log.warn('close error', err, ',', err.code, ',', err.message, ',', err.errno) if (err.code === 'ERR_SERVER_NOT_RUNNING') { log.info('代理服务关闭成功') resolve() return } - log.info('代理服务关闭失败', err) + log.warn('代理服务关闭失败', err) reject(err) } else { log.info('代理服务关闭成功') diff --git a/packages/mitmproxy/src/lib/interceptor/impl/abort.js b/packages/mitmproxy/src/lib/interceptor/impl/req/abort.js similarity index 100% rename from packages/mitmproxy/src/lib/interceptor/impl/abort.js rename to packages/mitmproxy/src/lib/interceptor/impl/req/abort.js diff --git a/packages/mitmproxy/src/lib/interceptor/impl/req/cacheReq.js b/packages/mitmproxy/src/lib/interceptor/impl/req/cacheReq.js new file mode 100644 index 0000000..afc7db9 --- /dev/null +++ b/packages/mitmproxy/src/lib/interceptor/impl/req/cacheReq.js @@ -0,0 +1,103 @@ +function getMaxAge (interceptOpt) { + // 秒 + if (interceptOpt.cacheSeconds > 0 || interceptOpt.cacheMaxAge > 0 || interceptOpt.cache > 0) { + return interceptOpt.cacheSeconds || interceptOpt.cacheMaxAge || interceptOpt.cache + } + // 分钟 + if (interceptOpt.cacheMinutes > 0) { + return interceptOpt.cacheMinutes * 60 // 60:1分钟 + } + // 小时 + if (interceptOpt.cacheHours > 0) { + return interceptOpt.cacheHours * 3600 // 60 * 60 一小时 + } + // 天 + if (interceptOpt.cacheDays > 0) { + return interceptOpt.cacheDays * 86400 // 60 * 60 * 24 一天 + } + // 星期 + if (interceptOpt.cacheWeeks > 0) { + return interceptOpt.cacheWeeks * 604800 // 60 * 60 * 24 * 7 一周 + } + // 月 + if (interceptOpt.cacheMonths > 0) { + return interceptOpt.cacheMonths * 2592000 // 60 * 60 * 24 * 30 一个月 + } + // 年 + if (interceptOpt.cacheYears > 0) { + return interceptOpt.cacheYears * 31536000 // 60 * 60 * 24 * 365 一年 + } + + return null +} + +// 获取 lastModifiedTime 的方法 +function getLastModifiedTimeFromIfModifiedSince (rOptions, log) { + // 获取 If-Modified-Since 和 If-None-Match 用于判断是否命中缓存 + const lastModified = rOptions.headers['if-modified-since'] + if (lastModified == null || lastModified.length === 0) { + return null // 没有lastModified,返回null + } + + try { + // 尝试解析 lastModified,并获取time + return new Date(lastModified).getTime() + } catch (e) { + // 为数字时,直接返回 + if (/\\d+/g.test(lastModified)) { + return lastModified - 0 + } + + log.warn(`cache intercept: 解析 if-modified-since 失败: '${lastModified}', error:`, e) + } + + return null +} + +module.exports = { + requestIntercept (context, interceptOpt, req, res, ssl, next) { + const { rOptions, log } = context + + if (rOptions.method !== 'GET') { + return // 非GET请求,不拦截 + } + + // 获取 Cache-Control 用于判断是否禁用缓存 + const cacheControl = rOptions.headers['cache-control'] + if (cacheControl && (cacheControl.indexOf('no-cache') >= 0 || cacheControl.indexOf('no-store') >= 0)) { + return // 禁用缓存,跳过当前拦截器 + } + // 获取 Pragma 用于判断是否禁用缓存 + const pragma = rOptions.headers.pragma + if (pragma && (pragma.indexOf('no-cache') >= 0 || pragma.indexOf('no-store') >= 0)) { + return // 禁用缓存,跳过当前拦截 + } + + // 最近编辑时间 + const lastModifiedTime = getLastModifiedTimeFromIfModifiedSince(rOptions, log) + if (lastModifiedTime == null) { + return // 没有 lastModified,不拦截 + } + + // 获取maxAge配置 + const maxAge = getMaxAge(interceptOpt) + // 判断缓存是否已过期 + const passTime = Date.now() - lastModifiedTime + if (passTime > maxAge * 1000) { + return // 缓存已过期,不拦截 + } + + // 缓存未过期,直接拦截请求并响应304 + res.writeHead(304, { + 'Dev-Sidecar-Interceptor': 'cacheReq' + }) + res.end() + + return true + }, + is (interceptOpt) { + const maxAge = getMaxAge(interceptOpt) + return maxAge != null && maxAge > 0 + }, + getMaxAge +} diff --git a/packages/mitmproxy/src/lib/interceptor/impl/proxy.js b/packages/mitmproxy/src/lib/interceptor/impl/req/proxy.js similarity index 100% rename from packages/mitmproxy/src/lib/interceptor/impl/proxy.js rename to packages/mitmproxy/src/lib/interceptor/impl/req/proxy.js diff --git a/packages/mitmproxy/src/lib/interceptor/impl/redirect.js b/packages/mitmproxy/src/lib/interceptor/impl/req/redirect.js similarity index 100% rename from packages/mitmproxy/src/lib/interceptor/impl/redirect.js rename to packages/mitmproxy/src/lib/interceptor/impl/req/redirect.js diff --git a/packages/mitmproxy/src/lib/interceptor/impl/sni.js b/packages/mitmproxy/src/lib/interceptor/impl/req/sni.js similarity index 100% rename from packages/mitmproxy/src/lib/interceptor/impl/sni.js rename to packages/mitmproxy/src/lib/interceptor/impl/req/sni.js diff --git a/packages/mitmproxy/src/lib/interceptor/impl/success.js b/packages/mitmproxy/src/lib/interceptor/impl/req/success.js similarity index 100% rename from packages/mitmproxy/src/lib/interceptor/impl/success.js rename to packages/mitmproxy/src/lib/interceptor/impl/req/success.js diff --git a/packages/mitmproxy/src/lib/interceptor/impl/res/cacheRes.js b/packages/mitmproxy/src/lib/interceptor/impl/res/cacheRes.js new file mode 100644 index 0000000..7cebb65 --- /dev/null +++ b/packages/mitmproxy/src/lib/interceptor/impl/res/cacheRes.js @@ -0,0 +1,89 @@ +const cacheReq = require('../req/cacheReq') + +module.exports = { + responseIntercept (context, interceptOpt, req, res, proxyReq, proxyRes, ssl, next) { + const { rOptions } = context + + // 只有GET请求,且响应码为2xx时才进行缓存 + if (rOptions.method !== 'GET' || proxyRes.statusCode < 200 || proxyRes.statusCode >= 300) { + return + } + + // 获取maxAge配置 + let maxAge = cacheReq.getMaxAge(interceptOpt) + // public 或 private + const cacheControlType = (interceptOpt.cacheControlType || 'public') + ', ' + // immutable属性 + const cacheImmutable = interceptOpt.cacheImmutable !== false ? ', immutable' : '' + + // 获取原响应头中的cache-control、last-modified、expires + const originalHeaders = { + cacheControl: null, + lastModified: null, + expires: null, + etag: null + } + for (let i = 0; i < proxyRes.rawHeaders.length; i += 2) { + // 尝试修改rawHeaders中的cache-control、last-modified、expires + if (proxyRes.rawHeaders[i].toLowerCase() === 'cache-control') { + originalHeaders.cacheControl = { value: proxyRes.rawHeaders[i + 1], valueIndex: i + 1 } + } else if (proxyRes.rawHeaders[i].toLowerCase() === 'last-modified') { + originalHeaders.lastModified = { value: proxyRes.rawHeaders[i + 1], valueIndex: i + 1 } + } else if (proxyRes.rawHeaders[i].toLowerCase() === 'expires') { + originalHeaders.expires = { value: proxyRes.rawHeaders[i + 1], valueIndex: i + 1 } + } else if (proxyRes.rawHeaders[i].toLowerCase() === 'etag') { + originalHeaders.etag = { value: proxyRes.rawHeaders[i + 1], valueIndex: i + 1 } + } + + // 如果已经设置了cache-control、last-modified、expires,则直接break + if (originalHeaders.cacheControl && originalHeaders.lastModified && originalHeaders.expires && originalHeaders.etag) { + break + } + } + + // 判断原max-age是否大于新max-age + if (originalHeaders.cacheControl) { + const maxAgeMatch = originalHeaders.cacheControl.value.match(/max-age=(\d+)/) + if (maxAgeMatch && maxAgeMatch[1] > maxAge) { + if (interceptOpt.cacheImmutable !== false && originalHeaders.cacheControl.value.indexOf('immutable') < 0) { + maxAge = maxAgeMatch[1] + } else { + return + } + } + } + + // 替换用的头信息 + const now = new Date() + const replaceHeaders = { + cacheControl: `${cacheControlType}max-age=${maxAge + 1}${cacheImmutable}`, + lastModified: now.toUTCString(), + expires: new Date(now.getTime() + maxAge * 1000).toUTCString() + } + // 开始替换 + // 替换cache-control + if (originalHeaders.cacheControl) { + proxyRes.rawHeaders[originalHeaders.cacheControl.valueIndex] = replaceHeaders.cacheControl + } else { + res.setHeader('Cache-Control', replaceHeaders.cacheControl) + } + // 替换last-modified + if (originalHeaders.lastModified) { + proxyRes.rawHeaders[originalHeaders.lastModified.valueIndex] = replaceHeaders.lastModified + } else { + res.setHeader('Last-Modified', replaceHeaders.lastModified) + } + // 替换expires + if (originalHeaders.expires) { + proxyRes.rawHeaders[originalHeaders.expires.valueIndex] = replaceHeaders.expires + } else { + res.setHeader('Expires', replaceHeaders.expires) + } + + res.setHeader('Dev-Sidecar-Cache-Response-Interceptor', 'cacheRes:maxAge=' + maxAge) + }, + is (interceptOpt) { + const maxAge = cacheReq.getMaxAge(interceptOpt) + return maxAge != null && maxAge > 0 + } +} diff --git a/packages/mitmproxy/src/lib/interceptor/impl/script.js b/packages/mitmproxy/src/lib/interceptor/impl/res/script.js similarity index 96% rename from packages/mitmproxy/src/lib/interceptor/impl/script.js rename to packages/mitmproxy/src/lib/interceptor/impl/res/script.js index 6d31690..b8828ea 100644 --- a/packages/mitmproxy/src/lib/interceptor/impl/script.js +++ b/packages/mitmproxy/src/lib/interceptor/impl/res/script.js @@ -1,5 +1,5 @@ const contextPath = '/____ds_script____/' -const monkey = require('../../monkey') +const monkey = require('../../../monkey') const CryptoJs = require('crypto-js') function getScript (key, script) { const scriptUrl = contextPath + key diff --git a/packages/mitmproxy/src/lib/interceptor/index.js b/packages/mitmproxy/src/lib/interceptor/index.js index 3f8b461..31de30d 100644 --- a/packages/mitmproxy/src/lib/interceptor/index.js +++ b/packages/mitmproxy/src/lib/interceptor/index.js @@ -1,9 +1,25 @@ -const proxy = require('./impl/proxy') -const redirect = require('./impl/redirect') -const abort = require('./impl/abort') -const success = require('./impl/success') -const script = require('./impl/script') -const sni = require('./impl/sni') -const modules = [proxy, redirect, abort, script, success, sni] +// request interceptor impls +const success = require('./impl/req/success') +const redirect = require('./impl/req/redirect') +const abort = require('./impl/req/abort') + +const cacheReq = require('./impl/req/cacheReq') + +const proxy = require('./impl/req/proxy') +const sni = require('./impl/req/sni') + +// response interceptor impls +const cacheRes = require('./impl/res/cacheRes') +const script = require('./impl/res/script') + +const modules = [ + // request interceptor impls + success, redirect, abort, + cacheReq, + proxy, sni, + + // response interceptor impls + cacheRes, script +] module.exports = modules