diff --git a/.github/workflows/gh-proxy.yml b/.github/workflows/gh-proxy.yml new file mode 100644 index 0000000..cfaa187 --- /dev/null +++ b/.github/workflows/gh-proxy.yml @@ -0,0 +1,42 @@ +name: "gh-proxy docker build" + +env: + PROJECT: gh-proxy + +on: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + env: + ACTIONS_ALLOW_UNSECURE_COMMANDS: true + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set tag + id: tag + run: | + TAG=$(cat ${{ env.PROJECT }}/Dockerfile | awk 'NR==4 {print $3}') + echo "::set-env name=TAG::$TAG" + - name: Docker Hub login + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: | + echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin + - name: Set up Docker Buildx + id: buildx + uses: crazy-max/ghaction-docker-buildx@v1 + with: + buildx-version: latest + - name: Build Dockerfile + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + run: | + docker buildx build \ + --platform=linux/amd64,linux/arm64 \ + --output "type=image,push=true" \ + --file ${{ env.PROJECT }}/Dockerfile ./${{ env.PROJECT }} \ + --tag $(echo "${DOCKER_USERNAME}" | tr '[:upper:]' '[:lower:]')/${{ env.PROJECT }}:latest \ + --tag $(echo "${DOCKER_USERNAME}" | tr '[:upper:]' '[:lower:]')/${{ env.PROJECT }}:${TAG} diff --git a/gh-proxy/Dockerfile b/gh-proxy/Dockerfile new file mode 100644 index 0000000..5117726 --- /dev/null +++ b/gh-proxy/Dockerfile @@ -0,0 +1,28 @@ +FROM stilleshan/uwsgi-nginx:python3.7 +LABEL maintainer="Sebastian Ramirez " + +ENV VERSION 2.1 + +RUN pip install flask requests + +COPY ./app /app +WORKDIR /app + +# Make /app/* available to be imported by Python globally to better support several use cases like Alembic migrations. +ENV PYTHONPATH=/app + +# Move the base entrypoint to reuse it +RUN mv /entrypoint.sh /uwsgi-nginx-entrypoint.sh +# Copy the entrypoint that will generate Nginx additional configs +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] + +# Run the start script provided by the parent image tiangolo/uwsgi-nginx. +# It will check for an /app/prestart.sh script (e.g. for migrations) +# And then will start Supervisor, which in turn will start Nginx and uWSGI + +EXPOSE 80 + +CMD ["/start.sh"] diff --git a/gh-proxy/LICENSE b/gh-proxy/LICENSE new file mode 100644 index 0000000..c813a85 --- /dev/null +++ b/gh-proxy/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 hunshcn + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/gh-proxy/README.md b/gh-proxy/README.md new file mode 100644 index 0000000..19fb65d --- /dev/null +++ b/gh-proxy/README.md @@ -0,0 +1,110 @@ +# gh-proxy + +## 简介 + +github release、archive以及项目文件的加速项目,支持clone,有Cloudflare Workers无服务器版本以及Python版本 + +## 演示 + +[https://gh.api.99988866.xyz/](https://gh.api.99988866.xyz/) + +演示站为公共服务,如有大规模使用需求请自行部署,演示站有点不堪重负 + +![imagea272c95887343279.png](https://img.maocdn.cn/img/2021/04/24/imagea272c95887343279.png) + +当然也欢迎[捐赠](#捐赠)以支持作者 + +## python版本和cf worker版本差异 + +- python版本支持进行文件大小限制,超过设定返回原地址 [issue #8](https://github.com/hunshcn/gh-proxy/issues/8) + +## 使用 + +直接在copy出来的url前加`https://gh.api.99988866.xyz/`即可 + +也可以直接访问,在input输入 + +***大量使用请自行部署,以上域名仅为演示使用。*** + +以下都是合法输入(仅示例,文件不存在): + +- 分支源码:https://github.com/hunshcn/project/archive/master.zip + +- release源码:https://github.com/hunshcn/project/archive/v0.1.0.tar.gz + +- release文件:https://github.com/hunshcn/project/releases/download/v0.1.0/example.zip + +- 分支文件:https://github.com/hunshcn/project/blob/master/filename + +- commit文件:https://github.com/hunshcn/project/blob/1111111111111111111111111111/filename + +- gist:https://gist.githubusercontent.com/cielpy/351557e6e465c12986419ac5a4dd2568/raw/cmd.py + +## cf worker版本部署 + +首页:https://workers.cloudflare.com + +注册,登陆,`Start building`,取一个子域名,`Create a Worker`。 + +复制 [index.js](https://cdn.jsdelivr.net/hunshcn/gh-proxy@master/index.js) 到左侧代码框,`Save and deploy`。如果正常,右侧应显示首页。 + +`index.js`默认配置下clone走github.com.cnpmjs.org,项目文件会走jsDeliver,如需走worker,修改Config变量即可 + +`ASSET_URL`是静态资源的url(实际上就是现在显示出来的那个输入框单页面) + +`PREFIX`是前缀,默认(根路径情况为"/"),如果自定义路由为example.com/gh/*,请将PREFIX改为 '/gh/',注意,少一个杠都会错! + +## Python版本部署 + +### Docker部署 + +``` +docker run -d --name="gh-proxy-py" \ + -p 0.0.0.0:80:80 \ + --restart=always \ + hunsh/gh-proxy-py:latest +``` + +第一个80是你要暴露出去的端口 + +### 直接部署 + +安装依赖(请使用python3) + +```pip install flask requests``` + +按需求修改`app/main.py`的前几项配置 + +### 注意 + +python版本的机器如果无法正常访问github.io会启动报错,请自行修改静态文件url + +workers版本默认配置下clone走github.com.cnpmjs.org,项目文件会走jsDeliver,如需走服务器,修改配置即可 + +python版本默认走服务器(2021.3.27更新) + +## Cloudflare Workers计费 + +到 `overview` 页面可参看使用情况。免费版每天有 10 万次免费请求,并且有每分钟1000次请求的限制。 + +如果不够用,可升级到 $5 的高级版本,每月可用 1000 万次请求(超出部分 $0.5/百万次请求)。 + +## Changelog + +* 2020.04.10 增加对`raw.githubusercontent.com`文件的支持 +* 2020.04.09 增加Python版本(使用Flask) +* 2020.03.23 新增了clone的支持 +* 2020.03.22 初始版本 + +## 链接 + +[我的博客](https://hunsh.net) + +## 参考 + +[jsproxy](https://github.com/EtherDream/jsproxy/) + +## 捐赠 + +![wx.png](https://img.maocdn.cn/img/2021/04/24/image.md.png) +![ali.png](https://www.helloimg.com/images/2021/04/24/BK9vmb.md.png) \ No newline at end of file diff --git a/gh-proxy/app/main.py b/gh-proxy/app/main.py new file mode 100644 index 0000000..9bab57c --- /dev/null +++ b/gh-proxy/app/main.py @@ -0,0 +1,137 @@ +# -*- coding: utf-8 -*- +import re + +import requests +from flask import Flask, Response, redirect, request +from requests.exceptions import ( + ChunkedEncodingError, + ContentDecodingError, ConnectionError, StreamConsumedError) +from requests.utils import ( + stream_decode_response_unicode, iter_slices, CaseInsensitiveDict) +from urllib3.exceptions import ( + DecodeError, ReadTimeoutError, ProtocolError) + +# config +# git使用cnpmjs镜像、分支文件使用jsDelivr镜像的开关,0为关闭,默认关闭 +jsdelivr = 0 +cnpmjs = 0 +size_limit = 1024 * 1024 * 1024 * 999 # 允许的文件大小,默认999GB,相当于无限制了 https://github.com/hunshcn/gh-proxy/issues/8 +HOST = '127.0.0.1' # 监听地址,建议监听本地然后由web服务器反代 +PORT = 80 # 监听端口 +ASSET_URL = 'https://hunshcn.github.io/gh-proxy' # 主页 + +app = Flask(__name__) +CHUNK_SIZE = 1024 * 10 +index_html = requests.get(ASSET_URL, timeout=10).text +icon_r = requests.get(ASSET_URL + '/favicon.ico', timeout=10).content +exp1 = re.compile(r'^(?:https?://)?github\.com/.+?/.+?/(?:releases|archive)/.*$') +exp2 = re.compile(r'^(?:https?://)?github\.com/.+?/.+?/(?:blob)/.*$') +exp3 = re.compile(r'^(?:https?://)?github\.com/.+?/.+?/(?:info|git-).*$') +exp4 = re.compile(r'^(?:https?://)?raw\.githubusercontent\.com/.+?/.+?/.+?/.+$') +exp5 = re.compile(r'^(?:https?://)?gist\.(?:githubusercontent|github)\.com/.+?/.+?/.+$') + +requests.sessions.default_headers = lambda: CaseInsensitiveDict() + + +@app.route('/') +def index(): + if 'q' in request.args: + return redirect('/' + request.args.get('q')) + return index_html + + +@app.route('/favicon.ico') +def icon(): + return Response(icon_r, content_type='image/vnd.microsoft.icon') + + +def iter_content(self, chunk_size=1, decode_unicode=False): + """rewrite requests function, set decode_content with False""" + + def generate(): + # Special case for urllib3. + if hasattr(self.raw, 'stream'): + try: + for chunk in self.raw.stream(chunk_size, decode_content=False): + yield chunk + except ProtocolError as e: + raise ChunkedEncodingError(e) + except DecodeError as e: + raise ContentDecodingError(e) + except ReadTimeoutError as e: + raise ConnectionError(e) + else: + # Standard file-like object. + while True: + chunk = self.raw.read(chunk_size) + if not chunk: + break + yield chunk + + self._content_consumed = True + + if self._content_consumed and isinstance(self._content, bool): + raise StreamConsumedError() + elif chunk_size is not None and not isinstance(chunk_size, int): + raise TypeError("chunk_size must be an int, it is instead a %s." % type(chunk_size)) + # simulate reading small chunks of the content + reused_chunks = iter_slices(self._content, chunk_size) + + stream_chunks = generate() + + chunks = reused_chunks if self._content_consumed else stream_chunks + + if decode_unicode: + chunks = stream_decode_response_unicode(chunks, self) + + return chunks + + +@app.route('/', methods=['GET', 'POST']) +def proxy(u): + u = u if u.startswith('http') else 'https://' + u + if u.rfind('://', 3, 9) == -1: + u = u.replace('s:/', 's://', 1) # uwsgi会将//传递为/ + if not any([i.match(u) for i in [exp1, exp2, exp3, exp4, exp5]]): + return Response('Invalid input.', status=403) + if jsdelivr and exp2.match(u): + u = u.replace('/blob/', '@', 1).replace('github.com', 'cdn.jsdelivr.net/gh', 1) + return redirect(u) + elif cnpmjs and exp3.match(u): + u = u.replace('github.com', 'github.com.cnpmjs.org', 1) + request.url.replace(request.base_url, '', 1) + return redirect(u) + elif jsdelivr and exp4.match(u): + u = re.sub(r'(\.com/.*?/.+?)/(.+?/)', r'\1@\2', u, 1) + u = u.replace('raw.githubusercontent.com', 'cdn.jsdelivr.net/gh', 1) + return redirect(u) + else: + if exp2.match(u): + u = u.replace('/blob/', '/raw/', 1) + headers = {} + r_headers = dict(request.headers) + if 'Host' in r_headers: + r_headers.pop('Host') + try: + url = u + request.url.replace(request.base_url, '', 1) + if url.startswith('https:/') and not url.startswith('https://'): + url = 'https://' + url[7:] + r = requests.request(method=request.method, url=url, data=request.data, headers=r_headers, stream=True) + headers = dict(r.headers) + + if 'Content-length' in r.headers and int(r.headers['Content-length']) > size_limit: + return redirect(u + request.url.replace(request.base_url, '', 1)) + + def generate(): + for chunk in iter_content(r, chunk_size=CHUNK_SIZE): + yield chunk + + return Response(generate(), headers=headers, status=r.status_code) + except Exception as e: + headers['content-type'] = 'text/html; charset=UTF-8' + return Response('server error ' + str(e), status=500, headers=headers) + # else: + # return Response('Illegal input', status=403, mimetype='text/html; charset=UTF-8') + + +if __name__ == '__main__': + app.run(host=HOST, port=PORT) diff --git a/gh-proxy/app/uwsgi.ini b/gh-proxy/app/uwsgi.ini new file mode 100644 index 0000000..327bbb2 --- /dev/null +++ b/gh-proxy/app/uwsgi.ini @@ -0,0 +1,3 @@ +[uwsgi] +module = main +callable = app diff --git a/gh-proxy/entrypoint.sh b/gh-proxy/entrypoint.sh new file mode 100644 index 0000000..c0f461d --- /dev/null +++ b/gh-proxy/entrypoint.sh @@ -0,0 +1,26 @@ +#! /usr/bin/env bash +set -e + +/uwsgi-nginx-entrypoint.sh + +# Get the listen port for Nginx, default to 80 +USE_LISTEN_PORT=${LISTEN_PORT:-80} + +if [ -f /app/nginx.conf ]; then + cp /app/nginx.conf /etc/nginx/nginx.conf +else + content_server='server {\n' + content_server=$content_server" listen ${USE_LISTEN_PORT};\n" + content_server=$content_server' location / {\n' + content_server=$content_server' try_files $uri @app;\n' + content_server=$content_server' }\n' + content_server=$content_server' location @app {\n' + content_server=$content_server' include uwsgi_params;\n' + content_server=$content_server' uwsgi_pass unix:///tmp/uwsgi.sock;\n' + content_server=$content_server' }\n' + content_server=$content_server'}\n' + # Save generated server /etc/nginx/conf.d/nginx.conf + printf "$content_server" > /etc/nginx/conf.d/nginx.conf +fi + +exec "$@" diff --git a/gh-proxy/index.js b/gh-proxy/index.js new file mode 100644 index 0000000..805b962 --- /dev/null +++ b/gh-proxy/index.js @@ -0,0 +1,165 @@ +'use strict' + +/** + * static files (404.html, sw.js, conf.js) + */ +const ASSET_URL = 'https://hunshcn.github.io/gh-proxy/' +// 前缀,如果自定义路由为example.com/gh/*,将PREFIX改为 '/gh/',注意,少一个杠都会错! +const PREFIX = '/' +// git使用cnpmjs镜像、分支文件使用jsDelivr镜像的开关,0为关闭,默认开启 +const Config = { + jsdelivr: 1, + cnpmjs: 1 +} + +/** @type {RequestInit} */ +const PREFLIGHT_INIT = { + status: 204, + headers: new Headers({ + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET,POST,PUT,PATCH,TRACE,DELETE,HEAD,OPTIONS', + 'access-control-max-age': '1728000', + }), +} + +/** + * @param {any} body + * @param {number} status + * @param {Object} headers + */ +function makeRes(body, status = 200, headers = {}) { + headers['access-control-allow-origin'] = '*' + return new Response(body, {status, headers}) +} + + +/** + * @param {string} urlStr + */ +function newUrl(urlStr) { + try { + return new URL(urlStr) + } catch (err) { + return null + } +} + + +addEventListener('fetch', e => { + const ret = fetchHandler(e) + .catch(err => makeRes('cfworker error:\n' + err.stack, 502)) + e.respondWith(ret) +}) + + +/** + * @param {FetchEvent} e + */ +async function fetchHandler(e) { + const req = e.request + const urlStr = req.url + const urlObj = new URL(urlStr) + let path = urlObj.searchParams.get('q') + if (path) { + return Response.redirect('https://' + urlObj.host + PREFIX + path, 301) + } + // cfworker 会把路径中的 `//` 合并成 `/` + path = urlObj.href.substr(urlObj.origin.length + PREFIX.length).replace(/^https?:\/+/, 'https://') + const exp1 = /^(?:https?:\/\/)?github\.com\/.+?\/.+?\/(?:releases|archive)\/.*$/i + const exp2 = /^(?:https?:\/\/)?github\.com\/.+?\/.+?\/(?:blob)\/.*$/i + const exp3 = /^(?:https?:\/\/)?github\.com\/.+?\/.+?\/(?:info|git-).*$/i + const exp4 = /^(?:https?:\/\/)?raw\.githubusercontent\.com\/.+?\/.+?\/.+?\/.+$/i + const exp5 = /^(?:https?:\/\/)?gist\.(?:githubusercontent|github)\.com\/.+?\/.+?\/.+$/i + if (path.search(exp1) === 0 || path.search(exp5) === 0 || !Config.cnpmjs && (path.search(exp3) === 0 || path.search(exp4) === 0)) { + return httpHandler(req, path) + } else if (path.search(exp2) === 0) { + if (Config.jsdelivr){ + const newUrl = path.replace('/blob/', '@').replace(/^(?:https?:\/\/)?github\.com/, 'https://cdn.jsdelivr.net/gh') + return Response.redirect(newUrl, 302) + }else{ + path = path.replace('/blob/', '/raw/') + return httpHandler(req, path) + } + } else if (path.search(exp3) === 0) { + const newUrl = path.replace(/^(?:https?:\/\/)?github\.com/, 'https://github.com.cnpmjs.org') + return Response.redirect(newUrl, 302) + } else if (path.search(exp4) === 0) { + const newUrl = path.replace(/(?<=com\/.+?\/.+?)\/(.+?\/)/, '@$1').replace(/^(?:https?:\/\/)?raw\.githubusercontent\.com/, 'https://cdn.jsdelivr.net/gh') + return Response.redirect(newUrl, 302) + } else { + return fetch(ASSET_URL + path) + } +} + + +/** + * @param {Request} req + * @param {string} pathname + */ +function httpHandler(req, pathname) { + const reqHdrRaw = req.headers + + // preflight + if (req.method === 'OPTIONS' && + reqHdrRaw.has('access-control-request-headers') + ) { + return new Response(null, PREFLIGHT_INIT) + } + + let rawLen = '' + + const reqHdrNew = new Headers(reqHdrRaw) + + let urlStr = pathname + if (urlStr.startsWith('github')) { + urlStr = 'https://' + urlStr + } + const urlObj = newUrl(urlStr) + + /** @type {RequestInit} */ + const reqInit = { + method: req.method, + headers: reqHdrNew, + redirect: 'follow', + body: req.body + } + return proxy(urlObj, reqInit, rawLen, 0) +} + + +/** + * + * @param {URL} urlObj + * @param {RequestInit} reqInit + */ +async function proxy(urlObj, reqInit, rawLen) { + const res = await fetch(urlObj.href, reqInit) + const resHdrOld = res.headers + const resHdrNew = new Headers(resHdrOld) + + // verify + if (rawLen) { + const newLen = resHdrOld.get('content-length') || '' + const badLen = (rawLen !== newLen) + + if (badLen) { + return makeRes(res.body, 400, { + '--error': `bad len: ${newLen}, except: ${rawLen}`, + 'access-control-expose-headers': '--error', + }) + } + } + const status = res.status + resHdrNew.set('access-control-expose-headers', '*') + resHdrNew.set('access-control-allow-origin', '*') + + resHdrNew.delete('content-security-policy') + resHdrNew.delete('content-security-policy-report-only') + resHdrNew.delete('clear-site-data') + + return new Response(res.body, { + status, + headers: resHdrNew, + }) +} +