Create gh-proxy

This commit is contained in:
Stille 2021-06-08 20:46:15 +08:00
parent d6b54061b6
commit 5eb506adbd
8 changed files with 532 additions and 0 deletions

42
.github/workflows/gh-proxy.yml vendored Normal file
View File

@ -0,0 +1,42 @@
name: "gh-proxy docker build"
env:
PROJECT: gh-proxy
on:
workflow_dispatch:
jobs:
build:
runs-on: ubuntu-latest
env:
ACTIONS_ALLOW_UNSECURE_COMMANDS: true
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Set tag
id: tag
run: |
TAG=$(cat ${{ env.PROJECT }}/Dockerfile | awk 'NR==4 {print $3}')
echo "::set-env name=TAG::$TAG"
- name: Docker Hub login
env:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
run: |
echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
- name: Set up Docker Buildx
id: buildx
uses: crazy-max/ghaction-docker-buildx@v1
with:
buildx-version: latest
- name: Build Dockerfile
env:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
run: |
docker buildx build \
--platform=linux/amd64,linux/arm64 \
--output "type=image,push=true" \
--file ${{ env.PROJECT }}/Dockerfile ./${{ env.PROJECT }} \
--tag $(echo "${DOCKER_USERNAME}" | tr '[:upper:]' '[:lower:]')/${{ env.PROJECT }}:latest \
--tag $(echo "${DOCKER_USERNAME}" | tr '[:upper:]' '[:lower:]')/${{ env.PROJECT }}:${TAG}

28
gh-proxy/Dockerfile Normal file
View File

@ -0,0 +1,28 @@
FROM stilleshan/uwsgi-nginx:python3.7
LABEL maintainer="Sebastian Ramirez <tiangolo@gmail.com>"
ENV VERSION 2.1
RUN pip install flask requests
COPY ./app /app
WORKDIR /app
# Make /app/* available to be imported by Python globally to better support several use cases like Alembic migrations.
ENV PYTHONPATH=/app
# Move the base entrypoint to reuse it
RUN mv /entrypoint.sh /uwsgi-nginx-entrypoint.sh
# Copy the entrypoint that will generate Nginx additional configs
COPY entrypoint.sh /entrypoint.sh
RUN chmod +x /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]
# Run the start script provided by the parent image tiangolo/uwsgi-nginx.
# It will check for an /app/prestart.sh script (e.g. for migrations)
# And then will start Supervisor, which in turn will start Nginx and uWSGI
EXPOSE 80
CMD ["/start.sh"]

21
gh-proxy/LICENSE Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2020 hunshcn
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

110
gh-proxy/README.md Normal file
View File

@ -0,0 +1,110 @@
# gh-proxy
## 简介
github release、archive以及项目文件的加速项目支持clone有Cloudflare Workers无服务器版本以及Python版本
## 演示
[https://gh.api.99988866.xyz/](https://gh.api.99988866.xyz/)
演示站为公共服务,如有大规模使用需求请自行部署,演示站有点不堪重负
![imagea272c95887343279.png](https://img.maocdn.cn/img/2021/04/24/imagea272c95887343279.png)
当然也欢迎[捐赠](#捐赠)以支持作者
## python版本和cf worker版本差异
- python版本支持进行文件大小限制超过设定返回原地址 [issue #8](https://github.com/hunshcn/gh-proxy/issues/8)
## 使用
直接在copy出来的url前加`https://gh.api.99988866.xyz/`即可
也可以直接访问在input输入
***大量使用请自行部署,以上域名仅为演示使用。***
以下都是合法输入(仅示例,文件不存在):
- 分支源码https://github.com/hunshcn/project/archive/master.zip
- release源码https://github.com/hunshcn/project/archive/v0.1.0.tar.gz
- release文件https://github.com/hunshcn/project/releases/download/v0.1.0/example.zip
- 分支文件https://github.com/hunshcn/project/blob/master/filename
- commit文件https://github.com/hunshcn/project/blob/1111111111111111111111111111/filename
- gisthttps://gist.githubusercontent.com/cielpy/351557e6e465c12986419ac5a4dd2568/raw/cmd.py
## cf worker版本部署
首页https://workers.cloudflare.com
注册,登陆,`Start building`,取一个子域名,`Create a Worker`。
复制 [index.js](https://cdn.jsdelivr.net/hunshcn/gh-proxy@master/index.js) 到左侧代码框,`Save and deploy`。如果正常,右侧应显示首页。
`index.js`默认配置下clone走github.com.cnpmjs.org项目文件会走jsDeliver如需走worker修改Config变量即可
`ASSET_URL`是静态资源的url实际上就是现在显示出来的那个输入框单页面
`PREFIX`是前缀,默认(根路径情况为"/"如果自定义路由为example.com/gh/*请将PREFIX改为 '/gh/',注意,少一个杠都会错!
## Python版本部署
### Docker部署
```
docker run -d --name="gh-proxy-py" \
-p 0.0.0.0:80:80 \
--restart=always \
hunsh/gh-proxy-py:latest
```
第一个80是你要暴露出去的端口
### 直接部署
安装依赖请使用python3
```pip install flask requests```
按需求修改`app/main.py`的前几项配置
### 注意
python版本的机器如果无法正常访问github.io会启动报错请自行修改静态文件url
workers版本默认配置下clone走github.com.cnpmjs.org项目文件会走jsDeliver如需走服务器修改配置即可
python版本默认走服务器2021.3.27更新)
## Cloudflare Workers计费
`overview` 页面可参看使用情况。免费版每天有 10 万次免费请求并且有每分钟1000次请求的限制。
如果不够用,可升级到 $5 的高级版本,每月可用 1000 万次请求(超出部分 $0.5/百万次请求)。
## Changelog
* 2020.04.10 增加对`raw.githubusercontent.com`文件的支持
* 2020.04.09 增加Python版本使用Flask
* 2020.03.23 新增了clone的支持
* 2020.03.22 初始版本
## 链接
[我的博客](https://hunsh.net)
## 参考
[jsproxy](https://github.com/EtherDream/jsproxy/)
## 捐赠
![wx.png](https://img.maocdn.cn/img/2021/04/24/image.md.png)
![ali.png](https://www.helloimg.com/images/2021/04/24/BK9vmb.md.png)

137
gh-proxy/app/main.py Normal file
View File

@ -0,0 +1,137 @@
# -*- coding: utf-8 -*-
import re
import requests
from flask import Flask, Response, redirect, request
from requests.exceptions import (
ChunkedEncodingError,
ContentDecodingError, ConnectionError, StreamConsumedError)
from requests.utils import (
stream_decode_response_unicode, iter_slices, CaseInsensitiveDict)
from urllib3.exceptions import (
DecodeError, ReadTimeoutError, ProtocolError)
# config
# git使用cnpmjs镜像、分支文件使用jsDelivr镜像的开关0为关闭默认关闭
jsdelivr = 0
cnpmjs = 0
size_limit = 1024 * 1024 * 1024 * 999 # 允许的文件大小默认999GB相当于无限制了 https://github.com/hunshcn/gh-proxy/issues/8
HOST = '127.0.0.1' # 监听地址建议监听本地然后由web服务器反代
PORT = 80 # 监听端口
ASSET_URL = 'https://hunshcn.github.io/gh-proxy' # 主页
app = Flask(__name__)
CHUNK_SIZE = 1024 * 10
index_html = requests.get(ASSET_URL, timeout=10).text
icon_r = requests.get(ASSET_URL + '/favicon.ico', timeout=10).content
exp1 = re.compile(r'^(?:https?://)?github\.com/.+?/.+?/(?:releases|archive)/.*$')
exp2 = re.compile(r'^(?:https?://)?github\.com/.+?/.+?/(?:blob)/.*$')
exp3 = re.compile(r'^(?:https?://)?github\.com/.+?/.+?/(?:info|git-).*$')
exp4 = re.compile(r'^(?:https?://)?raw\.githubusercontent\.com/.+?/.+?/.+?/.+$')
exp5 = re.compile(r'^(?:https?://)?gist\.(?:githubusercontent|github)\.com/.+?/.+?/.+$')
requests.sessions.default_headers = lambda: CaseInsensitiveDict()
@app.route('/')
def index():
if 'q' in request.args:
return redirect('/' + request.args.get('q'))
return index_html
@app.route('/favicon.ico')
def icon():
return Response(icon_r, content_type='image/vnd.microsoft.icon')
def iter_content(self, chunk_size=1, decode_unicode=False):
"""rewrite requests function, set decode_content with False"""
def generate():
# Special case for urllib3.
if hasattr(self.raw, 'stream'):
try:
for chunk in self.raw.stream(chunk_size, decode_content=False):
yield chunk
except ProtocolError as e:
raise ChunkedEncodingError(e)
except DecodeError as e:
raise ContentDecodingError(e)
except ReadTimeoutError as e:
raise ConnectionError(e)
else:
# Standard file-like object.
while True:
chunk = self.raw.read(chunk_size)
if not chunk:
break
yield chunk
self._content_consumed = True
if self._content_consumed and isinstance(self._content, bool):
raise StreamConsumedError()
elif chunk_size is not None and not isinstance(chunk_size, int):
raise TypeError("chunk_size must be an int, it is instead a %s." % type(chunk_size))
# simulate reading small chunks of the content
reused_chunks = iter_slices(self._content, chunk_size)
stream_chunks = generate()
chunks = reused_chunks if self._content_consumed else stream_chunks
if decode_unicode:
chunks = stream_decode_response_unicode(chunks, self)
return chunks
@app.route('/<path:u>', methods=['GET', 'POST'])
def proxy(u):
u = u if u.startswith('http') else 'https://' + u
if u.rfind('://', 3, 9) == -1:
u = u.replace('s:/', 's://', 1) # uwsgi会将//传递为/
if not any([i.match(u) for i in [exp1, exp2, exp3, exp4, exp5]]):
return Response('Invalid input.', status=403)
if jsdelivr and exp2.match(u):
u = u.replace('/blob/', '@', 1).replace('github.com', 'cdn.jsdelivr.net/gh', 1)
return redirect(u)
elif cnpmjs and exp3.match(u):
u = u.replace('github.com', 'github.com.cnpmjs.org', 1) + request.url.replace(request.base_url, '', 1)
return redirect(u)
elif jsdelivr and exp4.match(u):
u = re.sub(r'(\.com/.*?/.+?)/(.+?/)', r'\1@\2', u, 1)
u = u.replace('raw.githubusercontent.com', 'cdn.jsdelivr.net/gh', 1)
return redirect(u)
else:
if exp2.match(u):
u = u.replace('/blob/', '/raw/', 1)
headers = {}
r_headers = dict(request.headers)
if 'Host' in r_headers:
r_headers.pop('Host')
try:
url = u + request.url.replace(request.base_url, '', 1)
if url.startswith('https:/') and not url.startswith('https://'):
url = 'https://' + url[7:]
r = requests.request(method=request.method, url=url, data=request.data, headers=r_headers, stream=True)
headers = dict(r.headers)
if 'Content-length' in r.headers and int(r.headers['Content-length']) > size_limit:
return redirect(u + request.url.replace(request.base_url, '', 1))
def generate():
for chunk in iter_content(r, chunk_size=CHUNK_SIZE):
yield chunk
return Response(generate(), headers=headers, status=r.status_code)
except Exception as e:
headers['content-type'] = 'text/html; charset=UTF-8'
return Response('server error ' + str(e), status=500, headers=headers)
# else:
# return Response('Illegal input', status=403, mimetype='text/html; charset=UTF-8')
if __name__ == '__main__':
app.run(host=HOST, port=PORT)

3
gh-proxy/app/uwsgi.ini Normal file
View File

@ -0,0 +1,3 @@
[uwsgi]
module = main
callable = app

26
gh-proxy/entrypoint.sh Normal file
View File

@ -0,0 +1,26 @@
#! /usr/bin/env bash
set -e
/uwsgi-nginx-entrypoint.sh
# Get the listen port for Nginx, default to 80
USE_LISTEN_PORT=${LISTEN_PORT:-80}
if [ -f /app/nginx.conf ]; then
cp /app/nginx.conf /etc/nginx/nginx.conf
else
content_server='server {\n'
content_server=$content_server" listen ${USE_LISTEN_PORT};\n"
content_server=$content_server' location / {\n'
content_server=$content_server' try_files $uri @app;\n'
content_server=$content_server' }\n'
content_server=$content_server' location @app {\n'
content_server=$content_server' include uwsgi_params;\n'
content_server=$content_server' uwsgi_pass unix:///tmp/uwsgi.sock;\n'
content_server=$content_server' }\n'
content_server=$content_server'}\n'
# Save generated server /etc/nginx/conf.d/nginx.conf
printf "$content_server" > /etc/nginx/conf.d/nginx.conf
fi
exec "$@"

165
gh-proxy/index.js Normal file
View File

@ -0,0 +1,165 @@
'use strict'
/**
* static files (404.html, sw.js, conf.js)
*/
const ASSET_URL = 'https://hunshcn.github.io/gh-proxy/'
// 前缀如果自定义路由为example.com/gh/*将PREFIX改为 '/gh/',注意,少一个杠都会错!
const PREFIX = '/'
// git使用cnpmjs镜像、分支文件使用jsDelivr镜像的开关0为关闭默认开启
const Config = {
jsdelivr: 1,
cnpmjs: 1
}
/** @type {RequestInit} */
const PREFLIGHT_INIT = {
status: 204,
headers: new Headers({
'access-control-allow-origin': '*',
'access-control-allow-methods': 'GET,POST,PUT,PATCH,TRACE,DELETE,HEAD,OPTIONS',
'access-control-max-age': '1728000',
}),
}
/**
* @param {any} body
* @param {number} status
* @param {Object<string, string>} headers
*/
function makeRes(body, status = 200, headers = {}) {
headers['access-control-allow-origin'] = '*'
return new Response(body, {status, headers})
}
/**
* @param {string} urlStr
*/
function newUrl(urlStr) {
try {
return new URL(urlStr)
} catch (err) {
return null
}
}
addEventListener('fetch', e => {
const ret = fetchHandler(e)
.catch(err => makeRes('cfworker error:\n' + err.stack, 502))
e.respondWith(ret)
})
/**
* @param {FetchEvent} e
*/
async function fetchHandler(e) {
const req = e.request
const urlStr = req.url
const urlObj = new URL(urlStr)
let path = urlObj.searchParams.get('q')
if (path) {
return Response.redirect('https://' + urlObj.host + PREFIX + path, 301)
}
// cfworker 会把路径中的 `//` 合并成 `/`
path = urlObj.href.substr(urlObj.origin.length + PREFIX.length).replace(/^https?:\/+/, 'https://')
const exp1 = /^(?:https?:\/\/)?github\.com\/.+?\/.+?\/(?:releases|archive)\/.*$/i
const exp2 = /^(?:https?:\/\/)?github\.com\/.+?\/.+?\/(?:blob)\/.*$/i
const exp3 = /^(?:https?:\/\/)?github\.com\/.+?\/.+?\/(?:info|git-).*$/i
const exp4 = /^(?:https?:\/\/)?raw\.githubusercontent\.com\/.+?\/.+?\/.+?\/.+$/i
const exp5 = /^(?:https?:\/\/)?gist\.(?:githubusercontent|github)\.com\/.+?\/.+?\/.+$/i
if (path.search(exp1) === 0 || path.search(exp5) === 0 || !Config.cnpmjs && (path.search(exp3) === 0 || path.search(exp4) === 0)) {
return httpHandler(req, path)
} else if (path.search(exp2) === 0) {
if (Config.jsdelivr){
const newUrl = path.replace('/blob/', '@').replace(/^(?:https?:\/\/)?github\.com/, 'https://cdn.jsdelivr.net/gh')
return Response.redirect(newUrl, 302)
}else{
path = path.replace('/blob/', '/raw/')
return httpHandler(req, path)
}
} else if (path.search(exp3) === 0) {
const newUrl = path.replace(/^(?:https?:\/\/)?github\.com/, 'https://github.com.cnpmjs.org')
return Response.redirect(newUrl, 302)
} else if (path.search(exp4) === 0) {
const newUrl = path.replace(/(?<=com\/.+?\/.+?)\/(.+?\/)/, '@$1').replace(/^(?:https?:\/\/)?raw\.githubusercontent\.com/, 'https://cdn.jsdelivr.net/gh')
return Response.redirect(newUrl, 302)
} else {
return fetch(ASSET_URL + path)
}
}
/**
* @param {Request} req
* @param {string} pathname
*/
function httpHandler(req, pathname) {
const reqHdrRaw = req.headers
// preflight
if (req.method === 'OPTIONS' &&
reqHdrRaw.has('access-control-request-headers')
) {
return new Response(null, PREFLIGHT_INIT)
}
let rawLen = ''
const reqHdrNew = new Headers(reqHdrRaw)
let urlStr = pathname
if (urlStr.startsWith('github')) {
urlStr = 'https://' + urlStr
}
const urlObj = newUrl(urlStr)
/** @type {RequestInit} */
const reqInit = {
method: req.method,
headers: reqHdrNew,
redirect: 'follow',
body: req.body
}
return proxy(urlObj, reqInit, rawLen, 0)
}
/**
*
* @param {URL} urlObj
* @param {RequestInit} reqInit
*/
async function proxy(urlObj, reqInit, rawLen) {
const res = await fetch(urlObj.href, reqInit)
const resHdrOld = res.headers
const resHdrNew = new Headers(resHdrOld)
// verify
if (rawLen) {
const newLen = resHdrOld.get('content-length') || ''
const badLen = (rawLen !== newLen)
if (badLen) {
return makeRes(res.body, 400, {
'--error': `bad len: ${newLen}, except: ${rawLen}`,
'access-control-expose-headers': '--error',
})
}
}
const status = res.status
resHdrNew.set('access-control-expose-headers', '*')
resHdrNew.set('access-control-allow-origin', '*')
resHdrNew.delete('content-security-policy')
resHdrNew.delete('content-security-policy-report-only')
resHdrNew.delete('clear-site-data')
return new Response(res.body, {
status,
headers: resHdrNew,
})
}