761 lines
18 KiB
Go
761 lines
18 KiB
Go
package service
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"net"
|
||
"net/http"
|
||
"net/url"
|
||
"regexp"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
"unicode"
|
||
"unicode/utf8"
|
||
)
|
||
|
||
const (
|
||
timelineGitHTTPTimeout = 8 * time.Second
|
||
timelineGitMaxBody = 1 << 20 // 1MB
|
||
timelineTitleMax = 200
|
||
timelineBodyMax = 2000
|
||
)
|
||
|
||
// TimelineGitItem 导入结果单项（已消毒）
|
||
type TimelineGitItem struct {
|
||
Date string `json:"date"`
|
||
Title string `json:"title"`
|
||
Body string `json:"body"`
|
||
SourceURL string `json:"source_url,omitempty"`
|
||
SHA string `json:"sha,omitempty"`
|
||
}
|
||
|
||
// TimelineGitImportResult 导入响应
|
||
type TimelineGitImportResult struct {
|
||
Items []TimelineGitItem `json:"items"`
|
||
Warning string `json:"warning,omitempty"`
|
||
Error string `json:"error,omitempty"`
|
||
}
|
||
|
||
var (
|
||
repoNameRe = regexp.MustCompile(`^[A-Za-z0-9._-]{1,100}$`)
|
||
shaHexRe = regexp.MustCompile(`^[0-9a-fA-F]{7,64}$`)
|
||
)
|
||
|
||
// ImportTimelineFromGit 按适配器解析 commits 页 URL 并拉取提交
|
||
func (s *SettingService) ImportTimelineFromGit(urls []string, followPages bool) (*TimelineGitImportResult, error) {
|
||
cfg, err := s.loadTimelineGitAdapter()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
cleanURLs := make([]string, 0, len(urls))
|
||
for _, u := range urls {
|
||
u = strings.TrimSpace(u)
|
||
if u != "" {
|
||
cleanURLs = append(cleanURLs, u)
|
||
}
|
||
}
|
||
if len(cleanURLs) == 0 {
|
||
return nil, errors.New("请提供至少一条 URL")
|
||
}
|
||
if len(cleanURLs) > 20 {
|
||
return nil, errors.New("一次最多 20 条 URL")
|
||
}
|
||
|
||
client := &http.Client{
|
||
Timeout: timelineGitHTTPTimeout,
|
||
Transport: &http.Transport{
|
||
// 解析后按公网 IP 拨号，避免 DNS rebinding（与 SMTP/S3 safeDial 同思路）
|
||
DialContext: publicOnlyDial,
|
||
TLSHandshakeTimeout: timelineGitHTTPTimeout,
|
||
ForceAttemptHTTP2: true,
|
||
},
|
||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||
if len(via) >= 3 {
|
||
return errors.New("重定向过多")
|
||
}
|
||
if err := assertSafeHTTPSURL(req.URL); err != nil {
|
||
return err
|
||
}
|
||
return nil
|
||
},
|
||
}
|
||
|
||
seenSHA := map[string]bool{}
|
||
var items []TimelineGitItem
|
||
var failMsgs []string
|
||
truncated := false
|
||
|
||
for _, rawURL := range cleanURLs {
|
||
part, partTrunc, err := s.importOneGitURL(client, cfg, rawURL, followPages, seenSHA, cfg.MaxCommits-len(items))
|
||
if err != nil {
|
||
failMsgs = append(failMsgs, fmt.Sprintf("%s：%s", truncateTimelineStr(rawURL, 80), err.Error()))
|
||
continue
|
||
}
|
||
items = append(items, part...)
|
||
if partTrunc {
|
||
truncated = true
|
||
}
|
||
if len(items) >= cfg.MaxCommits {
|
||
truncated = true
|
||
break
|
||
}
|
||
}
|
||
|
||
out := &TimelineGitImportResult{Items: items}
|
||
if truncated {
|
||
out.Warning = fmt.Sprintf("已达上限（最多 %d 条），可再贴后续页 URL", cfg.MaxCommits)
|
||
}
|
||
if len(failMsgs) > 0 {
|
||
out.Error = strings.Join(failMsgs, "；")
|
||
}
|
||
if len(items) == 0 && out.Error == "" {
|
||
out.Error = "未能解析出提交"
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
func (s *SettingService) importOneGitURL(
|
||
client *http.Client,
|
||
cfg *TimelineGitAdapter,
|
||
rawURL string,
|
||
followPages bool,
|
||
seenSHA map[string]bool,
|
||
remain int,
|
||
) ([]TimelineGitItem, bool, error) {
|
||
if remain <= 0 {
|
||
return nil, true, nil
|
||
}
|
||
u, err := url.Parse(rawURL)
|
||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||
return nil, false, errors.New("URL 无效")
|
||
}
|
||
if err := assertSafeHTTPSURL(u); err != nil {
|
||
return nil, false, err
|
||
}
|
||
host := strings.ToLower(u.Hostname())
|
||
path := u.EscapedPath()
|
||
if path == "" {
|
||
path = "/"
|
||
}
|
||
|
||
src, caps, singleSHA, err := matchGitSource(cfg, host, path)
|
||
if err != nil {
|
||
return nil, false, err
|
||
}
|
||
if !repoNameRe.MatchString(caps["owner"]) || !repoNameRe.MatchString(caps["repo"]) {
|
||
return nil, false, errors.New("仓库名非法")
|
||
}
|
||
if ref, ok := caps["ref"]; ok && ref != "" {
|
||
if strings.ContainsAny(ref, " \t\n\r") || utf8.RuneCountInString(ref) > 200 {
|
||
return nil, false, errors.New("分支名非法")
|
||
}
|
||
}
|
||
|
||
startPage := 1
|
||
if p := u.Query().Get("page"); p != "" {
|
||
if n, e := strconv.Atoi(p); e == nil && n >= 1 {
|
||
startPage = n
|
||
}
|
||
}
|
||
|
||
if singleSHA != "" {
|
||
item, err := fetchSingleCommit(client, src, host, caps, singleSHA)
|
||
if err != nil {
|
||
return nil, false, err
|
||
}
|
||
if item.SHA != "" && seenSHA[item.SHA] {
|
||
return nil, false, nil
|
||
}
|
||
if item.SHA != "" {
|
||
seenSHA[item.SHA] = true
|
||
}
|
||
return []TimelineGitItem{*item}, false, nil
|
||
}
|
||
|
||
var out []TimelineGitItem
|
||
truncated := false
|
||
page := startPage
|
||
maxPages := 1
|
||
if followPages {
|
||
maxPages = cfg.MaxPages
|
||
}
|
||
pagesDone := 0
|
||
nextURL := ""
|
||
|
||
for pagesDone < maxPages && len(out) < remain {
|
||
var apiURL string
|
||
if nextURL != "" {
|
||
apiURL = nextURL
|
||
nextURL = ""
|
||
} else {
|
||
apiURL, err = expandAPITemplate(src.APIURL, host, caps, page)
|
||
if err != nil {
|
||
return out, truncated, err
|
||
}
|
||
q := url.Values{}
|
||
for k, v := range src.Query {
|
||
q.Set(k, expandTemplate(v, host, caps, page))
|
||
}
|
||
parsed, e := url.Parse(apiURL)
|
||
if e != nil {
|
||
return out, truncated, errors.New("API URL 无效")
|
||
}
|
||
if len(q) > 0 {
|
||
existing := parsed.Query()
|
||
for k, vs := range q {
|
||
existing.Set(k, vs[0])
|
||
}
|
||
parsed.RawQuery = existing.Encode()
|
||
}
|
||
apiURL = parsed.String()
|
||
}
|
||
parsedAPI, err := url.Parse(apiURL)
|
||
if err != nil {
|
||
return out, truncated, errors.New("API URL 无效")
|
||
}
|
||
if err := assertSafeHTTPSURL(parsedAPI); err != nil {
|
||
return out, truncated, err
|
||
}
|
||
|
||
headers := http.Header{}
|
||
for k, v := range src.Headers {
|
||
if allowedAdapterHeaders[strings.ToLower(k)] {
|
||
headers.Set(k, v)
|
||
}
|
||
}
|
||
if headers.Get("User-Agent") == "" {
|
||
headers.Set("User-Agent", "jiang13-bbs")
|
||
}
|
||
|
||
body, linkNext, status, err := httpGetLimited(client, parsedAPI.String(), headers)
|
||
if err != nil {
|
||
return out, truncated, err
|
||
}
|
||
if status == 404 || status == 401 || status == 403 {
|
||
return out, truncated, errors.New("无法读取该仓库（私有、不存在或无权访问）")
|
||
}
|
||
if status == 429 {
|
||
return out, truncated, errors.New("远端限流，请稍后再试")
|
||
}
|
||
if status < 200 || status >= 300 {
|
||
return out, truncated, fmt.Errorf("远端返回 %d", status)
|
||
}
|
||
|
||
pageItems, err := parseCommitListJSON(body, src, host)
|
||
if err != nil {
|
||
return out, truncated, err
|
||
}
|
||
if len(pageItems) == 0 {
|
||
break
|
||
}
|
||
for _, it := range pageItems {
|
||
if it.SHA != "" && seenSHA[it.SHA] {
|
||
continue
|
||
}
|
||
if it.SHA != "" {
|
||
seenSHA[it.SHA] = true
|
||
}
|
||
out = append(out, it)
|
||
if len(out) >= remain {
|
||
truncated = true
|
||
break
|
||
}
|
||
}
|
||
pagesDone++
|
||
if !followPages {
|
||
break
|
||
}
|
||
if src.Pagination == "link_header" && linkNext != "" {
|
||
nu, e := url.Parse(linkNext)
|
||
if e != nil || assertSafeHTTPSURL(nu) != nil {
|
||
break
|
||
}
|
||
nextURL = nu.String()
|
||
} else if src.Pagination == "query_page" {
|
||
page++
|
||
} else {
|
||
break
|
||
}
|
||
}
|
||
if pagesDone >= maxPages && followPages {
|
||
truncated = true
|
||
}
|
||
return out, truncated, nil
|
||
}
|
||
|
||
func matchGitSource(cfg *TimelineGitAdapter, host, path string) (*TimelineGitSourceCfg, map[string]string, string, error) {
|
||
for i := range cfg.Sources {
|
||
src := &cfg.Sources[i]
|
||
if src.Host != "*" && !strings.EqualFold(src.Host, host) {
|
||
continue
|
||
}
|
||
if src.Host == "*" && strings.EqualFold(host, "github.com") {
|
||
continue
|
||
}
|
||
if src.CommitPath != "" {
|
||
re, err := regexp.Compile(src.CommitPath)
|
||
if err == nil {
|
||
if m := re.FindStringSubmatch(path); m != nil {
|
||
caps := subexpMap(re, m)
|
||
sha := caps["sha"]
|
||
if !shaHexRe.MatchString(sha) {
|
||
return nil, nil, "", errors.New("提交哈希非法")
|
||
}
|
||
return src, caps, sha, nil
|
||
}
|
||
}
|
||
}
|
||
re, err := regexp.Compile(src.ListPath)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
if m := re.FindStringSubmatch(path); m != nil {
|
||
return src, subexpMap(re, m), "", nil
|
||
}
|
||
}
|
||
return nil, nil, "", errors.New("地址不符或主机未配置")
|
||
}
|
||
|
||
func subexpMap(re *regexp.Regexp, m []string) map[string]string {
|
||
out := map[string]string{}
|
||
for i, name := range re.SubexpNames() {
|
||
if i == 0 || name == "" || i >= len(m) {
|
||
continue
|
||
}
|
||
out[name] = m[i]
|
||
}
|
||
return out
|
||
}
|
||
|
||
func expandTemplate(tpl, host string, caps map[string]string, page int) string {
|
||
r := strings.NewReplacer(
|
||
"{host}", host,
|
||
"{owner}", caps["owner"],
|
||
"{repo}", caps["repo"],
|
||
"{ref}", caps["ref"],
|
||
"{sha}", caps["sha"],
|
||
"{page}", strconv.Itoa(page),
|
||
)
|
||
return r.Replace(tpl)
|
||
}
|
||
|
||
func expandAPITemplate(tpl, host string, caps map[string]string, page int) (string, error) {
|
||
s := expandTemplate(tpl, host, caps, page)
|
||
u, err := url.Parse(s)
|
||
if err != nil {
|
||
return "", errors.New("API URL 无效")
|
||
}
|
||
if err := assertSafeHTTPSURL(u); err != nil {
|
||
return "", err
|
||
}
|
||
return u.String(), nil
|
||
}
|
||
|
||
func fetchSingleCommit(client *http.Client, src *TimelineGitSourceCfg, host string, caps map[string]string, sha string) (*TimelineGitItem, error) {
|
||
caps = copyCaps(caps)
|
||
caps["sha"] = sha
|
||
apiTpl := src.CommitAPI
|
||
if apiTpl == "" {
|
||
apiTpl = strings.TrimSuffix(src.APIURL, "/") + "/{sha}"
|
||
}
|
||
apiURL, err := expandAPITemplate(apiTpl, host, caps, 1)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
headers := http.Header{}
|
||
for k, v := range src.Headers {
|
||
if allowedAdapterHeaders[strings.ToLower(k)] {
|
||
headers.Set(k, v)
|
||
}
|
||
}
|
||
if headers.Get("User-Agent") == "" {
|
||
headers.Set("User-Agent", "jiang13-bbs")
|
||
}
|
||
body, _, status, err := httpGetLimited(client, apiURL, headers)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if status < 200 || status >= 300 {
|
||
return nil, fmt.Errorf("远端返回 %d", status)
|
||
}
|
||
var obj any
|
||
if err := json.Unmarshal(body, &obj); err != nil {
|
||
return nil, errors.New("响应非 JSON")
|
||
}
|
||
item, ok := mapCommitObject(obj, src, host)
|
||
if !ok {
|
||
return nil, errors.New("无法解析提交")
|
||
}
|
||
return &item, nil
|
||
}
|
||
|
||
func copyCaps(in map[string]string) map[string]string {
|
||
out := make(map[string]string, len(in))
|
||
for k, v := range in {
|
||
out[k] = v
|
||
}
|
||
return out
|
||
}
|
||
|
||
func parseCommitListJSON(body []byte, src *TimelineGitSourceCfg, host string) ([]TimelineGitItem, error) {
|
||
var root any
|
||
if err := json.Unmarshal(body, &root); err != nil {
|
||
return nil, errors.New("响应非 JSON")
|
||
}
|
||
arr, ok := root.([]any)
|
||
if !ok {
|
||
if m, isMap := root.(map[string]any); isMap {
|
||
if c, ok := m["commits"].([]any); ok {
|
||
arr = c
|
||
}
|
||
}
|
||
}
|
||
if arr == nil {
|
||
return nil, errors.New("响应不是提交列表")
|
||
}
|
||
var out []TimelineGitItem
|
||
for _, el := range arr {
|
||
item, ok := mapCommitObject(el, src, host)
|
||
if !ok {
|
||
continue
|
||
}
|
||
out = append(out, item)
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
func mapCommitObject(el any, src *TimelineGitSourceCfg, host string) (TimelineGitItem, bool) {
|
||
sha := jsonPathString(el, src.Item.SHA)
|
||
dateRaw := jsonPathString(el, src.Item.Date)
|
||
msg := jsonPathString(el, src.Item.Message)
|
||
srcURL := jsonPathString(el, src.Item.SourceURL)
|
||
if msg == "" && sha == "" {
|
||
return TimelineGitItem{}, false
|
||
}
|
||
sha = sanitizeSHA(sha)
|
||
title, body := splitCommitMessage(msg)
|
||
title = sanitizeTimelinePlain(title, timelineTitleMax)
|
||
body = sanitizeTimelinePlain(body, timelineBodyMax)
|
||
title = neutralizeDirectivePlain(title)
|
||
body = neutralizeDirectivePlain(body)
|
||
date := parseCommitDate(dateRaw)
|
||
srcURL = sanitizeSourceURL(srcURL, host)
|
||
return TimelineGitItem{
|
||
Date: date,
|
||
Title: title,
|
||
Body: body,
|
||
SourceURL: srcURL,
|
||
SHA: sha,
|
||
}, true
|
||
}
|
||
|
||
func jsonPathString(root any, path string) string {
|
||
if path == "" || root == nil {
|
||
return ""
|
||
}
|
||
cur := root
|
||
for _, part := range strings.Split(path, ".") {
|
||
m, ok := cur.(map[string]any)
|
||
if !ok {
|
||
return ""
|
||
}
|
||
cur, ok = m[part]
|
||
if !ok {
|
||
return ""
|
||
}
|
||
}
|
||
switch v := cur.(type) {
|
||
case string:
|
||
return v
|
||
case float64:
|
||
return strconv.FormatInt(int64(v), 10)
|
||
case json.Number:
|
||
return v.String()
|
||
default:
|
||
return ""
|
||
}
|
||
}
|
||
|
||
func splitCommitMessage(msg string) (title, body string) {
|
||
msg = strings.ReplaceAll(msg, "\r\n", "\n")
|
||
msg = strings.TrimSpace(msg)
|
||
if msg == "" {
|
||
return "提交", ""
|
||
}
|
||
parts := strings.SplitN(msg, "\n", 2)
|
||
title = strings.TrimSpace(parts[0])
|
||
if title == "" {
|
||
title = "提交"
|
||
}
|
||
if len(parts) > 1 {
|
||
body = stripCommitTrailers(strings.TrimSpace(parts[1]))
|
||
}
|
||
return title, body
|
||
}
|
||
|
||
// stripCommitTrailers 去掉 Co-authored-by 等尾部 trailer 块（及前导空行）。
|
||
func stripCommitTrailers(body string) string {
|
||
if body == "" {
|
||
return ""
|
||
}
|
||
lines := strings.Split(body, "\n")
|
||
// 从末尾向前：连续 trailer / 空行；遇到非 trailer 正文则停
|
||
end := len(lines)
|
||
for end > 0 {
|
||
t := strings.TrimSpace(lines[end-1])
|
||
if t == "" || isCommitTrailerLine(t) {
|
||
end--
|
||
continue
|
||
}
|
||
break
|
||
}
|
||
// 若尾部有 trailer，再去掉其前的空行分隔
|
||
for end > 0 && strings.TrimSpace(lines[end-1]) == "" {
|
||
end--
|
||
}
|
||
return strings.TrimSpace(strings.Join(lines[:end], "\n"))
|
||
}
|
||
|
||
func isCommitTrailerLine(t string) bool {
|
||
lower := strings.ToLower(t)
|
||
prefixes := []string{
|
||
"co-authored-by:",
|
||
"signed-off-by:",
|
||
"reviewed-by:",
|
||
"acked-by:",
|
||
"tested-by:",
|
||
"reported-by:",
|
||
"suggested-by:",
|
||
"helped-by:",
|
||
}
|
||
for _, p := range prefixes {
|
||
if strings.HasPrefix(lower, p) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func sanitizeSHA(s string) string {
|
||
s = strings.TrimSpace(s)
|
||
if !shaHexRe.MatchString(s) {
|
||
return ""
|
||
}
|
||
if len(s) > 64 {
|
||
return s[:64]
|
||
}
|
||
return s
|
||
}
|
||
|
||
func sanitizeTimelinePlain(s string, max int) string {
|
||
var b strings.Builder
|
||
b.Grow(len(s))
|
||
for _, r := range s {
|
||
if r == 0 || (r < 0x20 && r != '\n' && r != '\t') || r == 0x7f {
|
||
continue
|
||
}
|
||
if r >= 0x202A && r <= 0x202E {
|
||
continue
|
||
}
|
||
if r >= 0x2066 && r <= 0x2069 {
|
||
continue
|
||
}
|
||
if unicode.Is(unicode.Cs, r) {
|
||
continue
|
||
}
|
||
b.WriteRune(r)
|
||
}
|
||
out := b.String()
|
||
if utf8.RuneCountInString(out) > max {
|
||
runes := []rune(out)
|
||
out = string(runes[:max])
|
||
}
|
||
return out
|
||
}
|
||
|
||
func neutralizeDirectivePlain(s string) string {
|
||
lines := strings.Split(s, "\n")
|
||
for i, line := range lines {
|
||
t := strings.TrimSpace(line)
|
||
lower := strings.ToLower(t)
|
||
if lower == "[/hide]" || lower == "[/timeline]" || lower == "[timeline]" ||
|
||
strings.HasPrefix(lower, "[hide") || strings.HasPrefix(t, "<!-- git:") {
|
||
lines[i] = " " + line
|
||
}
|
||
}
|
||
return strings.Join(lines, "\n")
|
||
}
|
||
|
||
func parseCommitDate(raw string) string {
|
||
raw = strings.TrimSpace(raw)
|
||
if raw == "" {
|
||
return time.Now().UTC().Format("2006-01-02")
|
||
}
|
||
layouts := []string{
|
||
time.RFC3339Nano,
|
||
time.RFC3339,
|
||
"2006-01-02T15:04:05Z",
|
||
"2006-01-02 15:04:05",
|
||
"2006-01-02",
|
||
}
|
||
for _, layout := range layouts {
|
||
if t, err := time.Parse(layout, raw); err == nil {
|
||
return t.UTC().Format("2006-01-02")
|
||
}
|
||
}
|
||
return time.Now().UTC().Format("2006-01-02")
|
||
}
|
||
|
||
func sanitizeSourceURL(raw, allowedHost string) string {
|
||
raw = strings.TrimSpace(raw)
|
||
if raw == "" {
|
||
return ""
|
||
}
|
||
u, err := url.Parse(raw)
|
||
if err != nil {
|
||
return ""
|
||
}
|
||
if !strings.EqualFold(u.Scheme, "https") {
|
||
return ""
|
||
}
|
||
h := strings.ToLower(u.Hostname())
|
||
ah := strings.ToLower(allowedHost)
|
||
if h == ah || (ah == "github.com" && (h == "github.com" || h == "www.github.com")) {
|
||
return u.String()
|
||
}
|
||
if ah == "api.github.com" && (h == "github.com" || h == "www.github.com") {
|
||
return u.String()
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func assertSafeHTTPSURL(u *url.URL) error {
|
||
if u == nil {
|
||
return errors.New("URL 无效")
|
||
}
|
||
if !strings.EqualFold(u.Scheme, "https") {
|
||
return errors.New("仅允许 https")
|
||
}
|
||
host := u.Hostname()
|
||
if host == "" {
|
||
return errors.New("主机为空")
|
||
}
|
||
if strings.EqualFold(host, "localhost") || strings.HasSuffix(strings.ToLower(host), ".localhost") {
|
||
return errors.New("禁止访问内网地址")
|
||
}
|
||
ips, err := net.LookupIP(host)
|
||
if err != nil {
|
||
return errors.New("无法解析主机")
|
||
}
|
||
if len(ips) == 0 {
|
||
return errors.New("无法解析主机")
|
||
}
|
||
for _, ip := range ips {
|
||
if !isPublicIP(ip) {
|
||
return errors.New("禁止访问内网地址")
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// publicOnlyDial 先解析主机、拒绝私网，再按 IP 拨号，关闭 DNS rebinding 窗口。
|
||
func publicOnlyDial(ctx context.Context, network, address string) (net.Conn, error) {
|
||
host, port, err := net.SplitHostPort(address)
|
||
if err != nil {
|
||
return nil, errors.New("连接地址无效")
|
||
}
|
||
ips, err := net.DefaultResolver.LookupIPAddr(ctx, host)
|
||
if err != nil || len(ips) == 0 {
|
||
return nil, errors.New("地址解析失败")
|
||
}
|
||
var last error
|
||
dialer := &net.Dialer{Timeout: timelineGitHTTPTimeout}
|
||
for _, a := range ips {
|
||
if !isPublicIP(a.IP) {
|
||
last = errors.New("禁止访问内网地址")
|
||
continue
|
||
}
|
||
c, e := dialer.DialContext(ctx, network, net.JoinHostPort(a.IP.String(), port))
|
||
if e == nil {
|
||
return c, nil
|
||
}
|
||
last = e
|
||
}
|
||
if last == nil {
|
||
last = errors.New("禁止访问内网地址")
|
||
}
|
||
return nil, last
|
||
}
|
||
|
||
func isPublicIP(ip net.IP) bool {
|
||
if ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsMulticast() || ip.IsUnspecified() {
|
||
return false
|
||
}
|
||
if ip4 := ip.To4(); ip4 != nil {
|
||
if ip4[0] == 169 && ip4[1] == 254 {
|
||
return false
|
||
}
|
||
// CGNAT / 文档网段等
|
||
if ip4[0] == 100 && ip4[1] >= 64 && ip4[1] <= 127 {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
func httpGetLimited(client *http.Client, rawURL string, headers http.Header) ([]byte, string, int, error) {
|
||
ctx, cancel := context.WithTimeout(context.Background(), timelineGitHTTPTimeout)
|
||
defer cancel()
|
||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||
if err != nil {
|
||
return nil, "", 0, err
|
||
}
|
||
for k, vs := range headers {
|
||
for _, v := range vs {
|
||
req.Header.Add(k, v)
|
||
}
|
||
}
|
||
resp, err := client.Do(req)
|
||
if err != nil {
|
||
return nil, "", 0, errors.New("请求失败")
|
||
}
|
||
defer resp.Body.Close()
|
||
limited := io.LimitReader(resp.Body, timelineGitMaxBody+1)
|
||
body, err := io.ReadAll(limited)
|
||
if err != nil {
|
||
return nil, "", resp.StatusCode, errors.New("读取响应失败")
|
||
}
|
||
if len(body) > timelineGitMaxBody {
|
||
return nil, "", resp.StatusCode, errors.New("响应过大")
|
||
}
|
||
linkNext := parseLinkNext(resp.Header.Get("Link"))
|
||
return body, linkNext, resp.StatusCode, nil
|
||
}
|
||
|
||
func parseLinkNext(link string) string {
|
||
parts := strings.Split(link, ",")
|
||
for _, p := range parts {
|
||
p = strings.TrimSpace(p)
|
||
if !strings.Contains(p, `rel="next"`) && !strings.Contains(p, `rel=next`) {
|
||
continue
|
||
}
|
||
start := strings.Index(p, "<")
|
||
end := strings.Index(p, ">")
|
||
if start >= 0 && end > start {
|
||
return p[start+1 : end]
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func truncateTimelineStr(s string, n int) string {
|
||
if utf8.RuneCountInString(s) <= n {
|
||
return s
|
||
}
|
||
return string([]rune(s)[:n]) + "…"
|
||
}
|