package service import ( "context" "encoding/json" "errors" "fmt" "io" "net" "net/http" "net/url" "regexp" "strconv" "strings" "time" "unicode" "unicode/utf8" ) const ( timelineGitHTTPTimeout = 8 * time.Second timelineGitMaxBody = 1 << 20 // 1MB timelineTitleMax = 200 timelineBodyMax = 2000 ) // TimelineGitItem 导入结果单项(已消毒) type TimelineGitItem struct { Date string `json:"date"` Title string `json:"title"` Body string `json:"body"` SourceURL string `json:"source_url,omitempty"` SHA string `json:"sha,omitempty"` } // TimelineGitImportResult 导入响应 type TimelineGitImportResult struct { Items []TimelineGitItem `json:"items"` Warning string `json:"warning,omitempty"` Error string `json:"error,omitempty"` } var ( repoNameRe = regexp.MustCompile(`^[A-Za-z0-9._-]{1,100}$`) shaHexRe = regexp.MustCompile(`^[0-9a-fA-F]{7,64}$`) ) // ImportTimelineFromGit 按适配器解析 commits 页 URL 并拉取提交 func (s *SettingService) ImportTimelineFromGit(urls []string, followPages bool) (*TimelineGitImportResult, error) { cfg, err := s.loadTimelineGitAdapter() if err != nil { return nil, err } cleanURLs := make([]string, 0, len(urls)) for _, u := range urls { u = strings.TrimSpace(u) if u != "" { cleanURLs = append(cleanURLs, u) } } if len(cleanURLs) == 0 { return nil, errors.New("请提供至少一条 URL") } if len(cleanURLs) > 20 { return nil, errors.New("一次最多 20 条 URL") } client := &http.Client{ Timeout: timelineGitHTTPTimeout, Transport: &http.Transport{ // 解析后按公网 IP 拨号,避免 DNS rebinding(与 SMTP/S3 safeDial 同思路) DialContext: publicOnlyDial, TLSHandshakeTimeout: timelineGitHTTPTimeout, ForceAttemptHTTP2: true, }, CheckRedirect: func(req *http.Request, via []*http.Request) error { if len(via) >= 3 { return errors.New("重定向过多") } if err := assertSafeHTTPSURL(req.URL); err != nil { return err } return nil }, } seenSHA := map[string]bool{} var items []TimelineGitItem var failMsgs []string truncated := false for _, rawURL := range cleanURLs { part, partTrunc, err := s.importOneGitURL(client, cfg, rawURL, followPages, seenSHA, cfg.MaxCommits-len(items)) if err != nil { failMsgs = append(failMsgs, fmt.Sprintf("%s:%s", truncateTimelineStr(rawURL, 80), err.Error())) continue } items = append(items, part...) if partTrunc { truncated = true } if len(items) >= cfg.MaxCommits { truncated = true break } } out := &TimelineGitImportResult{Items: items} if truncated { out.Warning = fmt.Sprintf("已达上限(最多 %d 条),可再贴后续页 URL", cfg.MaxCommits) } if len(failMsgs) > 0 { out.Error = strings.Join(failMsgs, ";") } if len(items) == 0 && out.Error == "" { out.Error = "未能解析出提交" } return out, nil } func (s *SettingService) importOneGitURL( client *http.Client, cfg *TimelineGitAdapter, rawURL string, followPages bool, seenSHA map[string]bool, remain int, ) ([]TimelineGitItem, bool, error) { if remain <= 0 { return nil, true, nil } u, err := url.Parse(rawURL) if err != nil || u.Scheme == "" || u.Host == "" { return nil, false, errors.New("URL 无效") } if err := assertSafeHTTPSURL(u); err != nil { return nil, false, err } host := strings.ToLower(u.Hostname()) path := u.EscapedPath() if path == "" { path = "/" } src, caps, singleSHA, err := matchGitSource(cfg, host, path) if err != nil { return nil, false, err } if !repoNameRe.MatchString(caps["owner"]) || !repoNameRe.MatchString(caps["repo"]) { return nil, false, errors.New("仓库名非法") } if ref, ok := caps["ref"]; ok && ref != "" { if strings.ContainsAny(ref, " \t\n\r") || utf8.RuneCountInString(ref) > 200 { return nil, false, errors.New("分支名非法") } } startPage := 1 if p := u.Query().Get("page"); p != "" { if n, e := strconv.Atoi(p); e == nil && n >= 1 { startPage = n } } if singleSHA != "" { item, err := fetchSingleCommit(client, src, host, caps, singleSHA) if err != nil { return nil, false, err } if item.SHA != "" && seenSHA[item.SHA] { return nil, false, nil } if item.SHA != "" { seenSHA[item.SHA] = true } return []TimelineGitItem{*item}, false, nil } var out []TimelineGitItem truncated := false page := startPage maxPages := 1 if followPages { maxPages = cfg.MaxPages } pagesDone := 0 nextURL := "" for pagesDone < maxPages && len(out) < remain { var apiURL string if nextURL != "" { apiURL = nextURL nextURL = "" } else { apiURL, err = expandAPITemplate(src.APIURL, host, caps, page) if err != nil { return out, truncated, err } q := url.Values{} for k, v := range src.Query { q.Set(k, expandTemplate(v, host, caps, page)) } parsed, e := url.Parse(apiURL) if e != nil { return out, truncated, errors.New("API URL 无效") } if len(q) > 0 { existing := parsed.Query() for k, vs := range q { existing.Set(k, vs[0]) } parsed.RawQuery = existing.Encode() } apiURL = parsed.String() } parsedAPI, err := url.Parse(apiURL) if err != nil { return out, truncated, errors.New("API URL 无效") } if err := assertSafeHTTPSURL(parsedAPI); err != nil { return out, truncated, err } headers := http.Header{} for k, v := range src.Headers { if allowedAdapterHeaders[strings.ToLower(k)] { headers.Set(k, v) } } if headers.Get("User-Agent") == "" { headers.Set("User-Agent", "jiang13-bbs") } body, linkNext, status, err := httpGetLimited(client, parsedAPI.String(), headers) if err != nil { return out, truncated, err } if status == 404 || status == 401 || status == 403 { return out, truncated, errors.New("无法读取该仓库(私有、不存在或无权访问)") } if status == 429 { return out, truncated, errors.New("远端限流,请稍后再试") } if status < 200 || status >= 300 { return out, truncated, fmt.Errorf("远端返回 %d", status) } pageItems, err := parseCommitListJSON(body, src, host) if err != nil { return out, truncated, err } if len(pageItems) == 0 { break } for _, it := range pageItems { if it.SHA != "" && seenSHA[it.SHA] { continue } if it.SHA != "" { seenSHA[it.SHA] = true } out = append(out, it) if len(out) >= remain { truncated = true break } } pagesDone++ if !followPages { break } if src.Pagination == "link_header" && linkNext != "" { nu, e := url.Parse(linkNext) if e != nil || assertSafeHTTPSURL(nu) != nil { break } nextURL = nu.String() } else if src.Pagination == "query_page" { page++ } else { break } } if pagesDone >= maxPages && followPages { truncated = true } return out, truncated, nil } func matchGitSource(cfg *TimelineGitAdapter, host, path string) (*TimelineGitSourceCfg, map[string]string, string, error) { for i := range cfg.Sources { src := &cfg.Sources[i] if src.Host != "*" && !strings.EqualFold(src.Host, host) { continue } if src.Host == "*" && strings.EqualFold(host, "github.com") { continue } if src.CommitPath != "" { re, err := regexp.Compile(src.CommitPath) if err == nil { if m := re.FindStringSubmatch(path); m != nil { caps := subexpMap(re, m) sha := caps["sha"] if !shaHexRe.MatchString(sha) { return nil, nil, "", errors.New("提交哈希非法") } return src, caps, sha, nil } } } re, err := regexp.Compile(src.ListPath) if err != nil { continue } if m := re.FindStringSubmatch(path); m != nil { return src, subexpMap(re, m), "", nil } } return nil, nil, "", errors.New("地址不符或主机未配置") } func subexpMap(re *regexp.Regexp, m []string) map[string]string { out := map[string]string{} for i, name := range re.SubexpNames() { if i == 0 || name == "" || i >= len(m) { continue } out[name] = m[i] } return out } func expandTemplate(tpl, host string, caps map[string]string, page int) string { r := strings.NewReplacer( "{host}", host, "{owner}", caps["owner"], "{repo}", caps["repo"], "{ref}", caps["ref"], "{sha}", caps["sha"], "{page}", strconv.Itoa(page), ) return r.Replace(tpl) } func expandAPITemplate(tpl, host string, caps map[string]string, page int) (string, error) { s := expandTemplate(tpl, host, caps, page) u, err := url.Parse(s) if err != nil { return "", errors.New("API URL 无效") } if err := assertSafeHTTPSURL(u); err != nil { return "", err } return u.String(), nil } func fetchSingleCommit(client *http.Client, src *TimelineGitSourceCfg, host string, caps map[string]string, sha string) (*TimelineGitItem, error) { caps = copyCaps(caps) caps["sha"] = sha apiTpl := src.CommitAPI if apiTpl == "" { apiTpl = strings.TrimSuffix(src.APIURL, "/") + "/{sha}" } apiURL, err := expandAPITemplate(apiTpl, host, caps, 1) if err != nil { return nil, err } headers := http.Header{} for k, v := range src.Headers { if allowedAdapterHeaders[strings.ToLower(k)] { headers.Set(k, v) } } if headers.Get("User-Agent") == "" { headers.Set("User-Agent", "jiang13-bbs") } body, _, status, err := httpGetLimited(client, apiURL, headers) if err != nil { return nil, err } if status < 200 || status >= 300 { return nil, fmt.Errorf("远端返回 %d", status) } var obj any if err := json.Unmarshal(body, &obj); err != nil { return nil, errors.New("响应非 JSON") } item, ok := mapCommitObject(obj, src, host) if !ok { return nil, errors.New("无法解析提交") } return &item, nil } func copyCaps(in map[string]string) map[string]string { out := make(map[string]string, len(in)) for k, v := range in { out[k] = v } return out } func parseCommitListJSON(body []byte, src *TimelineGitSourceCfg, host string) ([]TimelineGitItem, error) { var root any if err := json.Unmarshal(body, &root); err != nil { return nil, errors.New("响应非 JSON") } arr, ok := root.([]any) if !ok { if m, isMap := root.(map[string]any); isMap { if c, ok := m["commits"].([]any); ok { arr = c } } } if arr == nil { return nil, errors.New("响应不是提交列表") } var out []TimelineGitItem for _, el := range arr { item, ok := mapCommitObject(el, src, host) if !ok { continue } out = append(out, item) } return out, nil } func mapCommitObject(el any, src *TimelineGitSourceCfg, host string) (TimelineGitItem, bool) { sha := jsonPathString(el, src.Item.SHA) dateRaw := jsonPathString(el, src.Item.Date) msg := jsonPathString(el, src.Item.Message) srcURL := jsonPathString(el, src.Item.SourceURL) if msg == "" && sha == "" { return TimelineGitItem{}, false } sha = sanitizeSHA(sha) title, body := splitCommitMessage(msg) title = sanitizeTimelinePlain(title, timelineTitleMax) body = sanitizeTimelinePlain(body, timelineBodyMax) title = neutralizeDirectivePlain(title) body = neutralizeDirectivePlain(body) date := parseCommitDate(dateRaw) srcURL = sanitizeSourceURL(srcURL, host) return TimelineGitItem{ Date: date, Title: title, Body: body, SourceURL: srcURL, SHA: sha, }, true } func jsonPathString(root any, path string) string { if path == "" || root == nil { return "" } cur := root for _, part := range strings.Split(path, ".") { m, ok := cur.(map[string]any) if !ok { return "" } cur, ok = m[part] if !ok { return "" } } switch v := cur.(type) { case string: return v case float64: return strconv.FormatInt(int64(v), 10) case json.Number: return v.String() default: return "" } } func splitCommitMessage(msg string) (title, body string) { msg = strings.ReplaceAll(msg, "\r\n", "\n") msg = strings.TrimSpace(msg) if msg == "" { return "提交", "" } parts := strings.SplitN(msg, "\n", 2) title = strings.TrimSpace(parts[0]) if title == "" { title = "提交" } if len(parts) > 1 { body = stripCommitTrailers(strings.TrimSpace(parts[1])) } return title, body } // stripCommitTrailers 去掉 Co-authored-by 等尾部 trailer 块(及前导空行)。 func stripCommitTrailers(body string) string { if body == "" { return "" } lines := strings.Split(body, "\n") // 从末尾向前:连续 trailer / 空行;遇到非 trailer 正文则停 end := len(lines) for end > 0 { t := strings.TrimSpace(lines[end-1]) if t == "" || isCommitTrailerLine(t) { end-- continue } break } // 若尾部有 trailer,再去掉其前的空行分隔 for end > 0 && strings.TrimSpace(lines[end-1]) == "" { end-- } return strings.TrimSpace(strings.Join(lines[:end], "\n")) } func isCommitTrailerLine(t string) bool { lower := strings.ToLower(t) prefixes := []string{ "co-authored-by:", "signed-off-by:", "reviewed-by:", "acked-by:", "tested-by:", "reported-by:", "suggested-by:", "helped-by:", } for _, p := range prefixes { if strings.HasPrefix(lower, p) { return true } } return false } func sanitizeSHA(s string) string { s = strings.TrimSpace(s) if !shaHexRe.MatchString(s) { return "" } if len(s) > 64 { return s[:64] } return s } func sanitizeTimelinePlain(s string, max int) string { var b strings.Builder b.Grow(len(s)) for _, r := range s { if r == 0 || (r < 0x20 && r != '\n' && r != '\t') || r == 0x7f { continue } if r >= 0x202A && r <= 0x202E { continue } if r >= 0x2066 && r <= 0x2069 { continue } if unicode.Is(unicode.Cs, r) { continue } b.WriteRune(r) } out := b.String() if utf8.RuneCountInString(out) > max { runes := []rune(out) out = string(runes[:max]) } return out } func neutralizeDirectivePlain(s string) string { lines := strings.Split(s, "\n") for i, line := range lines { t := strings.TrimSpace(line) lower := strings.ToLower(t) if lower == "[/hide]" || lower == "[/timeline]" || lower == "[timeline]" || strings.HasPrefix(lower, "[hide") || strings.HasPrefix(t, "