package service import ( "encoding/json" "errors" "fmt" "net/http" "net/url" "regexp" "strings" ) const ( timelineReleaseMaxPages = 10 timelineReleaseMaxItems = 100 ) // timelineReleaseSource 内置 Release 适配(Gitea + GitHub) type timelineReleaseSource struct { id string host string // 精确主机或 "*"(通配) listPath string // releases 页正则 apiURL string // API 模板 query map[string]string headers map[string]string pagination string // link_header | query_page dateField string // 优先取的字段:published_at / created_at } var timelineReleaseSources = []timelineReleaseSource{ { id: "github_release", host: "github.com", listPath: `^/(?P[^/]+)/(?P[^/]+)/releases/?$`, apiURL: "https://api.github.com/repos/{owner}/{repo}/releases", query: map[string]string{"per_page": "100", "page": "{page}"}, headers: map[string]string{"User-Agent": "jiang13-bbs", "Accept": "application/vnd.github+json"}, pagination: "link_header", dateField: "published_at", }, { id: "gitea_release", host: "*", listPath: `^/(?P[^/]+)/(?P[^/]+)/releases/?$`, apiURL: "https://{host}/api/v1/repos/{owner}/{repo}/releases", query: map[string]string{"limit": "50", "page": "{page}"}, headers: map[string]string{"User-Agent": "jiang13-bbs", "Accept": "application/json"}, pagination: "query_page", dateField: "created_at", }, } // matchReleaseSource 按 host + path 匹配内置 release 适配 func matchReleaseSource(host, path string) (*timelineReleaseSource, map[string]string, error) { for i := range timelineReleaseSources { src := &timelineReleaseSources[i] if src.host != "*" && !strings.EqualFold(src.host, host) { continue } if src.host == "*" && strings.EqualFold(host, "github.com") { continue } re, err := regexp.Compile(src.listPath) if err != nil { continue } if m := re.FindStringSubmatch(path); m != nil { return src, subexpMap(re, m), nil } } return nil, nil, errors.New("地址不符或主机未配置") } // ImportTimelineFromReleases 按 releases 页 URL 拉取并解析发布记录 func (s *SettingService) ImportTimelineFromReleases(urls []string) (*TimelineGitImportResult, error) { cleanURLs := make([]string, 0, len(urls)) for _, u := range urls { u = strings.TrimSpace(u) if u != "" { cleanURLs = append(cleanURLs, u) } } if len(cleanURLs) == 0 { return nil, errors.New("请提供至少一条 URL") } if len(cleanURLs) > 20 { return nil, errors.New("一次最多 20 条 URL") } client := &http.Client{ Timeout: timelineGitHTTPTimeout, Transport: &http.Transport{ DialContext: publicOnlyDial, TLSHandshakeTimeout: timelineGitHTTPTimeout, ForceAttemptHTTP2: true, }, CheckRedirect: func(req *http.Request, via []*http.Request) error { if len(via) >= 3 { return errors.New("重定向过多") } if err := assertSafeHTTPSURL(req.URL); err != nil { return err } return nil }, } seenURL := map[string]bool{} var items []TimelineGitItem var failMsgs []string truncated := false for _, rawURL := range cleanURLs { part, partTrunc, err := s.importOneReleaseURL(client, rawURL, seenURL, timelineReleaseMaxItems-len(items)) if err != nil { failMsgs = append(failMsgs, fmt.Sprintf("%s:%s", truncateTimelineStr(rawURL, 80), err.Error())) continue } items = append(items, part...) if partTrunc { truncated = true } if len(items) >= timelineReleaseMaxItems { truncated = true break } } out := &TimelineGitImportResult{Items: items} if truncated { out.Warning = fmt.Sprintf("已达上限(最多 %d 条),可再贴后续页 URL", timelineReleaseMaxItems) } if len(failMsgs) > 0 { out.Error = strings.Join(failMsgs, ";") } if len(items) == 0 && out.Error == "" { out.Error = "未能解析出 Release" } return out, nil } func (s *SettingService) importOneReleaseURL( client *http.Client, rawURL string, seenURL map[string]bool, remain int, ) ([]TimelineGitItem, bool, error) { if remain <= 0 { return nil, true, nil } u, err := url.Parse(rawURL) if err != nil || u.Scheme == "" || u.Host == "" { return nil, false, errors.New("URL 无效") } if err := assertSafeHTTPSURL(u); err != nil { return nil, false, err } host := strings.ToLower(u.Hostname()) path := u.EscapedPath() if path == "" { path = "/" } src, caps, err := matchReleaseSource(host, path) if err != nil { return nil, false, err } if !repoNameRe.MatchString(caps["owner"]) || !repoNameRe.MatchString(caps["repo"]) { return nil, false, errors.New("仓库名非法") } startPage := 1 if p := u.Query().Get("page"); p != "" { if n, e := parseIntPage(p); e == nil && n >= 1 { startPage = n } } var out []TimelineGitItem truncated := false page := startPage maxPages := timelineReleaseMaxPages pagesDone := 0 nextURL := "" for pagesDone < maxPages && len(out) < remain { var apiURL string if nextURL != "" { apiURL = nextURL nextURL = "" } else { apiURL = expandTemplate(src.apiURL, host, caps, page) q := url.Values{} for k, v := range src.query { q.Set(k, expandTemplate(v, host, caps, page)) } parsed, e := url.Parse(apiURL) if e != nil { return out, truncated, errors.New("API URL 无效") } if len(q) > 0 { existing := parsed.Query() for k, vs := range q { existing.Set(k, vs[0]) } parsed.RawQuery = existing.Encode() } apiURL = parsed.String() } parsedAPI, err := url.Parse(apiURL) if err != nil { return out, truncated, errors.New("API URL 无效") } if err := assertSafeHTTPSURL(parsedAPI); err != nil { return out, truncated, err } headers := http.Header{} for k, v := range src.headers { if allowedAdapterHeaders[strings.ToLower(k)] { headers.Set(k, v) } } if headers.Get("User-Agent") == "" { headers.Set("User-Agent", "jiang13-bbs") } body, linkNext, status, err := httpGetLimited(client, parsedAPI.String(), headers) if err != nil { return out, truncated, err } if status == 404 || status == 401 || status == 403 { return out, truncated, errors.New("无法读取该仓库(私有、不存在或无权访问)") } if status == 429 { return out, truncated, errors.New("远端限流,请稍后再试") } if status < 200 || status >= 300 { return out, truncated, fmt.Errorf("远端返回 %d", status) } pageItems, err := parseReleaseListJSON(body, src, host) if err != nil { return out, truncated, err } if len(pageItems) == 0 { break } for _, it := range pageItems { if it.SourceURL != "" && seenURL[it.SourceURL] { continue } if it.SourceURL != "" { seenURL[it.SourceURL] = true } out = append(out, it) if len(out) >= remain { truncated = true break } } pagesDone++ if src.pagination == "link_header" && linkNext != "" { nu, e := url.Parse(linkNext) if e != nil || assertSafeHTTPSURL(nu) != nil { break } nextURL = nu.String() } else if src.pagination == "query_page" { page++ } else { break } } if pagesDone >= maxPages { truncated = true } return out, truncated, nil } func parseReleaseListJSON(body []byte, src *timelineReleaseSource, host string) ([]TimelineGitItem, error) { var root any if err := json.Unmarshal(body, &root); err != nil { return nil, errors.New("响应非 JSON") } arr, ok := root.([]any) if !ok { return nil, errors.New("响应不是 Release 列表") } var out []TimelineGitItem for _, el := range arr { item, ok := mapReleaseObject(el, src, host) if !ok { continue } out = append(out, item) } return out, nil } func mapReleaseObject(el any, src *timelineReleaseSource, host string) (TimelineGitItem, bool) { m, ok := el.(map[string]any) if !ok { return TimelineGitItem{}, false } // 跳过草稿 if draft, ok := m["draft"].(bool); ok && draft { return TimelineGitItem{}, false } name := jsonStringField(m, "name") tag := jsonStringField(m, "tag_name") title := strings.TrimSpace(name) if title == "" { title = strings.TrimSpace(tag) } if title == "" { return TimelineGitItem{}, false } body := jsonStringField(m, "body") dateRaw := jsonStringField(m, src.dateField) if dateRaw == "" { // 兜底用 created_at dateRaw = jsonStringField(m, "created_at") } srcURL := jsonStringField(m, "html_url") title = sanitizeTimelinePlain(title, timelineTitleMax) body = sanitizeTimelinePlain(body, timelineBodyMax) title = neutralizeDirectivePlain(title) body = neutralizeDirectivePlain(body) date := parseCommitDate(dateRaw) srcURL = sanitizeSourceURL(srcURL, host) return TimelineGitItem{ Date: date, Title: title, Body: body, SourceURL: srcURL, }, true } // jsonStringField 从 map[string]any 取字符串字段,兼容 string/number func jsonStringField(m map[string]any, key string) string { v, ok := m[key] if !ok || v == nil { return "" } switch x := v.(type) { case string: return x case float64: return fmt.Sprintf("%v", x) case json.Number: return x.String() default: return fmt.Sprintf("%v", x) } } func parseIntPage(s string) (int, error) { n := 0 for _, r := range s { if r < '0' || r > '9' { return 0, errors.New("page 非数字") } n = n*10 + int(r-'0') if n > 1_000_000 { return 0, errors.New("page 过大") } } return n, nil }