package service import ( "net/url" "strconv" "strings" "time" "unicode/utf8" "github.com/freefire/jiang13-bbs/model" "gorm.io/gorm" ) // ============ 管理端聚合查询(visit_events SQL 聚合;长趋势用 site_daily_stats) ============ // VisitDayPoint 日趋势点 type VisitDayPoint struct { Date string `json:"date"` PV int64 `json:"pv"` UV int64 `json:"uv"` } // VisitNamedCount 命名计数(设备/来源/爬虫等) type VisitNamedCount struct { Key string `json:"key"` Label string `json:"label"` Count int64 `json:"count"` } // VisitPageRow 受访页面行 type VisitPageRow struct { Path string `json:"path"` PV int64 `json:"pv"` UV int64 `json:"uv"` Share float64 `json:"share"` // 占区间总 PV 比例(0–100) } // VisitOverview 概览 Tab type VisitOverview struct { RangeDays int `json:"range_days"` Trend []VisitDayPoint `json:"trend"` TodayPV int64 `json:"today_pv"` TodayUV int64 `json:"today_uv"` YesterdayPV int64 `json:"yesterday_pv"` YesterdayUV int64 `json:"yesterday_uv"` LoggedInPV int64 `json:"logged_in_pv"` // 今日登录 PV Devices []VisitNamedCount `json:"devices"` Sources []VisitNamedCount `json:"sources"` TopPages []VisitPageRow `json:"top_pages"` } // VisitMinutePoint 分钟级 PV 点 type VisitMinutePoint struct { Minute string `json:"minute"` // YYYY-MM-DD HH24:MI PV int64 `json:"pv"` } // VisitEventItem 明细列表行 type VisitEventItem struct { ID uint64 `json:"id"` CreatedAt time.Time `json:"created_at"` Kind string `json:"kind"` Path string `json:"path"` RefHost string `json:"ref_host"` RefKind string `json:"ref_kind"` UserID uint `json:"user_id"` Username string `json:"username"` IP string `json:"ip"` Device string `json:"device"` OS string `json:"os"` Browser string `json:"browser"` Bot string `json:"bot"` Status int `json:"status"` } // VisitRealtime 实时 Tab type VisitRealtime struct { Minutes []VisitMinutePoint `json:"minutes"` PV5min int64 `json:"pv_5min"` PV30min int64 `json:"pv_30min"` Online int64 `json:"online"` TopPages []VisitNamedCount `json:"top_pages"` Recent []VisitEventItem `json:"recent"` } // VisitSourcesKindDay 来源按日(kind 分列行) type VisitSourcesKindDay struct { Date string `json:"date"` Items []VisitNamedCount `json:"items"` } // VisitSources 来源分析 Tab type VisitSources struct { RangeDays int `json:"range_days"` Kinds []VisitNamedCount `json:"kinds"` SearchTerms []VisitNamedCount `json:"search_terms"` // 站内搜索词 Top N(/?q= 结果页浏览) External []VisitNamedCount `json:"external"` Trend []VisitSourcesKindDay `json:"trend"` } // VisitPages 受访页面 Tab type VisitPages struct { RangeDays int `json:"range_days"` Total int64 `json:"total"` // 命中的 path 数 Items []VisitPageRow `json:"items"` } // VisitVisitors 访客画像 Tab type VisitVisitors struct { RangeDays int `json:"range_days"` Devices []VisitNamedCount `json:"devices"` OSes []VisitNamedCount `json:"oses"` Browsers []VisitNamedCount `json:"browsers"` LoggedInPV int64 `json:"logged_in_pv"` GuestPV int64 `json:"guest_pv"` NewVisitors int64 `json:"new_visitors"` Returning int64 `json:"returning"` Hourly []VisitNamedCount `json:"hourly"` // 0–23 时段 PV } // VisitSecurity 爬虫与安全 Tab type VisitSecurity struct { RangeDays int `json:"range_days"` Bots []VisitNamedCount `json:"bots"` BotTrend []VisitDayPoint `json:"bot_trend"` ProbeStatus []VisitNamedCount `json:"probe_status"` ProbeIPs []VisitNamedCount `json:"probe_ips"` LoginFailed int64 `json:"login_failed"` // 近 N 日登录失败次数 FailedUsers int64 `json:"failed_users"` // 涉及用户名数 } // VisitEventFilter 明细筛选 type VisitEventFilter struct { Kind string Bot string Path string IP string Start time.Time End time.Time Page int Size int } // VisitEventsResult 明细分页结果 type VisitEventsResult struct { Total int64 `json:"total"` Items []VisitEventItem `json:"items"` } // VisitSettingsPayload 设置 Tab(开关 + 保留天数 + 容量) type VisitSettingsPayload struct { Enabled bool `json:"enabled"` RetentionDays int `json:"retention_days"` Usage VisitUsage `json:"usage"` Dropped uint64 `json:"dropped"` } type pvUVRow struct { PV int64 `gorm:"column:pv"` UV int64 `gorm:"column:uv"` } func visitRangeStart(days int) time.Time { return todayDate().AddDate(0, 0, -(days - 1)) } // groupCounts 通用 GROUP BY 计数(expr 为聚合表达式),按计数降序取前 limit 条。 // extraSQL 追加到 WHERE(如 " AND ref_kind = 'search'")。 func (s *VisitStatsService) groupCounts(start time.Time, expr, kind, extraSQL string, limit int) ([]VisitNamedCount, error) { var rows []struct { Key string Count int64 } q := s.db.Model(&model.VisitEvent{}). Select(expr+" AS key, COUNT(*) AS count"). Where("created_at >= ?", start) if kind != "" { q = q.Where("kind = ?", kind) } if extraSQL != "" { q = q.Where(extraSQL) } if err := q.Group("key").Order("count DESC").Limit(limit).Scan(&rows).Error; err != nil { return nil, err } out := make([]VisitNamedCount, 0, len(rows)) for _, r := range rows { out = append(out, VisitNamedCount{Key: r.Key, Label: r.Key, Count: r.Count}) } return out, nil } // Overview 概览:趋势(site_daily_stats)+ 今日/昨日 + 设备/来源/热门页面 func (s *VisitStatsService) Overview(rangeQ string) (*VisitOverview, error) { days := parseRangeDays(rangeQ) start := visitRangeStart(days) out := &VisitOverview{RangeDays: days} // 日趋势:日聚合表(每天一行,不随明细清理丢失) if err := s.db.Model(&model.SiteDailyStats{}). Select("to_char(date, 'YYYY-MM-DD') AS date, pv, uv"). Where("date >= ?", start). Order("date ASC").Scan(&out.Trend).Error; err != nil { return nil, err } // 空结果时 GORM 不赋值,nil 切片会序列化成 JSON null(前端期望 []) if out.Trend == nil { out.Trend = []VisitDayPoint{} } var today pvUVRow if err := s.db.Model(&model.SiteDailyStats{}). Select("COALESCE(SUM(pv),0) AS pv, COALESCE(SUM(uv),0) AS uv"). Where("date = ?", todayDate()).Scan(&today).Error; err != nil { return nil, err } out.TodayPV, out.TodayUV = today.PV, today.UV var yesterday pvUVRow if err := s.db.Model(&model.SiteDailyStats{}). Select("COALESCE(SUM(pv),0) AS pv, COALESCE(SUM(uv),0) AS uv"). Where("date = ?", todayDate().AddDate(0, 0, -1)).Scan(&yesterday).Error; err != nil { return nil, err } out.YesterdayPV, out.YesterdayUV = yesterday.PV, yesterday.UV if err := s.db.Model(&model.SiteDailyStats{}). Select("COALESCE(SUM(logged_in_pv),0) AS logged_in_pv"). Where("date = ?", todayDate()).Scan(&out.LoggedInPV).Error; err != nil { return nil, err } var err error if out.Devices, err = s.groupCounts(start, "device", model.VisitKindPageview, "", 8); err != nil { return nil, err } if out.Sources, err = s.groupCounts(start, "ref_kind", model.VisitKindPageview, "", 8); err != nil { return nil, err } if out.TopPages, err = s.pageAgg(start, "", 1, 10); err != nil { return nil, err } return out, nil } // pageAgg path 聚合 PV/UV/占比(q 为 path 模糊匹配;page 从 1 起) func (s *VisitStatsService) pageAgg(start time.Time, q string, page, size int) ([]VisitPageRow, error) { var totalPV int64 if err := s.db.Model(&model.VisitEvent{}). Where("created_at >= ? AND kind = ?", start, model.VisitKindPageview). Scopes(pathFilter(q)). Count(&totalPV).Error; err != nil { return nil, err } if totalPV == 0 { return []VisitPageRow{}, nil } var rows []VisitPageRow if err := s.db.Model(&model.VisitEvent{}). Select("path, COUNT(*) AS pv, COUNT(DISTINCT vid_hash) AS uv"). Where("created_at >= ? AND kind = ?", start, model.VisitKindPageview). Scopes(pathFilter(q)). Group("path").Order("pv DESC"). Offset((page - 1) * size).Limit(size).Scan(&rows).Error; err != nil { return nil, err } for i := range rows { rows[i].Share = float64(rows[i].PV) * 100 / float64(totalPV) } return rows, nil } func pathFilter(q string) func(*gorm.DB) *gorm.DB { if q == "" { return func(db *gorm.DB) *gorm.DB { return db } } return func(db *gorm.DB) *gorm.DB { return db.Where("path ILIKE ?", "%"+q+"%") } } // Realtime 实时:近 60 分钟分钟级 PV + 在线 + 近 5 分钟热门路径 + 最新 20 条 func (s *VisitStatsService) Realtime() (*VisitRealtime, error) { out := &VisitRealtime{} now := time.Now() if err := s.db.Model(&model.VisitEvent{}). Select("to_char(created_at, 'YYYY-MM-DD HH24:MI') AS minute, COUNT(*) AS pv"). Where("created_at >= ? AND kind = ?", now.Add(-time.Hour), model.VisitKindPageview). Group("minute").Order("minute ASC").Scan(&out.Minutes).Error; err != nil { return nil, err } if out.Minutes == nil { out.Minutes = []VisitMinutePoint{} } if err := s.db.Model(&model.VisitEvent{}). Where("created_at >= ? AND kind = ?", now.Add(-5*time.Minute), model.VisitKindPageview). Count(&out.PV5min).Error; err != nil { return nil, err } if err := s.db.Model(&model.VisitEvent{}). Where("created_at >= ? AND kind = ?", now.Add(-30*time.Minute), model.VisitKindPageview). Count(&out.PV30min).Error; err != nil { return nil, err } // 在线口径与首页概览一致:users.last_seen_at 近 5 分钟且未封禁 if err := s.db.Model(&model.User{}). Where("last_seen_at IS NOT NULL AND last_seen_at >= ? AND banned = ?", now.Add(-5*time.Minute), false). Count(&out.Online).Error; err != nil { return nil, err } if err := s.db.Model(&model.VisitEvent{}). Select("path AS key, COUNT(*) AS count"). Where("created_at >= ? AND kind = ?", now.Add(-5*time.Minute), model.VisitKindPageview). Group("path").Order("count DESC").Limit(10). Scan(&out.TopPages).Error; err != nil { return nil, err } for i := range out.TopPages { out.TopPages[i].Label = out.TopPages[i].Key } if out.TopPages == nil { out.TopPages = []VisitNamedCount{} } res, err := s.Events(VisitEventFilter{Kind: model.VisitKindPageview, Page: 1, Size: 20}) if err != nil { return nil, err } out.Recent = res.Items return out, nil } // Sources 来源分析 func (s *VisitStatsService) Sources(rangeQ string) (*VisitSources, error) { days := parseRangeDays(rangeQ) start := visitRangeStart(days) out := &VisitSources{RangeDays: days} var err error if out.Kinds, err = s.groupCounts(start, "ref_kind", model.VisitKindPageview, "", 8); err != nil { return nil, err } if out.SearchTerms, err = s.siteSearchTerms(start, 10); err != nil { return nil, err } if out.External, err = s.groupCounts(start, "ref_host", model.VisitKindPageview, "ref_kind = '"+RefKindExternal+"'", 20); err != nil { return nil, err } if out.Trend, err = s.SourcesTrend(rangeQ); err != nil { return nil, err } return out, nil } // siteSearchTerms 站内搜索词:聚合搜索结果页(/?q=…,path 记录完整 query)的浏览量。 // 词以 URL 编码存于 path,按编码串分组后解码展示(站内 encodeURIComponent 编码确定性强)。 func (s *VisitStatsService) siteSearchTerms(start time.Time, limit int) ([]VisitNamedCount, error) { const expr = "substring(path from '[?&]q=([^&]*)')" var rows []struct { Key string Count int64 } if err := s.db.Model(&model.VisitEvent{}). Select(expr+" AS key, COUNT(*) AS count"). Where("created_at >= ? AND kind = ?", start, model.VisitKindPageview). Where(expr + " IS NOT NULL AND " + expr + " <> ''"). Group("key").Order("count DESC").Limit(limit). Scan(&rows).Error; err != nil { return nil, err } out := make([]VisitNamedCount, 0, len(rows)) for _, r := range rows { term := decodeSearchTerm(r.Key) if runes := []rune(term); len(runes) > 30 { term = string(runes[:30]) + "…" } out = append(out, VisitNamedCount{Key: term, Label: term, Count: r.Count}) } return out, nil } // decodeSearchTerm 解码 path 里 q 参数值(URL 编码)。path 512 字节截断可能: // 1) 把 %XX 转义拦腰切断(孤 % / 一位十六进制)→ 整串解码失败,剥掉残缺尾部重试; // 2) 恰好切在多字节字符的转义序列中间 → 解码「成功」但尾部是半截字符,去掉非法 UTF-8 尾巴。 func decodeSearchTerm(raw string) string { term := strings.ReplaceAll(raw, "+", " ") dec, err := url.QueryUnescape(term) if err != nil { // 残缺转义(如结尾孤 % 或一位十六进制):剥掉最后一个 % 起的尾部重试 if i := strings.LastIndex(term, "%"); i > 0 { dec, err = url.QueryUnescape(term[:i]) } } if err != nil { return term } return validUTF8Prefix(dec) } // validUTF8Prefix 返回 s 的最长合法 UTF-8 前缀(截断产生的半截多字节字符会被去掉; // 真实的 U+FFFD 字符合法保留,与非法字节可区分)。 func validUTF8Prefix(s string) string { for i := 0; i < len(s); { r, size := utf8.DecodeRuneInString(s[i:]) if r == utf8.RuneError && size == 1 { return s[:i] } i += size } return s } // SourcesTrend 来源按日明细(kind 分列行) func (s *VisitStatsService) SourcesTrend(rangeQ string) ([]VisitSourcesKindDay, error) { days := parseRangeDays(rangeQ) start := visitRangeStart(days) var rows []struct { Date string RefKind string Count int64 } if err := s.db.Model(&model.VisitEvent{}). Select("to_char(created_at, 'YYYY-MM-DD') AS date, ref_kind, COUNT(*) AS count"). Where("created_at >= ? AND kind = ?", start, model.VisitKindPageview). Group("date, ref_kind").Order("date ASC").Scan(&rows).Error; err != nil { return nil, err } byDay := make(map[string][]VisitNamedCount) var dates []string seen := make(map[string]struct{}) for _, r := range rows { if _, ok := seen[r.Date]; !ok { seen[r.Date] = struct{}{} dates = append(dates, r.Date) } byDay[r.Date] = append(byDay[r.Date], VisitNamedCount{Key: r.RefKind, Label: r.RefKind, Count: r.Count}) } out := make([]VisitSourcesKindDay, 0, len(dates)) for _, d := range dates { out = append(out, VisitSourcesKindDay{Date: d, Items: byDay[d]}) } return out, nil } // Pages 受访页面(分页 + 模糊) func (s *VisitStatsService) Pages(rangeQ, q string, page, size int) (*VisitPages, error) { days := parseRangeDays(rangeQ) start := visitRangeStart(days) if page < 1 { page = 1 } if size < 1 || size > 100 { size = 20 } out := &VisitPages{RangeDays: days} if err := s.db.Model(&model.VisitEvent{}). Where("created_at >= ? AND kind = ?", start, model.VisitKindPageview). Scopes(pathFilter(q)). Distinct("path").Count(&out.Total).Error; err != nil { return nil, err } items, err := s.pageAgg(start, q, page, size) if err != nil { return nil, err } if items == nil { items = []VisitPageRow{} } out.Items = items return out, nil } // Visitors 访客画像 func (s *VisitStatsService) Visitors(rangeQ string) (*VisitVisitors, error) { days := parseRangeDays(rangeQ) start := visitRangeStart(days) out := &VisitVisitors{RangeDays: days} var err error if out.Devices, err = s.groupCounts(start, "device", model.VisitKindPageview, "", 8); err != nil { return nil, err } if out.OSes, err = s.groupCounts(start, "os", model.VisitKindPageview, "", 10); err != nil { return nil, err } if out.Browsers, err = s.groupCounts(start, "browser", model.VisitKindPageview, "", 10); err != nil { return nil, err } var lg pvUVRow if err := s.db.Model(&model.VisitEvent{}). Select("COALESCE(SUM(CASE WHEN user_id > 0 THEN 1 ELSE 0 END),0) AS pv, COALESCE(SUM(CASE WHEN user_id = 0 THEN 1 ELSE 0 END),0) AS uv"). Where("created_at >= ? AND kind = ?", start, model.VisitKindPageview). Scan(&lg).Error; err != nil { return nil, err } out.LoggedInPV, out.GuestPV = lg.PV, lg.UV // 新老访客:区间内 vid 首次出现时间晚于区间起点视为新访客 var uvRows []struct { VidHash string FirstAt time.Time `gorm:"column:first_at"` } if err := s.db.Model(&model.VisitEvent{}). Select("vid_hash, MIN(created_at) AS first_at"). Where("created_at >= ? AND kind = ? AND vid_hash <> ''", start, model.VisitKindPageview). Group("vid_hash").Scan(&uvRows).Error; err != nil { return nil, err } for _, r := range uvRows { if r.FirstAt.After(start) { out.NewVisitors++ } else { out.Returning++ } } // 24 小时时段 PV(区间合计) var hours []struct { Hour int Count int64 } if err := s.db.Model(&model.VisitEvent{}). Select("EXTRACT(HOUR FROM created_at)::int AS hour, COUNT(*) AS count"). Where("created_at >= ? AND kind = ?", start, model.VisitKindPageview). Group("hour").Order("hour ASC").Scan(&hours).Error; err != nil { return nil, err } out.Hourly = make([]VisitNamedCount, 24) for i := range out.Hourly { label := strconv.Itoa(i) if i < 10 { label = "0" + label } out.Hourly[i].Key = label out.Hourly[i].Label = label + " 时" } for _, h := range hours { if h.Hour >= 0 && h.Hour < 24 { out.Hourly[h.Hour].Count = h.Count } } return out, nil } // Security 爬虫与安全 func (s *VisitStatsService) Security(rangeQ string) (*VisitSecurity, error) { days := parseRangeDays(rangeQ) start := visitRangeStart(days) out := &VisitSecurity{RangeDays: days} var err error if out.Bots, err = s.groupCounts(start, "bot", model.VisitKindBot, "", 20); err != nil { return nil, err } if err := s.db.Model(&model.VisitEvent{}). Select("to_char(created_at, 'YYYY-MM-DD') AS date, COUNT(*) AS pv"). Where("created_at >= ? AND kind = ?", start, model.VisitKindBot). Group("date").Order("date ASC").Scan(&out.BotTrend).Error; err != nil { return nil, err } if out.BotTrend == nil { out.BotTrend = []VisitDayPoint{} } if out.ProbeStatus, err = s.groupCounts(start, "status::text", model.VisitKindProbe, "", 15); err != nil { return nil, err } if out.ProbeIPs, err = s.groupCounts(start, "ip", model.VisitKindProbe, "", 20); err != nil { return nil, err } if err := s.db.Model(&model.LoginLog{}). Where("success = ? AND created_at >= ?", false, start). Count(&out.LoginFailed).Error; err != nil { return nil, err } if err := s.db.Raw(`SELECT COUNT(DISTINCT username) FROM login_logs WHERE success = ? AND created_at >= ?`, false, start).Scan(&out.FailedUsers).Error; err != nil { return nil, err } return out, nil } // Events 访问明细(筛选 + 分页 + 补用户昵称) func (s *VisitStatsService) Events(f VisitEventFilter) (*VisitEventsResult, error) { if f.Page < 1 { f.Page = 1 } if f.Size < 1 || f.Size > 200 { f.Size = 50 } apply := func(db *gorm.DB) *gorm.DB { if f.Kind != "" { db = db.Where("kind = ?", f.Kind) } if f.Bot != "" { db = db.Where("bot = ?", f.Bot) } if f.Path != "" { db = db.Where("path ILIKE ?", "%"+f.Path+"%") } if f.IP != "" { db = db.Where("ip = ?", f.IP) } if !f.Start.IsZero() { db = db.Where("created_at >= ?", f.Start) } if !f.End.IsZero() { db = db.Where("created_at < ?", f.End) } return db } out := &VisitEventsResult{} if err := apply(s.db.Model(&model.VisitEvent{})).Count(&out.Total).Error; err != nil { return nil, err } var events []model.VisitEvent if err := apply(s.db.Model(&model.VisitEvent{})). Order("id DESC"). Offset((f.Page - 1) * f.Size).Limit(f.Size). Find(&events).Error; err != nil { return nil, err } // 批量补昵称 userIDs := make([]uint, 0, len(events)) for _, e := range events { if e.UserID > 0 { userIDs = append(userIDs, e.UserID) } } nameMap := make(map[uint]string, len(userIDs)) if len(userIDs) > 0 { var users []model.User if err := s.db.Select("id, username, nickname"). Where("id IN ?", userIDs).Find(&users).Error; err != nil { return nil, err } for _, u := range users { nameMap[u.ID] = u.Nickname } } out.Items = make([]VisitEventItem, 0, len(events)) for _, e := range events { out.Items = append(out.Items, VisitEventItem{ ID: e.ID, CreatedAt: e.CreatedAt, Kind: e.Kind, Path: e.Path, RefHost: e.RefHost, RefKind: e.RefKind, UserID: e.UserID, Username: nameMap[e.UserID], IP: e.IP, Device: e.Device, OS: e.OS, Browser: e.Browser, Bot: e.Bot, Status: e.Status, }) } return out, nil } // SettingsPayload 设置 Tab 数据 func (s *VisitStatsService) SettingsPayload() (*VisitSettingsPayload, error) { out := &VisitSettingsPayload{ Enabled: s.Enabled(), RetentionDays: s.retentionDays(), Dropped: s.DroppedCount(), } u, err := s.Usage() if err != nil { return nil, err } out.Usage = u return out, nil } // SaveSettings 保存采集开关与保留天数 func (s *VisitStatsService) SaveSettings(enabled bool, retentionDays int) error { if err := s.setting.SetAnalyticsEnabled(enabled); err != nil { return err } return s.setting.SetAnalyticsRetentionDays(retentionDays) }