feat: 站点访问统计与可配置侧边栏组件

- 新增访问统计:visitstats 服务与查询、visit_capture 中间件、爬虫/来源识别、管理端 analytics 页面
- 新增侧边栏:sidebar 服务与 handler、管理端 sidebar 配置页、前端侧边栏组件
- 新增 widget 嵌入代码生成(widgetCode)与 widgetApi
- 遥测逻辑迁移至 handler 层,删除 service/telemetry
- env 示例补充 Docker 部署可信代理 IP 说明
This commit is contained in:
2026-09-24 18:53:45 +08:00
parent 3e55b5d230
commit 7b327bc8cc
60 changed files with 6539 additions and 572 deletions

View File

@@ -0,0 +1,616 @@
package service
import (
"strconv"
"time"
"github.com/freefire/jiang13-bbs/model"
"gorm.io/gorm"
)
// ============ 管理端聚合查询(visit_events SQL 聚合;长趋势用 site_daily_stats) ============
// VisitDayPoint 日趋势点
type VisitDayPoint struct {
Date string `json:"date"`
PV int64 `json:"pv"`
UV int64 `json:"uv"`
}
// VisitNamedCount 命名计数(设备/来源/爬虫等)
type VisitNamedCount struct {
Key string `json:"key"`
Label string `json:"label"`
Count int64 `json:"count"`
}
// VisitPageRow 受访页面行
type VisitPageRow struct {
Path string `json:"path"`
PV int64 `json:"pv"`
UV int64 `json:"uv"`
Share float64 `json:"share"` // 占区间总 PV 比例(0–100)
}
// VisitOverview 概览 Tab
type VisitOverview struct {
RangeDays int `json:"range_days"`
Trend []VisitDayPoint `json:"trend"`
TodayPV int64 `json:"today_pv"`
TodayUV int64 `json:"today_uv"`
YesterdayPV int64 `json:"yesterday_pv"`
YesterdayUV int64 `json:"yesterday_uv"`
LoggedInPV int64 `json:"logged_in_pv"` // 今日登录 PV
Devices []VisitNamedCount `json:"devices"`
Sources []VisitNamedCount `json:"sources"`
TopPages []VisitPageRow `json:"top_pages"`
}
// VisitMinutePoint 分钟级 PV 点
type VisitMinutePoint struct {
Minute string `json:"minute"` // YYYY-MM-DD HH24:MI
PV int64 `json:"pv"`
}
// VisitEventItem 明细列表行
type VisitEventItem struct {
ID uint64 `json:"id"`
CreatedAt time.Time `json:"created_at"`
Kind string `json:"kind"`
Path string `json:"path"`
RefHost string `json:"ref_host"`
RefKind string `json:"ref_kind"`
UserID uint `json:"user_id"`
Username string `json:"username"`
IP string `json:"ip"`
Device string `json:"device"`
OS string `json:"os"`
Browser string `json:"browser"`
Bot string `json:"bot"`
Status int `json:"status"`
}
// VisitRealtime 实时 Tab
type VisitRealtime struct {
Minutes []VisitMinutePoint `json:"minutes"`
PV5min int64 `json:"pv_5min"`
PV30min int64 `json:"pv_30min"`
Online int64 `json:"online"`
TopPages []VisitNamedCount `json:"top_pages"`
Recent []VisitEventItem `json:"recent"`
}
// VisitSourcesKindDay 来源按日(kind 分列行)
type VisitSourcesKindDay struct {
Date string `json:"date"`
Items []VisitNamedCount `json:"items"`
}
// VisitSources 来源分析 Tab
type VisitSources struct {
RangeDays int `json:"range_days"`
Kinds []VisitNamedCount `json:"kinds"`
SearchHost []VisitNamedCount `json:"search_hosts"`
External []VisitNamedCount `json:"external"`
Trend []VisitSourcesKindDay `json:"trend"`
}
// VisitPages 受访页面 Tab
type VisitPages struct {
RangeDays int `json:"range_days"`
Total int64 `json:"total"` // 命中的 path 数
Items []VisitPageRow `json:"items"`
}
// VisitVisitors 访客画像 Tab
type VisitVisitors struct {
RangeDays int `json:"range_days"`
Devices []VisitNamedCount `json:"devices"`
OSes []VisitNamedCount `json:"oses"`
Browsers []VisitNamedCount `json:"browsers"`
LoggedInPV int64 `json:"logged_in_pv"`
GuestPV int64 `json:"guest_pv"`
NewVisitors int64 `json:"new_visitors"`
Returning int64 `json:"returning"`
Hourly []VisitNamedCount `json:"hourly"` // 0–23 时段 PV
}
// VisitSecurity 爬虫与安全 Tab
type VisitSecurity struct {
RangeDays int `json:"range_days"`
Bots []VisitNamedCount `json:"bots"`
BotTrend []VisitDayPoint `json:"bot_trend"`
ProbeStatus []VisitNamedCount `json:"probe_status"`
ProbeIPs []VisitNamedCount `json:"probe_ips"`
LoginFailed int64 `json:"login_failed"` // 近 N 日登录失败次数
FailedUsers int64 `json:"failed_users"` // 涉及用户名数
}
// VisitEventFilter 明细筛选
type VisitEventFilter struct {
Kind string
Bot string
Path string
IP string
Start time.Time
End time.Time
Page int
Size int
}
// VisitEventsResult 明细分页结果
type VisitEventsResult struct {
Total int64 `json:"total"`
Items []VisitEventItem `json:"items"`
}
// VisitSettingsPayload 设置 Tab(开关 + 保留天数 + 容量)
type VisitSettingsPayload struct {
Enabled bool `json:"enabled"`
RetentionDays int `json:"retention_days"`
Usage VisitUsage `json:"usage"`
Dropped uint64 `json:"dropped"`
}
type pvUVRow struct {
PV int64 `gorm:"column:pv"`
UV int64 `gorm:"column:uv"`
}
func visitRangeStart(days int) time.Time {
return todayDate().AddDate(0, 0, -(days - 1))
}
// groupCounts 通用 GROUP BY 计数(expr 为聚合表达式),按计数降序取前 limit 条。
// extraSQL 追加到 WHERE(如 " AND ref_kind = 'search'")。
func (s *VisitStatsService) groupCounts(start time.Time, expr, kind, extraSQL string, limit int) ([]VisitNamedCount, error) {
var rows []struct {
Key string
Count int64
}
q := s.db.Model(&model.VisitEvent{}).
Select(expr+" AS key, COUNT(*) AS count").
Where("created_at >= ?", start)
if kind != "" {
q = q.Where("kind = ?", kind)
}
if extraSQL != "" {
q = q.Where(extraSQL)
}
if err := q.Group("key").Order("count DESC").Limit(limit).Scan(&rows).Error; err != nil {
return nil, err
}
out := make([]VisitNamedCount, 0, len(rows))
for _, r := range rows {
out = append(out, VisitNamedCount{Key: r.Key, Label: r.Key, Count: r.Count})
}
return out, nil
}
// Overview 概览:趋势(site_daily_stats)+ 今日/昨日 + 设备/来源/热门页面
func (s *VisitStatsService) Overview(rangeQ string) (*VisitOverview, error) {
days := parseRangeDays(rangeQ)
start := visitRangeStart(days)
out := &VisitOverview{RangeDays: days}
// 日趋势:日聚合表(每天一行,不随明细清理丢失)
if err := s.db.Model(&model.SiteDailyStats{}).
Select("to_char(date, 'YYYY-MM-DD') AS date, pv, uv").
Where("date >= ?", start).
Order("date ASC").Scan(&out.Trend).Error; err != nil {
return nil, err
}
// 空结果时 GORM 不赋值,nil 切片会序列化成 JSON null(前端期望 [])
if out.Trend == nil {
out.Trend = []VisitDayPoint{}
}
var today pvUVRow
if err := s.db.Model(&model.SiteDailyStats{}).
Select("COALESCE(SUM(pv),0) AS pv, COALESCE(SUM(uv),0) AS uv").
Where("date = ?", todayDate()).Scan(&today).Error; err != nil {
return nil, err
}
out.TodayPV, out.TodayUV = today.PV, today.UV
var yesterday pvUVRow
if err := s.db.Model(&model.SiteDailyStats{}).
Select("COALESCE(SUM(pv),0) AS pv, COALESCE(SUM(uv),0) AS uv").
Where("date = ?", todayDate().AddDate(0, 0, -1)).Scan(&yesterday).Error; err != nil {
return nil, err
}
out.YesterdayPV, out.YesterdayUV = yesterday.PV, yesterday.UV
if err := s.db.Model(&model.SiteDailyStats{}).
Select("COALESCE(SUM(logged_in_pv),0) AS logged_in_pv").
Where("date = ?", todayDate()).Scan(&out.LoggedInPV).Error; err != nil {
return nil, err
}
var err error
if out.Devices, err = s.groupCounts(start, "device", model.VisitKindPageview, "", 8); err != nil {
return nil, err
}
if out.Sources, err = s.groupCounts(start, "ref_kind", model.VisitKindPageview, "", 8); err != nil {
return nil, err
}
if out.TopPages, err = s.pageAgg(start, "", 1, 10); err != nil {
return nil, err
}
return out, nil
}
// pageAgg path 聚合 PV/UV/占比(q 为 path 模糊匹配;page 从 1 起)
func (s *VisitStatsService) pageAgg(start time.Time, q string, page, size int) ([]VisitPageRow, error) {
var totalPV int64
if err := s.db.Model(&model.VisitEvent{}).
Where("created_at >= ? AND kind = ?", start, model.VisitKindPageview).
Scopes(pathFilter(q)).
Count(&totalPV).Error; err != nil {
return nil, err
}
if totalPV == 0 {
return []VisitPageRow{}, nil
}
var rows []VisitPageRow
if err := s.db.Model(&model.VisitEvent{}).
Select("path, COUNT(*) AS pv, COUNT(DISTINCT vid_hash) AS uv").
Where("created_at >= ? AND kind = ?", start, model.VisitKindPageview).
Scopes(pathFilter(q)).
Group("path").Order("pv DESC").
Offset((page - 1) * size).Limit(size).Scan(&rows).Error; err != nil {
return nil, err
}
for i := range rows {
rows[i].Share = float64(rows[i].PV) * 100 / float64(totalPV)
}
return rows, nil
}
func pathFilter(q string) func(*gorm.DB) *gorm.DB {
if q == "" {
return func(db *gorm.DB) *gorm.DB { return db }
}
return func(db *gorm.DB) *gorm.DB { return db.Where("path ILIKE ?", "%"+q+"%") }
}
// Realtime 实时:近 60 分钟分钟级 PV + 在线 + 近 5 分钟热门路径 + 最新 20 条
func (s *VisitStatsService) Realtime() (*VisitRealtime, error) {
out := &VisitRealtime{}
now := time.Now()
if err := s.db.Model(&model.VisitEvent{}).
Select("to_char(created_at, 'YYYY-MM-DD HH24:MI') AS minute, COUNT(*) AS pv").
Where("created_at >= ? AND kind = ?", now.Add(-time.Hour), model.VisitKindPageview).
Group("minute").Order("minute ASC").Scan(&out.Minutes).Error; err != nil {
return nil, err
}
if out.Minutes == nil {
out.Minutes = []VisitMinutePoint{}
}
if err := s.db.Model(&model.VisitEvent{}).
Where("created_at >= ? AND kind = ?", now.Add(-5*time.Minute), model.VisitKindPageview).
Count(&out.PV5min).Error; err != nil {
return nil, err
}
if err := s.db.Model(&model.VisitEvent{}).
Where("created_at >= ? AND kind = ?", now.Add(-30*time.Minute), model.VisitKindPageview).
Count(&out.PV30min).Error; err != nil {
return nil, err
}
// 在线口径与首页概览一致:users.last_seen_at 近 5 分钟且未封禁
if err := s.db.Model(&model.User{}).
Where("last_seen_at IS NOT NULL AND last_seen_at >= ? AND banned = ?", now.Add(-5*time.Minute), false).
Count(&out.Online).Error; err != nil {
return nil, err
}
if err := s.db.Model(&model.VisitEvent{}).
Select("path AS key, COUNT(*) AS count").
Where("created_at >= ? AND kind = ?", now.Add(-5*time.Minute), model.VisitKindPageview).
Group("path").Order("count DESC").Limit(10).
Scan(&out.TopPages).Error; err != nil {
return nil, err
}
for i := range out.TopPages {
out.TopPages[i].Label = out.TopPages[i].Key
}
if out.TopPages == nil {
out.TopPages = []VisitNamedCount{}
}
res, err := s.Events(VisitEventFilter{Kind: model.VisitKindPageview, Page: 1, Size: 20})
if err != nil {
return nil, err
}
out.Recent = res.Items
return out, nil
}
// Sources 来源分析
func (s *VisitStatsService) Sources(rangeQ string) (*VisitSources, error) {
days := parseRangeDays(rangeQ)
start := visitRangeStart(days)
out := &VisitSources{RangeDays: days}
var err error
if out.Kinds, err = s.groupCounts(start, "ref_kind", model.VisitKindPageview, "", 8); err != nil {
return nil, err
}
if out.SearchHost, err = s.groupCounts(start, "ref_host", model.VisitKindPageview,
"ref_kind = '"+RefKindSearch+"'", 20); err != nil {
return nil, err
}
if out.External, err = s.groupCounts(start, "ref_host", model.VisitKindPageview,
"ref_kind = '"+RefKindExternal+"'", 20); err != nil {
return nil, err
}
if out.Trend, err = s.SourcesTrend(rangeQ); err != nil {
return nil, err
}
return out, nil
}
// SourcesTrend 来源按日明细(kind 分列行)
func (s *VisitStatsService) SourcesTrend(rangeQ string) ([]VisitSourcesKindDay, error) {
days := parseRangeDays(rangeQ)
start := visitRangeStart(days)
var rows []struct {
Date string
RefKind string
Count int64
}
if err := s.db.Model(&model.VisitEvent{}).
Select("to_char(created_at, 'YYYY-MM-DD') AS date, ref_kind, COUNT(*) AS count").
Where("created_at >= ? AND kind = ?", start, model.VisitKindPageview).
Group("date, ref_kind").Order("date ASC").Scan(&rows).Error; err != nil {
return nil, err
}
byDay := make(map[string][]VisitNamedCount)
var dates []string
seen := make(map[string]struct{})
for _, r := range rows {
if _, ok := seen[r.Date]; !ok {
seen[r.Date] = struct{}{}
dates = append(dates, r.Date)
}
byDay[r.Date] = append(byDay[r.Date], VisitNamedCount{Key: r.RefKind, Label: r.RefKind, Count: r.Count})
}
out := make([]VisitSourcesKindDay, 0, len(dates))
for _, d := range dates {
out = append(out, VisitSourcesKindDay{Date: d, Items: byDay[d]})
}
return out, nil
}
// Pages 受访页面(分页 + 模糊)
func (s *VisitStatsService) Pages(rangeQ, q string, page, size int) (*VisitPages, error) {
days := parseRangeDays(rangeQ)
start := visitRangeStart(days)
if page < 1 {
page = 1
}
if size < 1 || size > 100 {
size = 20
}
out := &VisitPages{RangeDays: days}
if err := s.db.Model(&model.VisitEvent{}).
Where("created_at >= ? AND kind = ?", start, model.VisitKindPageview).
Scopes(pathFilter(q)).
Distinct("path").Count(&out.Total).Error; err != nil {
return nil, err
}
items, err := s.pageAgg(start, q, page, size)
if err != nil {
return nil, err
}
if items == nil {
items = []VisitPageRow{}
}
out.Items = items
return out, nil
}
// Visitors 访客画像
func (s *VisitStatsService) Visitors(rangeQ string) (*VisitVisitors, error) {
days := parseRangeDays(rangeQ)
start := visitRangeStart(days)
out := &VisitVisitors{RangeDays: days}
var err error
if out.Devices, err = s.groupCounts(start, "device", model.VisitKindPageview, "", 8); err != nil {
return nil, err
}
if out.OSes, err = s.groupCounts(start, "os", model.VisitKindPageview, "", 10); err != nil {
return nil, err
}
if out.Browsers, err = s.groupCounts(start, "browser", model.VisitKindPageview, "", 10); err != nil {
return nil, err
}
var lg pvUVRow
if err := s.db.Model(&model.VisitEvent{}).
Select("COALESCE(SUM(CASE WHEN user_id > 0 THEN 1 ELSE 0 END),0) AS pv, COALESCE(SUM(CASE WHEN user_id = 0 THEN 1 ELSE 0 END),0) AS uv").
Where("created_at >= ? AND kind = ?", start, model.VisitKindPageview).
Scan(&lg).Error; err != nil {
return nil, err
}
out.LoggedInPV, out.GuestPV = lg.PV, lg.UV
// 新老访客:区间内 vid 首次出现时间晚于区间起点视为新访客
var uvRows []struct {
VidHash string
FirstAt time.Time `gorm:"column:first_at"`
}
if err := s.db.Model(&model.VisitEvent{}).
Select("vid_hash, MIN(created_at) AS first_at").
Where("created_at >= ? AND kind = ? AND vid_hash <> ''", start, model.VisitKindPageview).
Group("vid_hash").Scan(&uvRows).Error; err != nil {
return nil, err
}
for _, r := range uvRows {
if r.FirstAt.After(start) {
out.NewVisitors++
} else {
out.Returning++
}
}
// 24 小时时段 PV(区间合计)
var hours []struct {
Hour int
Count int64
}
if err := s.db.Model(&model.VisitEvent{}).
Select("EXTRACT(HOUR FROM created_at)::int AS hour, COUNT(*) AS count").
Where("created_at >= ? AND kind = ?", start, model.VisitKindPageview).
Group("hour").Order("hour ASC").Scan(&hours).Error; err != nil {
return nil, err
}
out.Hourly = make([]VisitNamedCount, 24)
for i := range out.Hourly {
label := strconv.Itoa(i)
if i < 10 {
label = "0" + label
}
out.Hourly[i].Key = label
out.Hourly[i].Label = label + " 时"
}
for _, h := range hours {
if h.Hour >= 0 && h.Hour < 24 {
out.Hourly[h.Hour].Count = h.Count
}
}
return out, nil
}
// Security 爬虫与安全
func (s *VisitStatsService) Security(rangeQ string) (*VisitSecurity, error) {
days := parseRangeDays(rangeQ)
start := visitRangeStart(days)
out := &VisitSecurity{RangeDays: days}
var err error
if out.Bots, err = s.groupCounts(start, "bot", model.VisitKindBot, "", 20); err != nil {
return nil, err
}
if err := s.db.Model(&model.VisitEvent{}).
Select("to_char(created_at, 'YYYY-MM-DD') AS date, COUNT(*) AS pv").
Where("created_at >= ? AND kind = ?", start, model.VisitKindBot).
Group("date").Order("date ASC").Scan(&out.BotTrend).Error; err != nil {
return nil, err
}
if out.BotTrend == nil {
out.BotTrend = []VisitDayPoint{}
}
if out.ProbeStatus, err = s.groupCounts(start, "status::text", model.VisitKindProbe, "", 15); err != nil {
return nil, err
}
if out.ProbeIPs, err = s.groupCounts(start, "ip", model.VisitKindProbe, "", 20); err != nil {
return nil, err
}
if err := s.db.Model(&model.LoginLog{}).
Where("success = ? AND created_at >= ?", false, start).
Count(&out.LoginFailed).Error; err != nil {
return nil, err
}
if err := s.db.Raw(`SELECT COUNT(DISTINCT username) FROM login_logs WHERE success = ? AND created_at >= ?`,
false, start).Scan(&out.FailedUsers).Error; err != nil {
return nil, err
}
return out, nil
}
// Events 访问明细(筛选 + 分页 + 补用户昵称)
func (s *VisitStatsService) Events(f VisitEventFilter) (*VisitEventsResult, error) {
if f.Page < 1 {
f.Page = 1
}
if f.Size < 1 || f.Size > 200 {
f.Size = 50
}
apply := func(db *gorm.DB) *gorm.DB {
if f.Kind != "" {
db = db.Where("kind = ?", f.Kind)
}
if f.Bot != "" {
db = db.Where("bot = ?", f.Bot)
}
if f.Path != "" {
db = db.Where("path ILIKE ?", "%"+f.Path+"%")
}
if f.IP != "" {
db = db.Where("ip = ?", f.IP)
}
if !f.Start.IsZero() {
db = db.Where("created_at >= ?", f.Start)
}
if !f.End.IsZero() {
db = db.Where("created_at < ?", f.End)
}
return db
}
out := &VisitEventsResult{}
if err := apply(s.db.Model(&model.VisitEvent{})).Count(&out.Total).Error; err != nil {
return nil, err
}
var events []model.VisitEvent
if err := apply(s.db.Model(&model.VisitEvent{})).
Order("id DESC").
Offset((f.Page - 1) * f.Size).Limit(f.Size).
Find(&events).Error; err != nil {
return nil, err
}
// 批量补昵称
userIDs := make([]uint, 0, len(events))
for _, e := range events {
if e.UserID > 0 {
userIDs = append(userIDs, e.UserID)
}
}
nameMap := make(map[uint]string, len(userIDs))
if len(userIDs) > 0 {
var users []model.User
if err := s.db.Select("id, username, nickname").
Where("id IN ?", userIDs).Find(&users).Error; err != nil {
return nil, err
}
for _, u := range users {
nameMap[u.ID] = u.Nickname
}
}
out.Items = make([]VisitEventItem, 0, len(events))
for _, e := range events {
out.Items = append(out.Items, VisitEventItem{
ID: e.ID, CreatedAt: e.CreatedAt, Kind: e.Kind, Path: e.Path,
RefHost: e.RefHost, RefKind: e.RefKind,
UserID: e.UserID, Username: nameMap[e.UserID],
IP: e.IP, Device: e.Device, OS: e.OS, Browser: e.Browser,
Bot: e.Bot, Status: e.Status,
})
}
return out, nil
}
// SettingsPayload 设置 Tab 数据
func (s *VisitStatsService) SettingsPayload() (*VisitSettingsPayload, error) {
out := &VisitSettingsPayload{
Enabled: s.Enabled(),
RetentionDays: s.retentionDays(),
Dropped: s.DroppedCount(),
}
u, err := s.Usage()
if err != nil {
return nil, err
}
out.Usage = u
return out, nil
}
// SaveSettings 保存采集开关与保留天数
func (s *VisitStatsService) SaveSettings(enabled bool, retentionDays int) error {
if err := s.setting.SetAnalyticsEnabled(enabled); err != nil {
return err
}
return s.setting.SetAnalyticsRetentionDays(retentionDays)
}