feat: 媒体库 blob 去重存储与品牌/注册等前端调整
新增： - blob_store 二进制去重存储 + blob_dedup 测试 其它： - 后端 settings/auth/media_library/upload/operations 服务与 handler 调整 - 前端品牌（BrandLockup/BrandCropModal/BrandSeoPanel）、注册、媒体库、分析详情页、Header/SiteChrome 等更新
This commit is contained in:
223
backend/service/blob_dedup_test.go
Normal file
223
backend/service/blob_dedup_test.go
Normal file
@@ -0,0 +1,223 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/freefire/jiang13-bbs/model"
|
||||
webpenc "github.com/gen2brain/webp"
|
||||
"github.com/glebarez/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func newBlobTestDB(t *testing.T) *gorm.DB {
|
||||
t.Helper()
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite: %v", err)
|
||||
}
|
||||
if err := db.AutoMigrate(
|
||||
&model.Blob{}, &model.Attachment{}, &model.PostAttachment{},
|
||||
&model.User{}, &model.Post{},
|
||||
); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
return db
|
||||
}
|
||||
|
||||
func webpFixtureBytes() []byte {
|
||||
src := image.NewRGBA(image.Rect(0, 0, 20, 10))
|
||||
src.Set(0, 0, color.RGBA{10, 20, 30, 255})
|
||||
var buf bytes.Buffer
|
||||
_ = webpenc.Encode(&buf, src, webpenc.Options{Quality: ImageWebPQuality})
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func TestSaveImageBlobDedupAndRefcount(t *testing.T) {
|
||||
db := newBlobTestDB(t)
|
||||
dir := t.TempDir()
|
||||
s := NewUploadService(db, dir)
|
||||
if err := s.EnsureDir(); err != nil {
|
||||
t.Fatalf("ensure dir: %v", err)
|
||||
}
|
||||
data := webpFixtureBytes()
|
||||
|
||||
att1, err := s.SaveImage(1, bytes.NewReader(data), "post")
|
||||
if err != nil {
|
||||
t.Fatalf("SaveImage 1: %v", err)
|
||||
}
|
||||
att2, err := s.SaveImage(2, bytes.NewReader(data), "post")
|
||||
if err != nil {
|
||||
t.Fatalf("SaveImage 2: %v", err)
|
||||
}
|
||||
if att1.URL != att2.URL || att1.BlobHash == "" || att1.BlobHash != att2.BlobHash {
|
||||
t.Fatalf("identical content should share url/hash: %q vs %q", att1.URL, att2.URL)
|
||||
}
|
||||
|
||||
var blob model.Blob
|
||||
if err := db.First(&blob, "hash = ? AND public = ?", att1.BlobHash, true).Error; err != nil {
|
||||
t.Fatalf("load blob: %v", err)
|
||||
}
|
||||
if blob.RefCount != 2 {
|
||||
t.Fatalf("refcount want 2, got %d", blob.RefCount)
|
||||
}
|
||||
blobPath := filepath.Join(dir, filepath.FromSlash(att1.URL[len("/uploads/"):]))
|
||||
if fi, err := os.Stat(blobPath); err != nil || fi.IsDir() {
|
||||
t.Fatalf("shared physical file missing: %v", err)
|
||||
}
|
||||
|
||||
// 删一个引用：文件保留
|
||||
if err := s.DeleteAttachment(1, att1.ID); err != nil {
|
||||
t.Fatalf("DeleteAttachment 1: %v", err)
|
||||
}
|
||||
db.First(&blob, "hash = ? AND public = ?", att1.BlobHash, true)
|
||||
if blob.RefCount != 1 {
|
||||
t.Fatalf("refcount want 1, got %d", blob.RefCount)
|
||||
}
|
||||
if _, err := os.Stat(blobPath); err != nil {
|
||||
t.Fatalf("blob file should survive while referenced: %v", err)
|
||||
}
|
||||
|
||||
// 最后一个引用删除：文件与 blob 行一起回收
|
||||
if err := s.DeleteAttachment(2, att2.ID); err != nil {
|
||||
t.Fatalf("DeleteAttachment 2: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(blobPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("blob file should be removed after last release, stat err=%v", err)
|
||||
}
|
||||
var n int64
|
||||
db.Model(&model.Blob{}).Where("hash = ? AND public = ?", att1.BlobHash, true).Count(&n)
|
||||
if n != 0 {
|
||||
t.Fatalf("blob row should be gone, count=%d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSaveImageTranscodedJPEGDedup(t *testing.T) {
|
||||
db := newBlobTestDB(t)
|
||||
dir := t.TempDir()
|
||||
s := NewUploadService(db, dir)
|
||||
if err := s.EnsureDir(); err != nil {
|
||||
t.Fatalf("ensure dir: %v", err)
|
||||
}
|
||||
|
||||
src := image.NewRGBA(image.Rect(0, 0, 48, 36))
|
||||
src.Set(1, 2, color.RGBA{200, 100, 50, 255})
|
||||
var in bytes.Buffer
|
||||
if err := jpeg.Encode(&in, src, nil); err != nil {
|
||||
t.Fatalf("encode jpeg: %v", err)
|
||||
}
|
||||
|
||||
att1, err := s.SaveImage(1, bytes.NewReader(in.Bytes()), "post")
|
||||
if err != nil {
|
||||
t.Fatalf("SaveImage 1: %v", err)
|
||||
}
|
||||
att2, err := s.SaveImage(1, bytes.NewReader(in.Bytes()), "post")
|
||||
if err != nil {
|
||||
t.Fatalf("SaveImage 2: %v", err)
|
||||
}
|
||||
// hash 必须对转码后的字节计算：同源 JPEG 两次上传应命中同一个 WebP blob
|
||||
if att1.URL != att2.URL || att1.BlobHash != att2.BlobHash {
|
||||
t.Fatalf("transcoded JPEG should dedup: %q(%s) vs %q(%s)", att1.URL, att1.BlobHash, att2.URL, att2.BlobHash)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSaveDraftFileBlobDedupAndRefcount(t *testing.T) {
|
||||
db := newBlobTestDB(t)
|
||||
root := t.TempDir()
|
||||
s := NewPostFileService(db, root)
|
||||
if err := s.EnsureDir(); err != nil {
|
||||
t.Fatalf("ensure dir: %v", err)
|
||||
}
|
||||
data := []byte("paid attachment content v1")
|
||||
|
||||
a1, err := s.SaveDraftFile(1, "report.pdf", bytes.NewReader(data), 50)
|
||||
if err != nil {
|
||||
t.Fatalf("SaveDraftFile 1: %v", err)
|
||||
}
|
||||
a2, err := s.SaveDraftFile(2, "another-name.pdf", bytes.NewReader(data), 0)
|
||||
if err != nil {
|
||||
t.Fatalf("SaveDraftFile 2: %v", err)
|
||||
}
|
||||
if a1.BlobHash == "" || a1.BlobHash != a2.BlobHash || a1.StoredName != a2.StoredName {
|
||||
t.Fatalf("same bytes should share blob: %+v vs %+v", a1, a2)
|
||||
}
|
||||
// 原始文件名仍各自保留
|
||||
if a1.Name != "report.pdf" || a2.Name != "another-name.pdf" {
|
||||
t.Fatalf("original names not preserved: %q / %q", a1.Name, a2.Name)
|
||||
}
|
||||
blobPath := s.FilePath(a1)
|
||||
if _, err := os.Stat(blobPath); err != nil {
|
||||
t.Fatalf("shared private blob missing: %v", err)
|
||||
}
|
||||
|
||||
var blob model.Blob
|
||||
if err := db.First(&blob, "hash = ? AND public = ?", a1.BlobHash, false).Error; err != nil {
|
||||
t.Fatalf("load private blob: %v", err)
|
||||
}
|
||||
if blob.RefCount != 2 {
|
||||
t.Fatalf("refcount want 2, got %d", blob.RefCount)
|
||||
}
|
||||
|
||||
if err := s.DeleteOwn(1, a1.ID); err != nil {
|
||||
t.Fatalf("DeleteOwn 1: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(blobPath); err != nil {
|
||||
t.Fatalf("private blob should survive with one ref: %v", err)
|
||||
}
|
||||
if err := s.DeleteOwn(2, a2.ID); err != nil {
|
||||
t.Fatalf("DeleteOwn 2: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(blobPath); !os.IsNotExist(err) {
|
||||
t.Fatalf("private blob should be removed after last release, stat err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlobPublicPrivateVisibilityIsolation(t *testing.T) {
|
||||
db := newBlobTestDB(t)
|
||||
root := t.TempDir()
|
||||
img := NewUploadService(db, filepath.Join(root, "uploads"))
|
||||
pf := NewPostFileService(db, filepath.Join(root, "private"))
|
||||
if err := img.EnsureDir(); err != nil {
|
||||
t.Fatalf("ensure upload dir: %v", err)
|
||||
}
|
||||
if err := pf.EnsureDir(); err != nil {
|
||||
t.Fatalf("ensure private dir: %v", err)
|
||||
}
|
||||
data := webpFixtureBytes()
|
||||
|
||||
att, err := img.SaveImage(1, bytes.NewReader(data), "post")
|
||||
if err != nil {
|
||||
t.Fatalf("SaveImage: %v", err)
|
||||
}
|
||||
file, err := pf.SaveDraftFile(1, "x.webp", bytes.NewReader(data), 0)
|
||||
if err != nil {
|
||||
t.Fatalf("SaveDraftFile: %v", err)
|
||||
}
|
||||
if att.BlobHash != file.BlobHash {
|
||||
t.Fatalf("test premise: same bytes should produce same hash")
|
||||
}
|
||||
|
||||
// 同字节在公开/私有命名空间各持一条 blob，互不复用，防止越权读取
|
||||
var pubs, privs int64
|
||||
db.Model(&model.Blob{}).Where("hash = ? AND public = ?", att.BlobHash, true).Count(&pubs)
|
||||
db.Model(&model.Blob{}).Where("hash = ? AND public = ?", att.BlobHash, false).Count(&privs)
|
||||
if pubs != 1 || privs != 1 {
|
||||
t.Fatalf("want separate public/private blobs, pubs=%d privs=%d", pubs, privs)
|
||||
}
|
||||
publicPath := filepath.Join(root, "uploads", filepath.FromSlash(att.URL[len("/uploads/"):]))
|
||||
privatePath := pf.FilePath(file)
|
||||
if publicPath == privatePath {
|
||||
t.Fatal("public and private blob must not share a path")
|
||||
}
|
||||
if _, err := os.Stat(publicPath); err != nil {
|
||||
t.Fatalf("public blob missing: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(privatePath); err != nil {
|
||||
t.Fatalf("private blob missing: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user