feat: make database migration testable and backfill assets
This commit is contained in:
@@ -227,7 +227,7 @@ func (a *App) bootstrapAdmin() {
|
|||||||
if err := a.db.Create(&u).Error; err != nil {
|
if err := a.db.Create(&u).Error; err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
if err := a.db.Model(&Task{}).Where("user_id = 0").Update("user_id", u.ID).Error; err != nil {
|
if err := a.db.Model(&Task{}).Where("user_id = 0 OR user_id IS NULL").Update("user_id", u.ID).Error; err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
if err := a.db.Create(&CreditAccount{UserID: u.ID}).Error; err != nil {
|
if err := a.db.Create(&CreditAccount{UserID: u.ID}).Error; err != nil {
|
||||||
|
|||||||
@@ -15,18 +15,22 @@ func main() {
|
|||||||
_ = os.MkdirAll(filepath.Dir(dbPath), 0755)
|
_ = os.MkdirAll(filepath.Dir(dbPath), 0755)
|
||||||
db, err := gorm.Open(sqlite.Open(dbPath), &gorm.Config{})
|
db, err := gorm.Open(sqlite.Open(dbPath), &gorm.Config{})
|
||||||
must(err)
|
must(err)
|
||||||
must(db.AutoMigrate(&Task{}, &MediaAsset{}, &Setting{}, &User{}, &UserPermission{}, &CreditAccount{}, &CreditTransaction{}, &CreditReservation{}, &RedeemBatch{}, &RedeemCode{}, &RedeemRecord{}, &WebDAVConfig{}, &Session{}, &AuditLog{}))
|
must(migrateDatabase(db))
|
||||||
// Replace legacy global task/reservation uniqueness constraints with scoped indexes.
|
|
||||||
must(db.Exec("DROP INDEX IF EXISTS idx_tasks_source_rid_quality").Error)
|
|
||||||
must(db.Exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_tasks_user_source_rid_quality ON tasks(user_id, source, r_id, quality)").Error)
|
|
||||||
must(db.Exec("DROP INDEX IF EXISTS idx_credit_reservations_task_id").Error)
|
|
||||||
must(db.Exec("CREATE INDEX IF NOT EXISTS idx_credit_reservations_task_id ON credit_reservations(task_id)").Error)
|
|
||||||
|
|
||||||
client := &http.Client{Timeout: 2 * 60 * 1e9}
|
client := &http.Client{Timeout: 2 * 60 * 1e9}
|
||||||
cfg := loadConfig(db)
|
cfg := loadConfig(db)
|
||||||
a := &App{db: db, cfg: cfg, queue: make(chan uint, 100), httpClient: client}
|
a := &App{db: db, cfg: cfg, queue: make(chan uint, 100), httpClient: client}
|
||||||
a.loadSettings()
|
a.loadSettings()
|
||||||
a.bootstrapAdmin()
|
a.bootstrapAdmin()
|
||||||
|
must(backfillMediaAssets(db, a.cfg.DownloadDir))
|
||||||
|
if os.Getenv("MIGRATION_CHECK") == "1" {
|
||||||
|
var users, tasks, assets int64
|
||||||
|
db.Model(&User{}).Count(&users)
|
||||||
|
db.Model(&Task{}).Count(&tasks)
|
||||||
|
db.Model(&MediaAsset{}).Count(&assets)
|
||||||
|
println("migration check ok", "users", users, "tasks", tasks, "assets", assets)
|
||||||
|
return
|
||||||
|
}
|
||||||
normalizeConfig(&a.cfg)
|
normalizeConfig(&a.cfg)
|
||||||
a.am = &AMAPIClient{BaseURL: a.cfg.AMAPI, HTTP: client}
|
a.am = &AMAPIClient{BaseURL: a.cfg.AMAPI, HTTP: client}
|
||||||
if a.cfg.WebDAVURL != "" {
|
if a.cfg.WebDAVURL != "" {
|
||||||
|
|||||||
@@ -0,0 +1,78 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
"gorm.io/gorm"
|
||||||
|
)
|
||||||
|
|
||||||
|
// migrateDatabase applies schema changes and repairs legacy indexes. It is
|
||||||
|
// intentionally separate from startup so migration behavior can be tested on
|
||||||
|
// a disposable SQLite database before touching production data.
|
||||||
|
func migrateDatabase(db *gorm.DB) error {
|
||||||
|
if err := db.AutoMigrate(&Task{}, &MediaAsset{}, &Setting{}, &User{}, &UserPermission{}, &CreditAccount{}, &CreditTransaction{}, &CreditReservation{}, &RedeemBatch{}, &RedeemCode{}, &RedeemRecord{}, &WebDAVConfig{}, &Session{}, &AuditLog{}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := db.Exec("DROP INDEX IF EXISTS idx_tasks_source_rid_quality").Error; err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := db.Exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_tasks_user_source_rid_quality ON tasks(user_id, source, r_id, quality)").Error; err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := db.Exec("DROP INDEX IF EXISTS idx_credit_reservations_task_id").Error; err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return db.Exec("CREATE INDEX IF NOT EXISTS idx_credit_reservations_task_id ON credit_reservations(task_id)").Error
|
||||||
|
}
|
||||||
|
|
||||||
|
// backfillMediaAssets turns legacy completed tasks into shared media assets.
|
||||||
|
// Existing physical files are reused; no media is moved or deleted.
|
||||||
|
func backfillMediaAssets(db *gorm.DB, downloadDir string) error {
|
||||||
|
var tasks []Task
|
||||||
|
if err := db.Where("(asset_id = 0 OR asset_id IS NULL) AND status = ?", "done").Find(&tasks).Error; err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for i := range tasks {
|
||||||
|
t := &tasks[i]
|
||||||
|
key := assetKey(t.Source, t.RID, t.Quality)
|
||||||
|
var asset MediaAsset
|
||||||
|
result := db.Where("asset_key = ?", key).Limit(1).Find(&asset)
|
||||||
|
if result.Error != nil {
|
||||||
|
return result.Error
|
||||||
|
}
|
||||||
|
if result.RowsAffected == 0 {
|
||||||
|
asset = MediaAsset{
|
||||||
|
AssetKey: key,
|
||||||
|
Source: t.Source,
|
||||||
|
RID: t.RID,
|
||||||
|
Quality: t.Quality,
|
||||||
|
LocalPath: t.LocalPath,
|
||||||
|
Format: t.Format,
|
||||||
|
ActualQuality: t.ActualQuality,
|
||||||
|
Size: t.DownloadedBytes,
|
||||||
|
Status: "done",
|
||||||
|
}
|
||||||
|
if asset.LocalPath != "" {
|
||||||
|
if fi, statErr := os.Stat(asset.LocalPath); statErr == nil && fi.Size() > 0 {
|
||||||
|
asset.Size = fi.Size()
|
||||||
|
}
|
||||||
|
} else if downloadDir != "" {
|
||||||
|
candidate := filepath.Join(downloadDir, safe(t.Artist)+" - "+safe(t.Title)+"."+t.Format)
|
||||||
|
if fi, statErr := os.Stat(candidate); statErr == nil && fi.Size() > 0 {
|
||||||
|
asset.LocalPath, asset.Size = candidate, fi.Size()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := db.Create(&asset).Error; err != nil {
|
||||||
|
return fmt.Errorf("create media asset for task %d: %w", t.ID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if t.AssetID != asset.ID {
|
||||||
|
if err := db.Model(t).Update("asset_id", asset.ID).Error; err != nil {
|
||||||
|
return fmt.Errorf("link media asset for task %d: %w", t.ID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/glebarez/sqlite"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
)
|
||||||
|
|
||||||
|
func openMigrationTestDB(t *testing.T) *gorm.DB {
|
||||||
|
t.Helper()
|
||||||
|
db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "migration.db")), &gorm.Config{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return db
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMigrateDatabaseLegacySchema(t *testing.T) {
|
||||||
|
db := openMigrationTestDB(t)
|
||||||
|
if err := db.Exec(`CREATE TABLE tasks (
|
||||||
|
id integer PRIMARY KEY AUTOINCREMENT,
|
||||||
|
source text, r_id text, title text, artist text, album text, pic text,
|
||||||
|
quality text, format text, status text, error text, local_path text,
|
||||||
|
remote_path text, created_at datetime, updated_at datetime, total_bytes integer,
|
||||||
|
downloaded_bytes integer, progress real, resolved_quality text, actual_quality text
|
||||||
|
)`).Error; err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := db.Exec("CREATE UNIQUE INDEX idx_tasks_source_rid_quality ON tasks(source, r_id, quality)").Error; err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := db.Exec("INSERT INTO tasks(source,r_id,quality,status) VALUES('kw','legacy-1','standard','done')").Error; err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := migrateDatabase(db); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var task Task
|
||||||
|
if err := db.First(&task).Error; err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if task.UserID != 0 {
|
||||||
|
t.Fatalf("legacy task user id = %d, want 0 before bootstrap", task.UserID)
|
||||||
|
}
|
||||||
|
var indexes []struct{ Name string }
|
||||||
|
if err := db.Raw("SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='tasks'").Scan(&indexes).Error; err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
found := false
|
||||||
|
for _, idx := range indexes {
|
||||||
|
if idx.Name == "idx_tasks_user_source_rid_quality" {
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
if idx.Name == "idx_tasks_source_rid_quality" {
|
||||||
|
t.Fatalf("legacy global task index still exists")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatalf("scoped task index not created")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackfillMediaAssetsLinksExistingFiles(t *testing.T) {
|
||||||
|
db := openMigrationTestDB(t)
|
||||||
|
if err := migrateDatabase(db); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, "Artist - Title.mp3")
|
||||||
|
if err := os.WriteFile(path, []byte("audio"), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
task := Task{UserID: 1, Source: "kw", RID: "rid-1", Title: "Title", Artist: "Artist", Quality: "standard", Format: "mp3", Status: "done", LocalPath: path}
|
||||||
|
if err := db.Create(&task).Error; err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := backfillMediaAssets(db, dir); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var asset MediaAsset
|
||||||
|
if err := db.Where("asset_key = ?", assetKey(task.Source, task.RID, task.Quality)).First(&asset).Error; err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if asset.Status != "done" || asset.LocalPath != path || asset.Size != int64(len("audio")) {
|
||||||
|
t.Fatalf("unexpected asset: %+v", asset)
|
||||||
|
}
|
||||||
|
var got Task
|
||||||
|
if err := db.First(&got, task.ID).Error; err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.AssetID != asset.ID {
|
||||||
|
t.Fatalf("task asset id = %d, want %d", got.AssetID, asset.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user