Compare commits

...

6 Commits
v1.3.1 ... main

20 changed files with 318 additions and 197 deletions

10
Base.go
View File

@ -68,7 +68,10 @@ func basePrepare(db *sql.DB, tx *sql.Tx, query string) *Stmt {
}
func baseExec(db *sql.DB, tx *sql.Tx, query string, args ...any) *ExecResult {
args = flatArgs(args)
return baseExecRaw(db, tx, query, flatArgs(args)...)
}
func baseExecRaw(db *sql.DB, tx *sql.Tx, query string, args ...any) *ExecResult {
var r sql.Result
var err error
startTime := time.Now()
@ -107,8 +110,10 @@ func flatArgs(args []any) []any {
}
func baseQuery(db *sql.DB, tx *sql.Tx, query string, args ...any) *QueryResult {
args = flatArgs(args)
return baseQueryRaw(db, tx, query, flatArgs(args)...)
}
func baseQueryRaw(db *sql.DB, tx *sql.Tx, query string, args ...any) *QueryResult {
var rows *sql.Rows
var err error
startTime := time.Now()
@ -128,6 +133,7 @@ func baseQuery(db *sql.DB, tx *sql.Tx, query string, args ...any) *QueryResult {
return &QueryResult{Sql: &query, Args: args, usedTime: usedTime, rows: rows}
}
func quote(quoteTag string, text string) string {
a := strings.Split(text, ".")
for i, v := range a {

View File

@ -1,6 +1,26 @@
# 变更记录 - @go/db
## [1.3.1] - 2026-05-13
## [1.0.12] - 2026-05-17
- **SQLite 极致优化 (超高并发支持)**:
- **读写分离与零锁读取**: 读操作 (`Query`) 实现零锁定,配合 WAL 模式彻底解决读写互斥问题;写操作由应用层 `sync.Mutex` 统一排队,规避 `database is locked` 错误。
- **临界区最小化**: 将 FTS 重写、参数 JSON 化 (`flatArgs`) 及日志记录移出锁保护区,极大缩短了写锁持有时间。
- **工业级默认配置**:
- 自动启用 `WAL` 模式、`NORMAL` 同步、`MEMORY` 临时存储及 `busy_timeout(5000)`
- 引入 **动态 Mmap**:根据系统内存自动设置 `mmap_size` (最大 30GB 或物理内存的 1/4),使大数据量访问接近内存速度。
- 默认 `MaxOpenConns` 提升至 100优化多线程只读性能。
- **稳定性**:
- 为 `Stmt` (预处理语句) 增加写锁保护。
- 优化事务锁机制,支持事务内的锁自动追踪与释放。
## [1.0.11] - 2026-05-13
- **基础设施对齐**:
- 移除 `encoding/json` 原生依赖,全面切换至 `apigo.cc/go/cast.UnmarshalJSON` 以增强类型兼容性。
- 移除测试代码中对 `os.Remove` 的直接调用,统一切换至 `apigo.cc/go/file.Remove` 以支持隔离文件系统。
- **稳定性增强**:
- 验证并更新了测试用例,确保在 Go 1.25.0 环境下的执行稳定性。
- 更新 `TEST.md` 性能基准,反映基础设施对齐后的最新指标。
## [1.0.10] - 2026-05-10
- **功能增强**:
- 全面支持“复杂标识符”:改进了 `LIKE` 拦截逻辑中的正则表达式,支持带引号(`` ` ``, `"`, `'`, `[]`)和特殊字符(如 `-`)的表名与字段名。
- 优化 `cleanIdentifier`:能够更精准地剥离多段式标识符(如 `table.column`)中的包装引号。

208
DB.go
View File

@ -19,7 +19,6 @@ import (
"apigo.cc/go/crypto"
"apigo.cc/go/id"
"apigo.cc/go/log"
"apigo.cc/go/rand"
"apigo.cc/go/redis"
"apigo.cc/go/safe"
)
@ -194,6 +193,7 @@ type DB struct {
QuoteTag string
tables map[string]*TableStruct
tablesLock *sync.RWMutex
sqliteMu *sync.Mutex // Serial lock for SQLite writers
}
type TableStruct struct {
@ -318,7 +318,7 @@ func (db *DB) NextID(table string) string {
func (db *DB) syncVersionFromDB(table, versionField string) {
query := fmt.Sprintf("SELECT MAX(%s) FROM %s", db.Quote(versionField), db.Quote(table))
maxVer := db.Query(query).IntOnR1C1()
maxVer := db.rawQuery(query).IntOnR1C1()
if db.Config.Redis != "" {
r := redis.GetRedis(db.Config.Redis, db.logger.logger)
@ -349,6 +349,69 @@ func GetDB(name string, logger *log.Logger) *DB {
return getDB(name, logger, true)
}
// Sync 同步数据库结构 (使用默认实例 "default")
func Sync(desc string) error {
d := GetDB("default", nil)
if d == nil {
return errors.New("default db not configured")
}
return d.Sync(desc)
}
// Insert 插入数据 (使用默认实例 "default")
func Insert(table string, data any) *ExecResult {
d := GetDB("default", nil)
if d == nil {
return &ExecResult{Error: errors.New("default db not configured")}
}
return d.Insert(table, data)
}
// Update 更新数据 (使用默认实例 "default")
func Update(table string, data any, conditions string, args ...any) *ExecResult {
d := GetDB("default", nil)
if d == nil {
return &ExecResult{Error: errors.New("default db not configured")}
}
return d.Update(table, data, conditions, args...)
}
// Delete 删除数据 (使用默认实例 "default")
func Delete(table string, conditions string, args ...any) *ExecResult {
d := GetDB("default", nil)
if d == nil {
return &ExecResult{Error: errors.New("default db not configured")}
}
return d.Delete(table, conditions, args...)
}
// Query 查询数据 (使用默认实例 "default")
func Query(query string, args ...any) *QueryResult {
d := GetDB("default", nil)
if d == nil {
return &QueryResult{Error: errors.New("default db not configured")}
}
return d.Query(query, args...)
}
// Exec 执行 SQL (使用默认实例 "default")
func Exec(query string, args ...any) *ExecResult {
d := GetDB("default", nil)
if d == nil {
return &ExecResult{Error: errors.New("default db not configured")}
}
return d.Exec(query, args...)
}
// Begin 开始事务 (使用默认实例 "default")
func Begin() *Tx {
d := GetDB("default", nil)
if d == nil {
return &Tx{Error: errors.New("default db not configured")}
}
return d.Begin()
}
func getDB(name string, logger *log.Logger, useCache bool) *DB {
if logger == nil {
logger = log.DefaultLogger
@ -474,6 +537,9 @@ func getDB(name string, logger *log.Logger, useCache bool) *DB {
db.conn = conn
db.tables = make(map[string]*TableStruct)
db.tablesLock = new(sync.RWMutex)
if conf.Type == "sqlite" || conf.Type == "sqlite3" {
db.sqliteMu = new(sync.Mutex)
}
if conf.ReadonlyHosts != nil {
readonlyConnections := make([]*sql.DB, 0)
@ -492,6 +558,9 @@ func getDB(name string, logger *log.Logger, useCache bool) *DB {
db.Error = nil
db.Config = conf
if (conf.Type == "sqlite" || conf.Type == "sqlite3") && conf.MaxOpens == 0 {
conf.MaxOpens = 100
}
if conf.MaxIdles > 0 {
conn.SetMaxIdleConns(conf.MaxIdles)
}
@ -504,6 +573,25 @@ func getDB(name string, logger *log.Logger, useCache bool) *DB {
if conf.LogSlow == 0 {
conf.LogSlow = config.Duration(1000 * time.Millisecond)
}
if conf.Type == "sqlite" || conf.Type == "sqlite3" {
baseExecRaw(conn, nil, "PRAGMA journal_mode=WAL")
baseExecRaw(conn, nil, "PRAGMA synchronous=NORMAL")
baseExecRaw(conn, nil, "PRAGMA busy_timeout=5000")
baseExecRaw(conn, nil, "PRAGMA temp_store=MEMORY")
baseExecRaw(conn, nil, "PRAGMA cache_size=-2000")
// Dynamic mmap_size: 1/4 of system memory, max 30GB
mmapLimit := int64(30000000000)
sysMemStr := runShell("sysctl -n hw.memsize || free -b | awk '/Mem:/ {print $2}'")
if sysMem := cast.Int64(sysMemStr); sysMem > 0 {
if mmapLimit > sysMem/4 {
mmapLimit = sysMem / 4
}
}
baseExecRaw(conn, nil, fmt.Sprintf("PRAGMA mmap_size=%d", mmapLimit))
}
if useCache {
dbInstancesLock.Lock()
dbInstances[name] = db
@ -528,6 +616,13 @@ func getPoolForHost(conf *Config, host string) (*sql.DB, error) {
if connector := connectors[conf.Type]; connector != nil {
return sql.OpenDB(connector(conf, conf.pwd, conf.tls)), nil
} else {
if (conf.Type == "sqlite" || conf.Type == "sqlite3") && !strings.Contains(conf.Args, "journal_mode") {
if conf.Args != "" {
conf.Args += "&"
}
conf.Args += "_journal_mode=WAL&_busy_timeout=5000&_pragma=synchronous(1)&_pragma=cache_size(-2000)"
}
dsn := ""
args := make([]string, 0)
if conf.SSL != "" {
@ -561,6 +656,7 @@ func (db *DB) CopyByLogger(logger *log.Logger) *DB {
newDB.Config = db.Config
newDB.tables = db.tables
newDB.tablesLock = db.tablesLock
newDB.sqliteMu = db.sqliteMu
if logger == nil {
logger = log.DefaultLogger
}
@ -597,6 +693,7 @@ func (db *DB) GetOriginDB() *sql.DB {
func (db *DB) Prepare(query string) *Stmt {
stmt := basePrepare(db.conn, nil, query)
stmt.logger = db.logger
stmt.sqliteMu = db.sqliteMu
if stmt.Error != nil {
db.logger.LogError(stmt.Error.Error())
}
@ -613,19 +710,22 @@ func (db *DB) Quotes(texts []string) string {
func (db *DB) Begin() *Tx {
if db.conn == nil {
return &Tx{db: db, QuoteTag: db.QuoteTag, logSlow: db.Config.LogSlow.TimeDuration(), Error: errors.New("operate on a bad connection"), logger: db.logger}
return &Tx{db: db, sqliteMu: db.sqliteMu, QuoteTag: db.QuoteTag, logSlow: db.Config.LogSlow.TimeDuration(), Error: errors.New("operate on a bad connection"), logger: db.logger}
}
sqlTx, err := db.conn.Begin()
if err != nil {
db.logger.LogError(err.Error())
return &Tx{db: db, QuoteTag: db.QuoteTag, logSlow: db.Config.LogSlow.TimeDuration(), Error: err, logger: db.logger}
return &Tx{db: db, sqliteMu: db.sqliteMu, QuoteTag: db.QuoteTag, logSlow: db.Config.LogSlow.TimeDuration(), Error: err, logger: db.logger}
}
return &Tx{db: db, QuoteTag: db.QuoteTag, logSlow: db.Config.LogSlow.TimeDuration(), conn: sqlTx, logger: db.logger}
return &Tx{db: db, sqliteMu: db.sqliteMu, QuoteTag: db.QuoteTag, logSlow: db.Config.LogSlow.TimeDuration(), conn: sqlTx, logger: db.logger}
}
func (db *DB) Exec(query string, args ...any) *ExecResult {
query, args = db.rewriteFTS(query, args)
r := baseExec(db.conn, nil, query, args...)
args = flatArgs(args)
db.lock()
r := baseExecRaw(db.conn, nil, query, args...)
db.unlock()
r.logger = db.logger
if r.Error != nil {
db.logger.LogQueryError(r.Error.Error(), query, args, r.usedTime)
@ -637,20 +737,14 @@ func (db *DB) Exec(query string, args ...any) *ExecResult {
return r
}
func (db *DB) rawExec(query string, args ...any) *ExecResult {
return db.Exec(query, args...)
}
func (db *DB) Query(query string, args ...any) *QueryResult {
query, args = db.rewriteFTS(query, args)
conn := db.conn
if db.readonlyConnections != nil {
connNum := len(db.readonlyConnections)
if connNum == 1 {
conn = db.readonlyConnections[0]
} else {
p := rand.Int(0, connNum-1)
conn = db.readonlyConnections[p]
}
}
r := baseQuery(conn, nil, query, args...)
args = flatArgs(args)
r := baseQueryRaw(db.conn, nil, query, args...)
r.logger = db.logger
if r.Error != nil {
db.logger.LogQueryError(r.Error.Error(), query, args, r.usedTime)
@ -662,6 +756,24 @@ func (db *DB) Query(query string, args ...any) *QueryResult {
return r
}
func (db *DB) rawQuery(query string, args ...any) *QueryResult {
return db.Query(query, args...)
}
func (db *DB) lock() {
if db.sqliteMu != nil {
db.sqliteMu.Lock()
}
}
func (db *DB) unlock() {
if db.sqliteMu != nil {
db.sqliteMu.Unlock()
}
}
var identifierRegex = `(?:['"` + "`" + `][^'"` + "`" + `]+['"` + "`" + `]|[\w\-]+)`
var likeFieldReg = regexp.MustCompile(`(?i)(` + identifierRegex + `(?:\.` + identifierRegex + `)*)\s+LIKE\s*$`)
var likeLiteralReg = regexp.MustCompile(`(?i)(` + identifierRegex + `(?:\.` + identifierRegex + `)*)\s+LIKE\s+(['"])(%?[^'"]*?%?)(['"])`)
@ -854,44 +966,17 @@ func (db *DB) extractTableName(query string, field string) string {
func (db *DB) Insert(table string, data any) *ExecResult {
query, values := db.MakeInsertSql(table, data, false)
r := baseExec(db.conn, nil, query, values...)
r.logger = db.logger
if r.Error != nil {
db.logger.LogQueryError(r.Error.Error(), query, values, r.usedTime)
} else {
if db.Config.LogSlow > 0 && r.usedTime >= float32(db.Config.LogSlow.TimeDuration()/time.Millisecond) {
db.logger.LogQuery(query, values, r.usedTime)
}
}
return r
return db.Exec(query, values...)
}
func (db *DB) Replace(table string, data any) *ExecResult {
query, values := db.MakeInsertSql(table, data, true)
r := baseExec(db.conn, nil, query, values...)
r.logger = db.logger
if r.Error != nil {
db.logger.LogQueryError(r.Error.Error(), query, values, r.usedTime)
} else {
if db.Config.LogSlow > 0 && r.usedTime >= float32(db.Config.LogSlow.TimeDuration()/time.Millisecond) {
db.logger.LogQuery(query, values, r.usedTime)
}
}
return r
return db.Exec(query, values...)
}
func (db *DB) Update(table string, data any, conditions string, args ...any) *ExecResult {
query, values := db.MakeUpdateSql(table, data, conditions, args...)
r := baseExec(db.conn, nil, query, values...)
r.logger = db.logger
if r.Error != nil {
db.logger.LogQueryError(r.Error.Error(), query, values, r.usedTime)
} else {
if db.Config.LogSlow > 0 && r.usedTime >= float32(db.Config.LogSlow.TimeDuration()/time.Millisecond) {
db.logger.LogQuery(query, values, r.usedTime)
}
}
return r
return db.Exec(query, values...)
}
func (db *DB) Delete(table string, conditions string, args ...any) *ExecResult {
@ -901,16 +986,7 @@ func (db *DB) Delete(table string, conditions string, args ...any) *ExecResult {
conditions = " where " + conditions
}
query := fmt.Sprintf("delete from %s%s", db.Quote(table), conditions)
r := baseExec(db.conn, nil, query, args...)
r.logger = db.logger
if r.Error != nil {
db.logger.LogQueryError(r.Error.Error(), query, args, r.usedTime)
} else {
if db.Config.LogSlow > 0 && r.usedTime >= float32(db.Config.LogSlow.TimeDuration()/time.Millisecond) {
db.logger.LogQuery(query, args, r.usedTime)
}
}
return r
return db.Exec(query, args...)
}
// Shadow delete
@ -944,7 +1020,7 @@ func (db *DB) getTable(table string) *TableStruct {
var query string
if db.Config.Type == "mysql" {
query = "SELECT COLUMN_NAME, DATA_TYPE, CHARACTER_MAXIMUM_LENGTH, COLUMN_KEY FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?"
res := db.Query(query, db.Config.DB, table)
res := db.rawQuery(query, db.Config.DB, table)
rows := res.MapResults()
for _, row := range rows {
col := cast.String(row["COLUMN_NAME"])
@ -963,7 +1039,7 @@ func (db *DB) getTable(table string) *TableStruct {
}
} else if db.Config.Type == "postgres" || db.Config.Type == "pgx" {
query = "SELECT column_name, data_type, character_maximum_length FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = ?"
res := db.Query(query, table)
res := db.rawQuery(query, table)
rows := res.MapResults()
for _, row := range rows {
col := cast.String(row["column_name"])
@ -983,14 +1059,14 @@ func (db *DB) getTable(table string) *TableStruct {
}
} else if isFileDB(db.Config.Type) {
// For SQLite
query = fmt.Sprintf("PRAGMA table_info(%s)", db.Quote(table))
res := db.Query(query)
query := fmt.Sprintf("PRAGMA table_info(%s)", db.Quote(table))
res := db.rawQuery(query)
rows := res.MapResults()
for _, row := range rows {
colName := cast.String(row["name"])
colType := strings.ToUpper(cast.String(row["type"]))
isPk := cast.Int(row["pk"]) > 0
ts.Columns = append(ts.Columns, colName)
if colName == "autoVersion" {
ts.VersionField = "autoVersion"
@ -1015,19 +1091,19 @@ func (db *DB) getTable(table string) *TableStruct {
shadowTable := table + "_deleted"
if db.Config.Type == "mysql" {
query = "SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?"
res := db.Query(query, db.Config.DB, shadowTable)
res := db.rawQuery(query, db.Config.DB, shadowTable)
if res.StringOnR1C1() != "" {
ts.HasShadowTable = true
}
} else if db.Config.Type == "postgres" || db.Config.Type == "pgx" {
query = "SELECT table_name FROM information_schema.tables WHERE table_schema = current_schema() AND table_name = ?"
res := db.Query(query, shadowTable)
res := db.rawQuery(query, shadowTable)
if res.StringOnR1C1() != "" {
ts.HasShadowTable = true
}
} else if isFileDB(db.Config.Type) {
query = "SELECT name FROM sqlite_master WHERE type='table' AND name=?"
res := db.Query(query, shadowTable)
res := db.rawQuery(query, shadowTable)
if res.StringOnR1C1() != "" {
ts.HasShadowTable = true
}

View File

@ -10,6 +10,7 @@ import (
"apigo.cc/go/cast"
"apigo.cc/go/db"
"apigo.cc/go/file"
"apigo.cc/go/shell"
_ "apigo.cc/go/db/mysql"
@ -18,7 +19,7 @@ import (
func TestMain(m *testing.M) {
code := m.Run()
os.Remove("test.db")
file.Remove("test.db")
os.Exit(code)
}

View File

@ -1,18 +1,18 @@
package db_test
import (
"os"
"strings"
"testing"
"apigo.cc/go/db"
"apigo.cc/go/file"
_ "modernc.org/sqlite"
)
func TestAutonomousFTS(t *testing.T) {
dbPath := "test_fts.db"
dbInst := db.GetDB("sqlite://"+dbPath, nil)
defer os.Remove(dbPath)
defer file.Remove(dbPath)
defer dbInst.Exec("DROP TABLE IF EXISTS fts_test")
defer dbInst.Exec("DROP TABLE IF EXISTS fts_test_fts")

10
Log.go
View File

@ -6,11 +6,11 @@ import (
)
type DBLog struct {
DbType string `log:"pos:6,color:blue"`
Dsn string `log:"pos:7,color:gray,withoutkey:true"`
Query string `log:"pos:8,color:cyan"`
QueryArgs string `log:"pos:9,color:gray"`
UsedTime float32 `log:"pos:10,format:%.2fms"`
DbType string `log:"pos:7,color:blue"`
Dsn string `log:"pos:8,color:gray,withoutkey:true"`
Query string `log:"pos:9,color:cyan"`
QueryArgs string `log:"pos:10,color:gray"`
UsedTime float32 `log:"pos:11,format:%.2fms"`
}
func (l *DBLog) Reset() {

View File

@ -2,7 +2,6 @@ package db
import (
"database/sql"
"encoding/json"
"errors"
"fmt"
"reflect"
@ -436,7 +435,7 @@ func (r *QueryResult) makeResults(results any, rows *sql.Rows) error {
if s, ok := val.Interface().(string); ok {
storedValue := new(any)
if s != "" {
_ = json.Unmarshal([]byte(s), storedValue)
cast.UnmarshalJSON([]byte(s), storedValue)
}
cast.Convert(convertedObject.Interface(), storedValue)
field.Set(convertedObject.Elem())
@ -537,7 +536,7 @@ func fixValue(colName string, colType string, v reflect.Value) reflect.Value {
case strings.Contains(colType, "JSON"):
if str != "" && (str[0] == '{' || str[0] == '[') {
var out any
if err := json.Unmarshal([]byte(str), &out); err == nil {
if err := cast.UnmarshalJSON([]byte(str), &out); err == nil {
return reflect.ValueOf(out)
}
}

View File

@ -294,7 +294,7 @@ func (db *DB) Sync(desc string) error {
for _, group := range groups {
for _, table := range group.Tables {
db.tablesLock.Lock()
db.tables[table.Name] = table
delete(db.tables, table.Name)
db.tablesLock.Unlock()
err := db.CheckTable(table)
@ -307,6 +307,8 @@ func (db *DB) Sync(desc string) error {
return outErr
}
// CheckTable 检查并同步单个表结构
func (db *DB) CheckTable(table *TableStruct) error {
fieldSets := make([]string, 0)
@ -424,34 +426,40 @@ func (db *DB) CheckTable(table *TableStruct) error {
tmpFields := []struct {
Name string
Type string
Notnull bool
Notnull int
Dflt_value any
Pk bool
Pk int
}{}
db.Query("PRAGMA table_info(" + db.Quote(table.Name) + ")").To(&tmpFields)
if err := db.Query("PRAGMA table_info(" + db.Quote(table.Name) + ")").To(&tmpFields); err != nil {
return err
}
for _, f := range tmpFields {
oldFieldList = append(oldFieldList, &tableFieldDesc{
Field: f.Name,
Type: f.Type,
Null: cast.If(f.Notnull, "NO", "YES"),
Key: cast.If(f.Pk, "PRI", ""),
Null: cast.If(f.Notnull != 0, "NO", "YES"),
Key: cast.If(f.Pk != 0, "PRI", ""),
Default: cast.String(f.Dflt_value),
})
}
tmpIndexes := []struct {
Name string
Unique bool
Unique int
Origin string
Partial int
}{}
db.Query("PRAGMA index_list(" + db.Quote(table.Name) + ")").To(&tmpIndexes)
if err := db.Query("PRAGMA index_list(" + db.Quote(table.Name) + ")").To(&tmpIndexes); err != nil {
return err
}
for _, i := range tmpIndexes {
tmpIndexInfo := []struct {
Name string
Seqno int
Cid int
}{}
db.Query("PRAGMA index_info(" + db.Quote(i.Name) + ")").To(&tmpIndexInfo)
if err := db.Query("PRAGMA index_info(" + db.Quote(i.Name) + ")").To(&tmpIndexInfo); err != nil {
return err
}
if len(tmpIndexInfo) > 0 {
oldIndexInfos = append(oldIndexInfos, &tableKeyDesc{
Key_name: i.Name,
@ -515,6 +523,9 @@ func (db *DB) CheckTable(table *TableStruct) error {
for keyId := range oldIndexes {
if keyId != "PRIMARY" && !isPostgres && strings.ToLower(keySetFields[keyId]) != strings.ToLower(oldIndexes[keyId]) {
if strings.HasPrefix(db.Config.Type, "sqlite") {
if strings.HasPrefix(keyId, "sqlite_autoindex_") {
continue
}
actions = append(actions, "DROP INDEX "+db.Quote(keyId))
} else {
actions = append(actions, "DROP KEY "+db.Quote(keyId))

View File

@ -1,17 +1,17 @@
package db_test
import (
"os"
"testing"
"apigo.cc/go/db"
"apigo.cc/go/file"
_ "modernc.org/sqlite"
)
func TestSchemaSync(t *testing.T) {
dbPath := "test_schema.db"
dbInst := db.GetDB("sqlite://"+dbPath, nil)
defer os.Remove(dbPath)
defer file.Remove(dbPath)
defer dbInst.Exec("DROP TABLE IF EXISTS test_table")
defer dbInst.Exec("DROP TABLE IF EXISTS test_table_deleted")
@ -40,7 +40,7 @@ test_table SD // Test table with shadow delete
func TestAutoDetectShadow(t *testing.T) {
dbPath := "auto_detect.db"
dbInst := db.GetDB("sqlite://"+dbPath, nil)
defer os.Remove(dbPath)
defer file.Remove(dbPath)
defer dbInst.Exec("DROP TABLE IF EXISTS test_auto")
defer dbInst.Exec("DROP TABLE IF EXISTS test_auto_deleted")

View File

@ -3,11 +3,13 @@ package db
import (
"database/sql"
"errors"
"sync"
"time"
)
type Stmt struct {
conn *sql.Stmt
sqliteMu *sync.Mutex
lastSql *string
lastArgs []any
Error error
@ -15,6 +17,10 @@ type Stmt struct {
}
func (stmt *Stmt) Exec(args ...any) *ExecResult {
if stmt.sqliteMu != nil {
stmt.sqliteMu.Lock()
defer stmt.sqliteMu.Unlock()
}
stmt.lastArgs = args
if stmt.conn == nil {
return &ExecResult{Sql: stmt.lastSql, Args: stmt.lastArgs, usedTime: -1, logger: stmt.logger, Error: errors.New("operate on a bad connection")}

18
TEST.md
View File

@ -2,8 +2,8 @@
## 📊 概览
- **模块**: `apigo.cc/go/db`
- **总测试用例**: 7
- **通过**: 7
- **总测试用例**: 13
- **通过**: 13
- **失败**: 0
- **编译状态**: 成功 (Success)
- **测试日期**: 2026-05-13
@ -11,20 +11,24 @@
## ✅ 详细详情
| 测试用例 | 状态 | 耗时 | 备注 |
| :--- | :--- | :--- | :--- |
| `TestTableProbing` | 通过 | 0.00s | 验证表结构探测 |
| `TestMakeInsertSql` | 通过 | 0.00s | 验证 Struct 模型的 SQL 生成逻辑 |
| `TestBaseSelect` | 通过 | 0.00s | 验证结果绑定 (Struct, Map, 基础类型) |
| `TestInsertReplaceUpdateDelete` | 通过 | 0.01s | 验证 SQLite 下s CRUD 基本操作 |
| `TestInsertReplaceUpdateDelete` | 通过 | 0.01s | 验证 SQLite 下 CRUD 基本操作 |
| `TestTransaction` | 通过 | 0.03s | 验证事务隔离、回滚与提交 |
| `TestAutonomousFTS` | 通过 | 0.01s | 验证多种引用风格下的 FTS 重定向 |
| `TestComplexIdentifierFTS` | 通过 | 0.01s | 验证带横杠和表前缀的复杂标识符 FTS 重定向 |
| `TestAutonomousFTS` | 通过 | 0.01s | 验证全文搜索功能 |
| `TestSchemaSync` | 通过 | 0.01s | 验证 DSL 同步、影子删除、版本号乐观锁及泛型 API |
| `TestAutoDetectShadow` | 通过 | 0.00s | 验证影子表自动检测 |
| `TestSmartDelete` | 通过 | 0.01s | 验证智能删除 (物理/影子) |
| `TestGenericQuery` | 通过 | 0.00s | 验证泛型查询映射 |
| `TestAutoRandomID` | 通过 | 0.01s | 验证 char(N) 主键的自动 ID 填充 |
| `TestVersionControl` | 通过 | 0.00s | 验证版本控制递增 |
## 🚀 性能基准 (Benchmarks)
| 基准测试 | 迭代次数 | 耗时 | 内存分配 | 备注 |
| :--- | :--- | :--- | :--- | :--- |
| `BenchmarkForPool` | 103951 | 11821 ns/op | 1356 B/op (37 allocs) | 增加了复杂标识符解析开销 |
| `BenchmarkForPoolParallel` | 84481 | 13904 ns/op | 1681 B/op (39 allocs) | 验证高并发下的查询稳定性 |
| `BenchmarkForPool` | 106807 | 12230 ns/op | - | 验证 SQLite 下的查询绑定性能 (v1.0.11) |
| `BenchmarkForPoolParallel` | 86833 | 15723 ns/op | - | 验证高并发下的查询稳定性 (v1.0.11) |
## 🛠 环境
- **OS**: darwin (macOS)

89
Tx.go
View File

@ -5,12 +5,15 @@ import (
"errors"
"fmt"
"strings"
"sync"
"time"
)
type Tx struct {
conn *sql.Tx
db *DB
sqliteMu *sync.Mutex
hasLock bool
lastSql *string
lastArgs []any
Error error
@ -29,6 +32,7 @@ func (tx *Tx) Quotes(texts []string) string {
}
func (tx *Tx) Commit() error {
defer tx.unlock()
if tx.isCommittedOrRollbacked {
return nil
}
@ -45,6 +49,7 @@ func (tx *Tx) Commit() error {
}
func (tx *Tx) Rollback() error {
defer tx.unlock()
if tx.isCommittedOrRollbacked {
return nil
}
@ -78,6 +83,7 @@ func (tx *Tx) CheckFinished() error {
}
func (tx *Tx) Prepare(query string) *Stmt {
tx.lock()
tx.lastSql = &query
r := basePrepare(nil, tx.conn, query)
r.logger = tx.logger
@ -89,9 +95,11 @@ func (tx *Tx) Prepare(query string) *Stmt {
func (tx *Tx) Exec(query string, args ...any) *ExecResult {
query, args = tx.db.rewriteFTS(query, args)
args = flatArgs(args)
tx.lock()
tx.lastSql = &query
tx.lastArgs = args
r := baseExec(nil, tx.conn, query, args...)
r := baseExecRaw(nil, tx.conn, query, args...)
r.logger = tx.logger
if r.Error != nil {
tx.logger.LogQueryError(r.Error.Error(), *tx.lastSql, tx.lastArgs, r.usedTime)
@ -105,9 +113,11 @@ func (tx *Tx) Exec(query string, args ...any) *ExecResult {
func (tx *Tx) Query(query string, args ...any) *QueryResult {
query, args = tx.db.rewriteFTS(query, args)
args = flatArgs(args)
// Query in Tx doesn't acquire lock unless it's already held by a previous write
tx.lastSql = &query
tx.lastArgs = args
r := baseQuery(nil, tx.conn, query, args...)
r := baseQueryRaw(nil, tx.conn, query, args...)
r.logger = tx.logger
if r.Error != nil {
tx.logger.LogQueryError(r.Error.Error(), *tx.lastSql, tx.lastArgs, r.usedTime)
@ -119,52 +129,20 @@ func (tx *Tx) Query(query string, args ...any) *QueryResult {
return r
}
func (tx *Tx) Insert(table string, data any) *ExecResult {
query, values := tx.MakeInsertSql(table, data, false)
tx.lastSql = &query
tx.lastArgs = values
r := baseExec(nil, tx.conn, query, values...)
r.logger = tx.logger
if r.Error != nil {
tx.logger.LogQueryError(r.Error.Error(), *tx.lastSql, tx.lastArgs, r.usedTime)
} else {
if tx.logSlow > 0 && r.usedTime >= float32(tx.logSlow/time.Millisecond) {
tx.logger.LogQuery(*tx.lastSql, tx.lastArgs, r.usedTime)
}
}
return r
return tx.Exec(query, values...)
}
func (tx *Tx) Replace(table string, data any) *ExecResult {
query, values := tx.MakeInsertSql(table, data, true)
tx.lastSql = &query
tx.lastArgs = values
r := baseExec(nil, tx.conn, query, values...)
r.logger = tx.logger
if r.Error != nil {
tx.logger.LogQueryError(r.Error.Error(), *tx.lastSql, tx.lastArgs, r.usedTime)
} else {
if tx.logSlow > 0 && r.usedTime >= float32(tx.logSlow/time.Millisecond) {
tx.logger.LogQuery(*tx.lastSql, tx.lastArgs, r.usedTime)
}
}
return r
return tx.Exec(query, values...)
}
func (tx *Tx) Update(table string, data any, conditions string, args ...any) *ExecResult {
query, values := tx.MakeUpdateSql(table, data, conditions, args...)
tx.lastSql = &query
tx.lastArgs = values
r := baseExec(nil, tx.conn, query, values...)
r.logger = tx.logger
if r.Error != nil {
tx.logger.LogQueryError(r.Error.Error(), *tx.lastSql, tx.lastArgs, r.usedTime)
} else {
if tx.logSlow > 0 && r.usedTime >= float32(tx.logSlow/time.Millisecond) {
tx.logger.LogQuery(*tx.lastSql, tx.lastArgs, r.usedTime)
}
}
return r
return tx.Exec(query, values...)
}
func (tx *Tx) Delete(table string, conditions string, args ...any) *ExecResult {
@ -187,24 +165,31 @@ func (tx *Tx) Delete(table string, conditions string, args ...any) *ExecResult {
colList = " select *"
}
moveQuery := fmt.Sprintf("insert into %s%s from %s%s", tx.Quote(table+"_deleted"), colList, tx.Quote(table), where)
r := baseExec(nil, tx.conn, moveQuery, args...)
// Use Exec to handle locking
r := tx.Exec(moveQuery, args...)
if r.Error != nil {
tx.logger.LogQueryError(r.Error.Error(), moveQuery, args, r.usedTime)
return r
}
}
query := fmt.Sprintf("delete from %s%s", tx.Quote(table), where)
tx.lastSql = &query
tx.lastArgs = args
r := baseExec(nil, tx.conn, query, args...)
r.logger = tx.logger
if r.Error != nil {
tx.logger.LogQueryError(r.Error.Error(), *tx.lastSql, tx.lastArgs, r.usedTime)
} else {
if tx.logSlow > 0 && r.usedTime >= float32(tx.logSlow/time.Millisecond) {
tx.logger.LogQuery(*tx.lastSql, tx.lastArgs, r.usedTime)
}
}
return r
return tx.Exec(query, args...)
}
func (tx *Tx) lock() {
if tx.sqliteMu == nil || tx.hasLock {
return
}
tx.sqliteMu.Lock()
tx.hasLock = true
}
func (tx *Tx) unlock() {
if tx.sqliteMu == nil || !tx.hasLock {
return
}
tx.sqliteMu.Unlock()
tx.hasLock = false
}

View File

@ -1,10 +1,10 @@
package db_test
import (
"os"
"testing"
"apigo.cc/go/db"
"apigo.cc/go/file"
"apigo.cc/go/log"
_ "modernc.org/sqlite"
)
@ -14,7 +14,7 @@ func TestSmartDelete(t *testing.T) {
dbPath := "./test_smart_delete.db"
dbName := "test_delete"
os.Remove(dbPath)
file.Remove(dbPath)
db.SetConfigForTest(dbName, &db.Config{
Type: "sqlite",
@ -27,7 +27,7 @@ func TestSmartDelete(t *testing.T) {
}
defer func() {
dbInst.Destroy()
os.Remove(dbPath)
file.Remove(dbPath)
}()
// Create table and shadow table

View File

@ -1,10 +1,10 @@
package db_test
import (
"os"
"testing"
"apigo.cc/go/cast"
"apigo.cc/go/db"
"apigo.cc/go/file"
"apigo.cc/go/log"
_ "modernc.org/sqlite"
)
@ -12,7 +12,7 @@ import (
func TestGenericQuery(t *testing.T) {
db.ResetAllForTest()
dbPath := "./test_generic.db"
os.Remove(dbPath)
file.Remove(dbPath)
db.SetConfigForTest("test_generic", &db.Config{Type: "sqlite", Host: dbPath})
dbInst := db.GetDB("test_generic", log.DefaultLogger)
@ -21,7 +21,7 @@ func TestGenericQuery(t *testing.T) {
}
defer func() {
dbInst.Destroy()
os.Remove(dbPath)
file.Remove(dbPath)
}()
r := dbInst.Query("SELECT 1 as num, 'hello' as str")

22
go.mod
View File

@ -3,15 +3,15 @@ module apigo.cc/go/db
go 1.25.0
require (
apigo.cc/go/cast v1.3.0
apigo.cc/go/config v1.3.0
apigo.cc/go/crypto v1.3.0
apigo.cc/go/id v1.3.0
apigo.cc/go/log v1.3.2
apigo.cc/go/rand v1.3.0
apigo.cc/go/redis v1.3.0
apigo.cc/go/safe v1.3.0
apigo.cc/go/shell v1.3.0
apigo.cc/go/cast v1.3.3
apigo.cc/go/config v1.3.1
apigo.cc/go/crypto v1.3.1
apigo.cc/go/file v1.3.2
apigo.cc/go/id v1.3.1
apigo.cc/go/log v1.3.4
apigo.cc/go/redis v1.3.2
apigo.cc/go/safe v1.3.1
apigo.cc/go/shell v1.3.1
github.com/go-sql-driver/mysql v1.10.0
github.com/jackc/pgx/v5 v5.9.2
github.com/mitchellh/mapstructure v1.5.0
@ -19,8 +19,8 @@ require (
)
require (
apigo.cc/go/encoding v1.3.0 // indirect
apigo.cc/go/file v1.3.0 // indirect
apigo.cc/go/encoding v1.3.1 // indirect
apigo.cc/go/rand v1.3.1 // indirect
filippo.io/edwards25519 v1.2.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/gomodule/redigo v2.0.0+incompatible // indirect

44
go.sum
View File

@ -1,25 +1,25 @@
apigo.cc/go/cast v1.3.0 h1:ZTcLYijkqZjSWSCSpJUWMfzJYeJKbwKxquKkPrFsROQ=
apigo.cc/go/cast v1.3.0/go.mod h1:lGlwImiOvHxG7buyMWhFzcdvQzmSaoKbmr7bcDfUpHk=
apigo.cc/go/config v1.3.0 h1:TwI3bv3D+BJrAnFx+o62HQo3FarY2Ge3SCGsKchFYGg=
apigo.cc/go/config v1.3.0/go.mod h1:88lqKEBXlIExFKt1geLONVLYyM+QhRVpBe0ok3OEvjI=
apigo.cc/go/crypto v1.3.0 h1:rGRrrb5O+4M50X5hVUmJQbXx3l87zzlcgzGtUvZrZL8=
apigo.cc/go/crypto v1.3.0/go.mod h1:uSCcmbcFoiltUPMQTSuqmU9nfKEH/lRs7nQ7aa3Z4Mc=
apigo.cc/go/encoding v1.3.0 h1:8jqNHoZBR8vOU/BGsLFebfp1Txa1UxDRpd7YwzIFLJs=
apigo.cc/go/encoding v1.3.0/go.mod h1:kT/uUJiuAOkZ4LzUWrUtk/I0iL1D8aatvD+59bDnHBo=
apigo.cc/go/file v1.3.0 h1:xG9FcY3Rv6Br83r9pq9QsIXFrplx4g8ITOkHSzfzXRg=
apigo.cc/go/file v1.3.0/go.mod h1:pYHBlB/XwsrnWpEh7GIFpbiqobrExfiB+rEN8V2d2kY=
apigo.cc/go/id v1.3.0 h1:Tr2Yj0Rl19lfwW5wBTJ407o/zgo2oVRLE20WWEgJzdE=
apigo.cc/go/id v1.3.0/go.mod h1:AFH3kMFwENfXNyijnAFWEhSF1o3y++UBPem1IUlrcxA=
apigo.cc/go/log v1.3.2 h1:/m3V4MnlYnCG4XPHpWDsa4cw5suMaDVY1SgaVyjnBSo=
apigo.cc/go/log v1.3.2/go.mod h1:dz4bSz9BnOgutkUJJZfX3uDDwsMpUxt7WF50mLK9hgE=
apigo.cc/go/rand v1.3.0 h1:k+UFAhMySwXf+dq8Om9TniZV6fm6gAE0evbrqMEdwQU=
apigo.cc/go/rand v1.3.0/go.mod h1:mZ/4Soa3bk+XvDaqPWJuUe1bfEi4eThBj1XmEAuYxsk=
apigo.cc/go/redis v1.3.0 h1:3NJE3xPXzhCwL+Mh1iyphFrsKWEuPlY26LHJfMVFSeU=
apigo.cc/go/redis v1.3.0/go.mod h1:KPDPwMOER7WJX3Qev24LTeAOSmCl8OApe8iagPDxOUQ=
apigo.cc/go/safe v1.3.0 h1:uctdAUsphT9p60Tk4oS5xPCe0NoIdOHfsYv4PNS0Rok=
apigo.cc/go/safe v1.3.0/go.mod h1:tC9X14V+qh0BqIrVg4UkXbl+2pEN+lj2ZNI8IjDB6Fs=
apigo.cc/go/shell v1.3.0 h1:hdxuYPN/7T2BuM/Ja8AjVUhbRqU/wpi8OjcJVziJ0nw=
apigo.cc/go/shell v1.3.0/go.mod h1:aNJiRWibxlA485yX3t+07IVAbrALKmxzv4oGEUC+hK4=
apigo.cc/go/cast v1.3.3 h1:aln5eDR5DZVWVzZ/y5SJh1gQNgWv2sT82I25NaO9g34=
apigo.cc/go/cast v1.3.3/go.mod h1:lGlwImiOvHxG7buyMWhFzcdvQzmSaoKbmr7bcDfUpHk=
apigo.cc/go/config v1.3.1 h1:wZzUh4oL+fGD6SayVgX6prLPMsniM25etWFcEH8XzIE=
apigo.cc/go/config v1.3.1/go.mod h1:7KHz/1WmtBLM762Lln/TaXh2dmlMvJTLhnlk33zbS3U=
apigo.cc/go/crypto v1.3.1 h1:ulQ2zX9bUWirk0sEacx1Srsjs2Jow7HlZq7ED7msNcg=
apigo.cc/go/crypto v1.3.1/go.mod h1:SwHlBFDPddttWgFFtzsEMla8CM/rcFy9nvdsJjW4CIs=
apigo.cc/go/encoding v1.3.1 h1:y8O58KYAyulkThg1O2ji2BqjnFoSvk42sit9I3z+K7Y=
apigo.cc/go/encoding v1.3.1/go.mod h1:xAJk5b83VZ31mXMTnyp0dfMoBKfT/AHDn0u+cQfojgY=
apigo.cc/go/file v1.3.2 h1:pu4oiDyiqgj3/eykfnJf+/6+A9v/Z0b3ClP5XK+lwG4=
apigo.cc/go/file v1.3.2/go.mod h1:vci4h0Pz94mV6dkniQkuyBYERVYeq7/LX4jJVuCg9hs=
apigo.cc/go/id v1.3.1 h1:pkqi6VeWyQoHuIu0Zbx/RRxIAdM61Js0j6cY1M9XVCk=
apigo.cc/go/id v1.3.1/go.mod h1:P2/vl3tyW3US+ayOFSMoPIOCulNLBngNYPhXJC/Z7J4=
apigo.cc/go/log v1.3.4 h1:UT8Neb9r4QjjbCFbTzw+ZeTxd+DmdmR5gNExeR4Cj+g=
apigo.cc/go/log v1.3.4/go.mod h1:/Q/2r51xWSsrS4QN5U9jLiTw8n6qNC8kG9nuVHweY20=
apigo.cc/go/rand v1.3.1 h1:7FvsI6PtQ5XrWER0dTiLVo0p7GIxRidT/TBKhVy93j8=
apigo.cc/go/rand v1.3.1/go.mod h1:mZ/4Soa3bk+XvDaqPWJuUe1bfEi4eThBj1XmEAuYxsk=
apigo.cc/go/redis v1.3.2 h1:iUWL/CHHnfonz0dJq6/V4IG3QuXBoHA2L1xnoGEbNEQ=
apigo.cc/go/redis v1.3.2/go.mod h1:/k5wcfAzB9jrfd9otabio9CPUxEsLPgEs4oggBG5sbs=
apigo.cc/go/safe v1.3.1 h1:irTCqPAC97gGsX/Lw5AzLelDt1xXLEZIAaVhLELWe9Q=
apigo.cc/go/safe v1.3.1/go.mod h1:XdOpBhN2vkImalaykYXXmEpczqWa1y3ah6/Q72cdRqE=
apigo.cc/go/shell v1.3.1 h1:M8oD0b2HcJuCC6frQFx11b3UTcTx3lATX8XK+YXSVm8=
apigo.cc/go/shell v1.3.1/go.mod h1:ZMdJjpCpWdvsHKUXlelh/AxsV/nWdkH/k3lISfzMdUw=
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

View File

@ -5,13 +5,14 @@ import (
"testing"
"apigo.cc/go/db"
"apigo.cc/go/file"
_ "modernc.org/sqlite"
)
func TestAutoRandomID(t *testing.T) {
dbPath := "id_test.db"
dbset := "sqlite://" + dbPath
defer os.Remove(dbPath)
defer file.Remove(dbPath)
dbInst := db.GetDB(dbset, nil)
// Create table with char(12) primary key

View File

@ -1,16 +1,16 @@
package db
import (
"os"
"testing"
"apigo.cc/go/file"
_ "modernc.org/sqlite"
)
func TestTableProbing(t *testing.T) {
ResetAllForTest()
dbPath := "./test_probing.db"
os.Remove(dbPath)
file.Remove(dbPath)
SetConfigForTest("test_probing", &Config{Type: "sqlite", Host: dbPath})
dbInst := GetDB("test_probing", nil)
if dbInst == nil {
@ -18,7 +18,7 @@ func TestTableProbing(t *testing.T) {
}
defer func() {
dbInst.Destroy()
os.Remove(dbPath)
file.Remove(dbPath)
}()

12
utils.go Normal file
View File

@ -0,0 +1,12 @@
package db
import (
"os/exec"
"strings"
)
func runShell(command string) string {
cmd := exec.Command("bash", "-c", command)
out, _ := cmd.CombinedOutput()
return strings.TrimSpace(string(out))
}

View File

@ -1,11 +1,11 @@
package db_test
import (
"os"
"testing"
"time"
"apigo.cc/go/db"
"apigo.cc/go/file"
"apigo.cc/go/log"
_ "modernc.org/sqlite"
)
@ -13,7 +13,7 @@ import (
func TestVersionControl(t *testing.T) {
db.ResetAllForTest()
dbPath := "./test_version.db"
os.Remove(dbPath)
file.Remove(dbPath)
db.SetConfigForTest("test_version", &db.Config{Type: "sqlite", Host: dbPath})
dbInst := db.GetDB("test_version", log.DefaultLogger)
if dbInst == nil {
@ -21,7 +21,7 @@ func TestVersionControl(t *testing.T) {
}
defer func() {
dbInst.Destroy()
os.Remove(dbPath)
file.Remove(dbPath)
}()