Compare commits
12 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f8ede4fdc7 | ||
|
|
7b1e5054d9 | ||
|
|
6f4d44dc3b | ||
|
|
12651fb142 | ||
|
|
ae534db085 | ||
|
|
b2136e170e | ||
|
|
9cdcdaeecd | ||
|
|
90e7052258 | ||
|
|
53f64af5f2 | ||
|
|
0db7cbc138 | ||
|
|
55e969b522 | ||
|
|
918d561b24 |
7
.gitignore
vendored
7
.gitignore
vendored
@ -1 +1,8 @@
|
||||
.log.meta.json
|
||||
.ai/
|
||||
.geminiignore
|
||||
.gemini
|
||||
env.json
|
||||
env.yml
|
||||
env.yaml
|
||||
/CODE-FULL.md
|
||||
|
||||
89
Base.go
89
Base.go
@ -10,7 +10,6 @@ import (
|
||||
"time"
|
||||
|
||||
"apigo.cc/go/cast"
|
||||
"apigo.cc/go/log"
|
||||
)
|
||||
|
||||
var structFieldsCache = sync.Map{}
|
||||
@ -69,7 +68,10 @@ func basePrepare(db *sql.DB, tx *sql.Tx, query string) *Stmt {
|
||||
}
|
||||
|
||||
func baseExec(db *sql.DB, tx *sql.Tx, query string, args ...any) *ExecResult {
|
||||
args = flatArgs(args)
|
||||
return baseExecRaw(db, tx, query, flatArgs(args)...)
|
||||
}
|
||||
|
||||
func baseExecRaw(db *sql.DB, tx *sql.Tx, query string, args ...any) *ExecResult {
|
||||
var r sql.Result
|
||||
var err error
|
||||
startTime := time.Now()
|
||||
@ -78,10 +80,10 @@ func baseExec(db *sql.DB, tx *sql.Tx, query string, args ...any) *ExecResult {
|
||||
} else if db != nil {
|
||||
r, err = db.Exec(query, args...)
|
||||
} else {
|
||||
return &ExecResult{Sql: &query, Args: args, usedTime: log.MakeUsedTime(startTime, time.Now()), Error: errors.New("operate on a bad connection")}
|
||||
return &ExecResult{Sql: &query, Args: args, usedTime: makeUsedTime(startTime, time.Now()), Error: errors.New("operate on a bad connection")}
|
||||
}
|
||||
endTime := time.Now()
|
||||
usedTime := log.MakeUsedTime(startTime, endTime)
|
||||
usedTime := makeUsedTime(startTime, endTime)
|
||||
|
||||
if err != nil {
|
||||
return &ExecResult{Sql: &query, Args: args, usedTime: usedTime, Error: err}
|
||||
@ -89,6 +91,10 @@ func baseExec(db *sql.DB, tx *sql.Tx, query string, args ...any) *ExecResult {
|
||||
return &ExecResult{Sql: &query, Args: args, usedTime: usedTime, result: r}
|
||||
}
|
||||
|
||||
func makeUsedTime(startTime, endTime time.Time) float32 {
|
||||
return float32(endTime.UnixNano()-startTime.UnixNano()) / 1e6
|
||||
}
|
||||
|
||||
func flatArgs(args []any) []any {
|
||||
for i, arg := range args {
|
||||
if arg == nil {
|
||||
@ -104,8 +110,10 @@ func flatArgs(args []any) []any {
|
||||
}
|
||||
|
||||
func baseQuery(db *sql.DB, tx *sql.Tx, query string, args ...any) *QueryResult {
|
||||
args = flatArgs(args)
|
||||
return baseQueryRaw(db, tx, query, flatArgs(args)...)
|
||||
}
|
||||
|
||||
func baseQueryRaw(db *sql.DB, tx *sql.Tx, query string, args ...any) *QueryResult {
|
||||
var rows *sql.Rows
|
||||
var err error
|
||||
startTime := time.Now()
|
||||
@ -114,10 +122,10 @@ func baseQuery(db *sql.DB, tx *sql.Tx, query string, args ...any) *QueryResult {
|
||||
} else if db != nil {
|
||||
rows, err = db.Query(query, args...)
|
||||
} else {
|
||||
return &QueryResult{Sql: &query, Args: args, usedTime: log.MakeUsedTime(startTime, time.Now()), Error: errors.New("operate on a bad connection")}
|
||||
return &QueryResult{Sql: &query, Args: args, usedTime: makeUsedTime(startTime, time.Now()), Error: errors.New("operate on a bad connection")}
|
||||
}
|
||||
endTime := time.Now()
|
||||
usedTime := log.MakeUsedTime(startTime, endTime)
|
||||
usedTime := makeUsedTime(startTime, endTime)
|
||||
|
||||
if err != nil {
|
||||
return &QueryResult{Sql: &query, Args: args, usedTime: usedTime, Error: err}
|
||||
@ -125,6 +133,7 @@ func baseQuery(db *sql.DB, tx *sql.Tx, query string, args ...any) *QueryResult {
|
||||
return &QueryResult{Sql: &query, Args: args, usedTime: usedTime, rows: rows}
|
||||
}
|
||||
|
||||
|
||||
func quote(quoteTag string, text string) string {
|
||||
a := strings.Split(text, ".")
|
||||
for i, v := range a {
|
||||
@ -140,8 +149,35 @@ func quotes(quoteTag string, texts []string) string {
|
||||
return strings.Join(texts, ",")
|
||||
}
|
||||
|
||||
func makeInsertSql(quoteTag string, table string, data any, useReplace bool, versionField string, nextVer int64, idField string, nextId string) (string, []any) {
|
||||
func makeInsertSql(quoteTag string, table string, data any, useReplace bool, versionField string, nextVer int64, idField string, nextId string, ts *TableStruct) (string, []any) {
|
||||
keys, vars, values := MakeKeysVarsValues(data)
|
||||
|
||||
// 全文检索影子列自动分词处理
|
||||
if ts != nil {
|
||||
for _, col := range ts.Columns {
|
||||
if strings.HasSuffix(col, "_tokens") {
|
||||
originCol := strings.TrimSuffix(col, "_tokens")
|
||||
for i, k := range keys {
|
||||
if k == originCol {
|
||||
found := false
|
||||
for _, k2 := range keys {
|
||||
if k2 == col {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
keys = append(keys, col)
|
||||
vars = append(vars, "?")
|
||||
values = append(values, BigramTokenize(cast.String(values[i])))
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if versionField != "" {
|
||||
found := false
|
||||
for _, k := range keys {
|
||||
@ -181,9 +217,36 @@ func makeInsertSql(quoteTag string, table string, data any, useReplace bool, ver
|
||||
return query, values
|
||||
}
|
||||
|
||||
func makeUpdateSql(quoteTag string, table string, data any, conditions string, versionField string, nextVer int64, args ...any) (string, []any) {
|
||||
func makeUpdateSql(quoteTag string, table string, data any, conditions string, versionField string, nextVer int64, ts *TableStruct, args ...any) (string, []any) {
|
||||
args = flatArgs(args)
|
||||
keys, vars, values := MakeKeysVarsValues(data)
|
||||
|
||||
// 全文检索影子列自动分词处理
|
||||
if ts != nil {
|
||||
for _, col := range ts.Columns {
|
||||
if strings.HasSuffix(col, "_tokens") {
|
||||
originCol := strings.TrimSuffix(col, "_tokens")
|
||||
for i, k := range keys {
|
||||
if k == originCol {
|
||||
found := false
|
||||
for _, k2 := range keys {
|
||||
if k2 == col {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
keys = append(keys, col)
|
||||
vars = append(vars, "?")
|
||||
values = append(values, BigramTokenize(cast.String(values[i])))
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
newKeys := make([]string, 0, len(keys))
|
||||
newValues := make([]any, 0, len(values))
|
||||
var oldVersion any
|
||||
@ -227,7 +290,7 @@ func (db *DB) MakeInsertSql(table string, data any, useReplace bool) (string, []
|
||||
if ts.IdField != "" {
|
||||
nextId = db.NextID(table)
|
||||
}
|
||||
return makeInsertSql(db.QuoteTag, table, data, useReplace, ts.VersionField, nextVer, ts.IdField, nextId)
|
||||
return makeInsertSql(db.QuoteTag, table, data, useReplace, ts.VersionField, nextVer, ts.IdField, nextId, ts)
|
||||
}
|
||||
|
||||
func (db *DB) MakeUpdateSql(table string, data any, conditions string, args ...any) (string, []any) {
|
||||
@ -236,7 +299,7 @@ func (db *DB) MakeUpdateSql(table string, data any, conditions string, args ...a
|
||||
if ts.VersionField != "" {
|
||||
nextVer = db.NextVersion(table)
|
||||
}
|
||||
return makeUpdateSql(db.QuoteTag, table, data, conditions, ts.VersionField, nextVer, args...)
|
||||
return makeUpdateSql(db.QuoteTag, table, data, conditions, ts.VersionField, nextVer, ts, args...)
|
||||
}
|
||||
|
||||
func (tx *Tx) MakeInsertSql(table string, data any, useReplace bool) (string, []any) {
|
||||
@ -249,7 +312,7 @@ func (tx *Tx) MakeInsertSql(table string, data any, useReplace bool) (string, []
|
||||
if ts.IdField != "" {
|
||||
nextId = tx.db.NextID(table)
|
||||
}
|
||||
return makeInsertSql(tx.QuoteTag, table, data, useReplace, ts.VersionField, nextVer, ts.IdField, nextId)
|
||||
return makeInsertSql(tx.QuoteTag, table, data, useReplace, ts.VersionField, nextVer, ts.IdField, nextId, ts)
|
||||
}
|
||||
|
||||
func (tx *Tx) MakeUpdateSql(table string, data any, conditions string, args ...any) (string, []any) {
|
||||
@ -258,7 +321,7 @@ func (tx *Tx) MakeUpdateSql(table string, data any, conditions string, args ...a
|
||||
if ts.VersionField != "" {
|
||||
nextVer = tx.db.NextVersion(table)
|
||||
}
|
||||
return makeUpdateSql(tx.QuoteTag, table, data, conditions, ts.VersionField, nextVer, args...)
|
||||
return makeUpdateSql(tx.QuoteTag, table, data, conditions, ts.VersionField, nextVer, ts, args...)
|
||||
}
|
||||
|
||||
func MakeKeysVarsValues(data any) ([]string, []string, []any) {
|
||||
|
||||
39
CHANGELOG.md
39
CHANGELOG.md
@ -1,5 +1,44 @@
|
||||
# 变更记录 - @go/db
|
||||
|
||||
## [1.0.12] - 2026-05-17
|
||||
- **SQLite 极致优化 (超高并发支持)**:
|
||||
- **读写分离与零锁读取**: 读操作 (`Query`) 实现零锁定,配合 WAL 模式彻底解决读写互斥问题;写操作由应用层 `sync.Mutex` 统一排队,规避 `database is locked` 错误。
|
||||
- **临界区最小化**: 将 FTS 重写、参数 JSON 化 (`flatArgs`) 及日志记录移出锁保护区,极大缩短了写锁持有时间。
|
||||
- **工业级默认配置**:
|
||||
- 自动启用 `WAL` 模式、`NORMAL` 同步、`MEMORY` 临时存储及 `busy_timeout(5000)`。
|
||||
- 引入 **动态 Mmap**:根据系统内存自动设置 `mmap_size` (最大 30GB 或物理内存的 1/4),使大数据量访问接近内存速度。
|
||||
- 默认 `MaxOpenConns` 提升至 100,优化多线程只读性能。
|
||||
- **稳定性**:
|
||||
- 为 `Stmt` (预处理语句) 增加写锁保护。
|
||||
- 优化事务锁机制,支持事务内的锁自动追踪与释放。
|
||||
|
||||
## [1.0.11] - 2026-05-13
|
||||
- **基础设施对齐**:
|
||||
- 移除 `encoding/json` 原生依赖,全面切换至 `apigo.cc/go/cast.UnmarshalJSON` 以增强类型兼容性。
|
||||
- 移除测试代码中对 `os.Remove` 的直接调用,统一切换至 `apigo.cc/go/file.Remove` 以支持隔离文件系统。
|
||||
- **稳定性增强**:
|
||||
- 验证并更新了测试用例,确保在 Go 1.25.0 环境下的执行稳定性。
|
||||
- 更新 `TEST.md` 性能基准,反映基础设施对齐后的最新指标。
|
||||
|
||||
## [1.0.10] - 2026-05-10
|
||||
- **功能增强**:
|
||||
- 全面支持“复杂标识符”:改进了 `LIKE` 拦截逻辑中的正则表达式,支持带引号(`` ` ``, `"`, `'`, `[]`)和特殊字符(如 `-`)的表名与字段名。
|
||||
- 优化 `cleanIdentifier`:能够更精准地剥离多段式标识符(如 `table.column`)中的包装引号。
|
||||
- 增强 `getFTSMatchSQLParts` 和 `extractTableName`:确保在各种引用风格下均能正确定位影子列和源表。
|
||||
- **基础设施对齐**:
|
||||
- 升级 `apigo.cc/go/log` 至 `v1.3.2`。
|
||||
- **测试增强**:
|
||||
- 新增 `TestComplexIdentifierFTS` 验证复杂标识符下的全文检索重定向。
|
||||
- 修复并增强 `TestAutonomousFTS` 以支持多种引用风格的兼容性测试。
|
||||
|
||||
## [1.3.0] - 2026-05-12
|
||||
- **基础设施对齐**:
|
||||
- 官方发布 v1.3.0 对齐版本。
|
||||
|
||||
## [1.0.11] - 2026-05-11
|
||||
- **基础设施对齐**:
|
||||
- 最终基础设施对齐。
|
||||
|
||||
## [1.0.10] - 2026-05-10
|
||||
- **基础设施对齐**:
|
||||
- 升级 `apigo.cc/go/redis` 至 `v1.0.8`。
|
||||
|
||||
439
DB.go
439
DB.go
@ -19,7 +19,6 @@ import (
|
||||
"apigo.cc/go/crypto"
|
||||
"apigo.cc/go/id"
|
||||
"apigo.cc/go/log"
|
||||
"apigo.cc/go/rand"
|
||||
"apigo.cc/go/redis"
|
||||
"apigo.cc/go/safe"
|
||||
)
|
||||
@ -159,11 +158,11 @@ func (dbInfo *Config) ConfigureBy(setting string) {
|
||||
}
|
||||
}
|
||||
if sslCa != "" && sslCert != "" && sslKey != "" {
|
||||
sslName := id.MakeID(12)
|
||||
sslName := id.Get12BytesUltraPerSecond()
|
||||
dbInfo.SSL = sslName
|
||||
decryptedCa, _ := confAes.DecryptBytes([]byte(sslCa))
|
||||
decryptedCert, _ := confAes.DecryptBytes([]byte(sslCert))
|
||||
decryptedKey, _ := confAes.DecryptBytes([]byte(sslKey))
|
||||
decryptedCa, _ := confAES.DecryptBytes([]byte(sslCa))
|
||||
decryptedCert, _ := confAES.DecryptBytes([]byte(sslCert))
|
||||
decryptedKey, _ := confAES.DecryptBytes([]byte(sslKey))
|
||||
tlsConf := BuildTLSConfig(decryptedCa, decryptedCert, decryptedKey, sslSkipVerify)
|
||||
if tlsConf != nil {
|
||||
dbInfo.tls = tlsConf
|
||||
@ -194,6 +193,7 @@ type DB struct {
|
||||
QuoteTag string
|
||||
tables map[string]*TableStruct
|
||||
tablesLock *sync.RWMutex
|
||||
sqliteMu *sync.Mutex // Serial lock for SQLite writers
|
||||
}
|
||||
|
||||
type TableStruct struct {
|
||||
@ -219,16 +219,19 @@ type TableField struct {
|
||||
Extra string
|
||||
Desc string
|
||||
IsVersion bool
|
||||
IsObject bool
|
||||
}
|
||||
|
||||
var confAes, _ = crypto.NewAESCBCAndEraseKey([]byte("?GQ$0K0GgLdO=f+~L68PLm$uhKr4'=tV"), []byte("VFs7@sK61cj^f?HZ"))
|
||||
var keysSetted = sync.Once{}
|
||||
var confAES *crypto.Symmetric
|
||||
|
||||
func init() {
|
||||
crypto.OnSetDefaultAES(func(aes *crypto.Symmetric) {
|
||||
confAES = aes
|
||||
})
|
||||
}
|
||||
|
||||
func SetEncryptKeys(key, iv []byte) {
|
||||
keysSetted.Do(func() {
|
||||
confAes.Close()
|
||||
confAes, _ = crypto.NewAESGCMAndEraseKey(key, iv)
|
||||
})
|
||||
crypto.SetDefaultAES(key, iv)
|
||||
}
|
||||
|
||||
type dbLogger struct {
|
||||
@ -280,22 +283,16 @@ func (db *DB) NextVersion(table string) int64 {
|
||||
return atomic.AddInt64(v.(*int64), 1)
|
||||
}
|
||||
|
||||
type idMaker interface {
|
||||
Get(size int) string
|
||||
GetForMysql(size int) string
|
||||
GetForPostgreSQL(size int) string
|
||||
}
|
||||
|
||||
func (db *DB) NextID(table string) string {
|
||||
ts := db.getTable(table)
|
||||
if ts.IdField == "" || ts.IdSize == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
var maker idMaker
|
||||
var maker *id.IDMaker
|
||||
if db.Config.Redis != "" {
|
||||
if v, ok := globalIdMakers.Load(db.Config.Redis); ok {
|
||||
maker = v.(idMaker)
|
||||
maker = v.(*id.IDMaker)
|
||||
} else {
|
||||
r := redis.GetRedis(db.Config.Redis, db.logger.logger)
|
||||
if r != nil {
|
||||
@ -321,7 +318,7 @@ func (db *DB) NextID(table string) string {
|
||||
|
||||
func (db *DB) syncVersionFromDB(table, versionField string) {
|
||||
query := fmt.Sprintf("SELECT MAX(%s) FROM %s", db.Quote(versionField), db.Quote(table))
|
||||
maxVer := db.Query(query).IntOnR1C1()
|
||||
maxVer := db.rawQuery(query).IntOnR1C1()
|
||||
|
||||
if db.Config.Redis != "" {
|
||||
r := redis.GetRedis(db.Config.Redis, db.logger.logger)
|
||||
@ -352,6 +349,69 @@ func GetDB(name string, logger *log.Logger) *DB {
|
||||
return getDB(name, logger, true)
|
||||
}
|
||||
|
||||
// Sync 同步数据库结构 (使用默认实例 "default")
|
||||
func Sync(desc string) error {
|
||||
d := GetDB("default", nil)
|
||||
if d == nil {
|
||||
return errors.New("default db not configured")
|
||||
}
|
||||
return d.Sync(desc)
|
||||
}
|
||||
|
||||
// Insert 插入数据 (使用默认实例 "default")
|
||||
func Insert(table string, data any) *ExecResult {
|
||||
d := GetDB("default", nil)
|
||||
if d == nil {
|
||||
return &ExecResult{Error: errors.New("default db not configured")}
|
||||
}
|
||||
return d.Insert(table, data)
|
||||
}
|
||||
|
||||
// Update 更新数据 (使用默认实例 "default")
|
||||
func Update(table string, data any, conditions string, args ...any) *ExecResult {
|
||||
d := GetDB("default", nil)
|
||||
if d == nil {
|
||||
return &ExecResult{Error: errors.New("default db not configured")}
|
||||
}
|
||||
return d.Update(table, data, conditions, args...)
|
||||
}
|
||||
|
||||
// Delete 删除数据 (使用默认实例 "default")
|
||||
func Delete(table string, conditions string, args ...any) *ExecResult {
|
||||
d := GetDB("default", nil)
|
||||
if d == nil {
|
||||
return &ExecResult{Error: errors.New("default db not configured")}
|
||||
}
|
||||
return d.Delete(table, conditions, args...)
|
||||
}
|
||||
|
||||
// Query 查询数据 (使用默认实例 "default")
|
||||
func Query(query string, args ...any) *QueryResult {
|
||||
d := GetDB("default", nil)
|
||||
if d == nil {
|
||||
return &QueryResult{Error: errors.New("default db not configured")}
|
||||
}
|
||||
return d.Query(query, args...)
|
||||
}
|
||||
|
||||
// Exec 执行 SQL (使用默认实例 "default")
|
||||
func Exec(query string, args ...any) *ExecResult {
|
||||
d := GetDB("default", nil)
|
||||
if d == nil {
|
||||
return &ExecResult{Error: errors.New("default db not configured")}
|
||||
}
|
||||
return d.Exec(query, args...)
|
||||
}
|
||||
|
||||
// Begin 开始事务 (使用默认实例 "default")
|
||||
func Begin() *Tx {
|
||||
d := GetDB("default", nil)
|
||||
if d == nil {
|
||||
return &Tx{Error: errors.New("default db not configured")}
|
||||
}
|
||||
return d.Begin()
|
||||
}
|
||||
|
||||
func getDB(name string, logger *log.Logger, useCache bool) *DB {
|
||||
if logger == nil {
|
||||
logger = log.DefaultLogger
|
||||
@ -451,7 +511,7 @@ func getDB(name string, logger *log.Logger, useCache bool) *DB {
|
||||
|
||||
if conf.Password != "" {
|
||||
if encryptedPassword, err := base64.URLEncoding.DecodeString(conf.Password); err == nil {
|
||||
if pwdSafeBuf, err := confAes.Decrypt(encryptedPassword); err == nil {
|
||||
if pwdSafeBuf, err := confAES.Decrypt(encryptedPassword); err == nil {
|
||||
conf.pwd = pwdSafeBuf
|
||||
}
|
||||
}
|
||||
@ -477,6 +537,9 @@ func getDB(name string, logger *log.Logger, useCache bool) *DB {
|
||||
db.conn = conn
|
||||
db.tables = make(map[string]*TableStruct)
|
||||
db.tablesLock = new(sync.RWMutex)
|
||||
if conf.Type == "sqlite" || conf.Type == "sqlite3" {
|
||||
db.sqliteMu = new(sync.Mutex)
|
||||
}
|
||||
|
||||
if conf.ReadonlyHosts != nil {
|
||||
readonlyConnections := make([]*sql.DB, 0)
|
||||
@ -495,6 +558,9 @@ func getDB(name string, logger *log.Logger, useCache bool) *DB {
|
||||
|
||||
db.Error = nil
|
||||
db.Config = conf
|
||||
if (conf.Type == "sqlite" || conf.Type == "sqlite3") && conf.MaxOpens == 0 {
|
||||
conf.MaxOpens = 100
|
||||
}
|
||||
if conf.MaxIdles > 0 {
|
||||
conn.SetMaxIdleConns(conf.MaxIdles)
|
||||
}
|
||||
@ -507,6 +573,25 @@ func getDB(name string, logger *log.Logger, useCache bool) *DB {
|
||||
if conf.LogSlow == 0 {
|
||||
conf.LogSlow = config.Duration(1000 * time.Millisecond)
|
||||
}
|
||||
|
||||
if conf.Type == "sqlite" || conf.Type == "sqlite3" {
|
||||
baseExecRaw(conn, nil, "PRAGMA journal_mode=WAL")
|
||||
baseExecRaw(conn, nil, "PRAGMA synchronous=NORMAL")
|
||||
baseExecRaw(conn, nil, "PRAGMA busy_timeout=5000")
|
||||
baseExecRaw(conn, nil, "PRAGMA temp_store=MEMORY")
|
||||
baseExecRaw(conn, nil, "PRAGMA cache_size=-2000")
|
||||
|
||||
// Dynamic mmap_size: 1/4 of system memory, max 30GB
|
||||
mmapLimit := int64(30000000000)
|
||||
sysMemStr := runShell("sysctl -n hw.memsize || free -b | awk '/Mem:/ {print $2}'")
|
||||
if sysMem := cast.Int64(sysMemStr); sysMem > 0 {
|
||||
if mmapLimit > sysMem/4 {
|
||||
mmapLimit = sysMem / 4
|
||||
}
|
||||
}
|
||||
baseExecRaw(conn, nil, fmt.Sprintf("PRAGMA mmap_size=%d", mmapLimit))
|
||||
}
|
||||
|
||||
if useCache {
|
||||
dbInstancesLock.Lock()
|
||||
dbInstances[name] = db
|
||||
@ -531,6 +616,13 @@ func getPoolForHost(conf *Config, host string) (*sql.DB, error) {
|
||||
if connector := connectors[conf.Type]; connector != nil {
|
||||
return sql.OpenDB(connector(conf, conf.pwd, conf.tls)), nil
|
||||
} else {
|
||||
if (conf.Type == "sqlite" || conf.Type == "sqlite3") && !strings.Contains(conf.Args, "journal_mode") {
|
||||
if conf.Args != "" {
|
||||
conf.Args += "&"
|
||||
}
|
||||
conf.Args += "_journal_mode=WAL&_busy_timeout=5000&_pragma=synchronous(1)&_pragma=cache_size(-2000)"
|
||||
}
|
||||
|
||||
dsn := ""
|
||||
args := make([]string, 0)
|
||||
if conf.SSL != "" {
|
||||
@ -564,6 +656,7 @@ func (db *DB) CopyByLogger(logger *log.Logger) *DB {
|
||||
newDB.Config = db.Config
|
||||
newDB.tables = db.tables
|
||||
newDB.tablesLock = db.tablesLock
|
||||
newDB.sqliteMu = db.sqliteMu
|
||||
if logger == nil {
|
||||
logger = log.DefaultLogger
|
||||
}
|
||||
@ -600,6 +693,7 @@ func (db *DB) GetOriginDB() *sql.DB {
|
||||
func (db *DB) Prepare(query string) *Stmt {
|
||||
stmt := basePrepare(db.conn, nil, query)
|
||||
stmt.logger = db.logger
|
||||
stmt.sqliteMu = db.sqliteMu
|
||||
if stmt.Error != nil {
|
||||
db.logger.LogError(stmt.Error.Error())
|
||||
}
|
||||
@ -616,18 +710,22 @@ func (db *DB) Quotes(texts []string) string {
|
||||
|
||||
func (db *DB) Begin() *Tx {
|
||||
if db.conn == nil {
|
||||
return &Tx{db: db, QuoteTag: db.QuoteTag, logSlow: db.Config.LogSlow.TimeDuration(), Error: errors.New("operate on a bad connection"), logger: db.logger}
|
||||
return &Tx{db: db, sqliteMu: db.sqliteMu, QuoteTag: db.QuoteTag, logSlow: db.Config.LogSlow.TimeDuration(), Error: errors.New("operate on a bad connection"), logger: db.logger}
|
||||
}
|
||||
sqlTx, err := db.conn.Begin()
|
||||
if err != nil {
|
||||
db.logger.LogError(err.Error())
|
||||
return &Tx{db: db, QuoteTag: db.QuoteTag, logSlow: db.Config.LogSlow.TimeDuration(), Error: err, logger: db.logger}
|
||||
return &Tx{db: db, sqliteMu: db.sqliteMu, QuoteTag: db.QuoteTag, logSlow: db.Config.LogSlow.TimeDuration(), Error: err, logger: db.logger}
|
||||
}
|
||||
return &Tx{db: db, QuoteTag: db.QuoteTag, logSlow: db.Config.LogSlow.TimeDuration(), conn: sqlTx, logger: db.logger}
|
||||
return &Tx{db: db, sqliteMu: db.sqliteMu, QuoteTag: db.QuoteTag, logSlow: db.Config.LogSlow.TimeDuration(), conn: sqlTx, logger: db.logger}
|
||||
}
|
||||
|
||||
func (db *DB) Exec(query string, args ...any) *ExecResult {
|
||||
r := baseExec(db.conn, nil, query, args...)
|
||||
query, args = db.rewriteFTS(query, args)
|
||||
args = flatArgs(args)
|
||||
db.lock()
|
||||
r := baseExecRaw(db.conn, nil, query, args...)
|
||||
db.unlock()
|
||||
r.logger = db.logger
|
||||
if r.Error != nil {
|
||||
db.logger.LogQueryError(r.Error.Error(), query, args, r.usedTime)
|
||||
@ -639,19 +737,14 @@ func (db *DB) Exec(query string, args ...any) *ExecResult {
|
||||
return r
|
||||
}
|
||||
|
||||
func (db *DB) Query(query string, args ...any) *QueryResult {
|
||||
conn := db.conn
|
||||
if db.readonlyConnections != nil {
|
||||
connNum := len(db.readonlyConnections)
|
||||
if connNum == 1 {
|
||||
conn = db.readonlyConnections[0]
|
||||
} else {
|
||||
p := rand.Int(0, connNum-1)
|
||||
conn = db.readonlyConnections[p]
|
||||
}
|
||||
}
|
||||
func (db *DB) rawExec(query string, args ...any) *ExecResult {
|
||||
return db.Exec(query, args...)
|
||||
}
|
||||
|
||||
r := baseQuery(conn, nil, query, args...)
|
||||
func (db *DB) Query(query string, args ...any) *QueryResult {
|
||||
query, args = db.rewriteFTS(query, args)
|
||||
args = flatArgs(args)
|
||||
r := baseQueryRaw(db.conn, nil, query, args...)
|
||||
r.logger = db.logger
|
||||
if r.Error != nil {
|
||||
db.logger.LogQueryError(r.Error.Error(), query, args, r.usedTime)
|
||||
@ -663,46 +756,227 @@ func (db *DB) Query(query string, args ...any) *QueryResult {
|
||||
return r
|
||||
}
|
||||
|
||||
func (db *DB) rawQuery(query string, args ...any) *QueryResult {
|
||||
return db.Query(query, args...)
|
||||
}
|
||||
|
||||
|
||||
func (db *DB) lock() {
|
||||
if db.sqliteMu != nil {
|
||||
db.sqliteMu.Lock()
|
||||
}
|
||||
}
|
||||
|
||||
func (db *DB) unlock() {
|
||||
if db.sqliteMu != nil {
|
||||
db.sqliteMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
var identifierRegex = `(?:['"` + "`" + `][^'"` + "`" + `]+['"` + "`" + `]|[\w\-]+)`
|
||||
var likeFieldReg = regexp.MustCompile(`(?i)(` + identifierRegex + `(?:\.` + identifierRegex + `)*)\s+LIKE\s*$`)
|
||||
var likeLiteralReg = regexp.MustCompile(`(?i)(` + identifierRegex + `(?:\.` + identifierRegex + `)*)\s+LIKE\s+(['"])(%?[^'"]*?%?)(['"])`)
|
||||
|
||||
func cleanIdentifier(s string) string {
|
||||
parts := strings.Split(s, ".")
|
||||
for i, p := range parts {
|
||||
p = strings.TrimSpace(p)
|
||||
if len(p) >= 2 {
|
||||
if (p[0] == '`' && p[len(p)-1] == '`') ||
|
||||
(p[0] == '"' && p[len(p)-1] == '"') ||
|
||||
(p[0] == '\'' && p[len(p)-1] == '\'') ||
|
||||
(p[0] == '[' && p[len(p)-1] == ']') {
|
||||
parts[i] = p[1 : len(p)-1]
|
||||
continue
|
||||
}
|
||||
}
|
||||
parts[i] = p
|
||||
}
|
||||
return strings.Join(parts, ".")
|
||||
}
|
||||
|
||||
func (db *DB) rewriteFTS(query string, args []any) (string, []any) {
|
||||
// 1. 处理硬编码的 LIKE 'literal'
|
||||
query = likeLiteralReg.ReplaceAllStringFunc(query, func(m string) string {
|
||||
matches := likeLiteralReg.FindStringSubmatch(m)
|
||||
if matches[2] != matches[4] {
|
||||
return m // 引号不匹配,跳过
|
||||
}
|
||||
field := matches[1]
|
||||
quoteMark := matches[2]
|
||||
literal := matches[3]
|
||||
|
||||
cleanField := cleanIdentifier(field)
|
||||
tableName := db.extractTableName(query, field)
|
||||
if tableName != "" {
|
||||
ts := db.getTable(tableName)
|
||||
colParts := strings.Split(cleanField, ".")
|
||||
colName := colParts[len(colParts)-1]
|
||||
tokensCol := colName + "_tokens"
|
||||
|
||||
hasTokens := false
|
||||
for _, c := range ts.Columns {
|
||||
if c == tokensCol {
|
||||
hasTokens = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if hasTokens {
|
||||
searchTerm := strings.Trim(literal, "% ")
|
||||
tokens := BigramTokenize(searchTerm)
|
||||
if db.Config.Type == "pg" || db.Config.Type == "pgsql" || db.Config.Type == "postgres" {
|
||||
tokens = strings.ReplaceAll(tokens, " ", " & ")
|
||||
}
|
||||
pre, suf := db.getFTSMatchSQLParts(query, field)
|
||||
return pre + quoteMark + tokens + quoteMark + suf
|
||||
}
|
||||
}
|
||||
return m
|
||||
})
|
||||
|
||||
if len(args) == 0 || !strings.Contains(strings.ToUpper(query), " LIKE ") {
|
||||
return query, args
|
||||
}
|
||||
|
||||
parts := strings.Split(query, "?")
|
||||
if len(parts)-1 != len(args) {
|
||||
// 存在误伤风险,安全降级
|
||||
return query, args
|
||||
}
|
||||
|
||||
newArgs := make([]any, len(args))
|
||||
copy(newArgs, args)
|
||||
|
||||
isModified := false
|
||||
for i := 0; i < len(args); i++ {
|
||||
match := likeFieldReg.FindStringSubmatch(parts[i])
|
||||
if len(match) > 1 {
|
||||
field := match[1]
|
||||
cleanField := cleanIdentifier(field)
|
||||
tableName := db.extractTableName(query, field)
|
||||
if tableName != "" {
|
||||
ts := db.getTable(tableName)
|
||||
colParts := strings.Split(cleanField, ".")
|
||||
colName := colParts[len(colParts)-1]
|
||||
tokensCol := colName + "_tokens"
|
||||
|
||||
hasTokens := false
|
||||
for _, c := range ts.Columns {
|
||||
if c == tokensCol {
|
||||
hasTokens = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if hasTokens {
|
||||
// 命中影子列,执行替换
|
||||
ftsPre, ftsSuf := db.getFTSMatchSQLParts(query, field)
|
||||
parts[i] = strings.Replace(parts[i], match[0], ftsPre, 1)
|
||||
parts[i+1] = ftsSuf + parts[i+1]
|
||||
|
||||
// 处理参数
|
||||
searchTerm := cast.String(args[i])
|
||||
searchTerm = strings.Trim(searchTerm, "% ")
|
||||
tokens := BigramTokenize(searchTerm)
|
||||
|
||||
if db.Config.Type == "pg" || db.Config.Type == "pgsql" || db.Config.Type == "postgres" {
|
||||
tokens = strings.ReplaceAll(tokens, " ", " & ")
|
||||
}
|
||||
newArgs[i] = tokens
|
||||
isModified = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if isModified {
|
||||
return strings.Join(parts, "?"), newArgs
|
||||
}
|
||||
|
||||
return query, args
|
||||
}
|
||||
|
||||
func (db *DB) getFTSMatchSQLParts(query string, field string) (string, string) {
|
||||
cleanField := cleanIdentifier(field)
|
||||
parts := strings.Split(cleanField, ".")
|
||||
colName := parts[len(parts)-1]
|
||||
|
||||
// 保持原字段引用方式(带引号或别名)
|
||||
tokensField := field + "_tokens"
|
||||
lastPart := field
|
||||
prefix := ""
|
||||
if idx := strings.LastIndex(field, "."); idx != -1 {
|
||||
prefix = field[:idx+1]
|
||||
lastPart = field[idx+1:]
|
||||
}
|
||||
|
||||
if len(lastPart) >= 2 && ((lastPart[0] == '`' && lastPart[len(lastPart)-1] == '`') ||
|
||||
(lastPart[0] == '"' && lastPart[len(lastPart)-1] == '"') ||
|
||||
(lastPart[0] == '[' && lastPart[len(lastPart)-1] == ']')) {
|
||||
tokensField = prefix + lastPart[:len(lastPart)-1] + "_tokens" + lastPart[len(lastPart)-1:]
|
||||
}
|
||||
|
||||
switch db.Config.Type {
|
||||
case "mysql":
|
||||
return fmt.Sprintf("MATCH(%s) AGAINST(", tokensField), " IN BOOLEAN MODE)"
|
||||
case "pg", "pgsql", "postgres":
|
||||
return fmt.Sprintf("%s @@ to_tsquery('simple', ", tokensField), ")"
|
||||
case "sqlite", "sqlite3":
|
||||
tableName := db.extractTableName(query, field)
|
||||
idField := "id"
|
||||
ts := db.getTable(tableName)
|
||||
if ts.IdField != "" {
|
||||
idField = ts.IdField
|
||||
}
|
||||
prefix := ""
|
||||
dotParts := strings.Split(field, ".")
|
||||
if len(dotParts) > 1 {
|
||||
prefix = dotParts[0] + "."
|
||||
}
|
||||
return fmt.Sprintf("%s%s IN (SELECT rowid FROM \"%s_fts\" WHERE \"%s_tokens\" MATCH ", prefix, idField, tableName, colName), ")"
|
||||
default:
|
||||
return fmt.Sprintf("%s LIKE ", field), ""
|
||||
}
|
||||
}
|
||||
|
||||
func (db *DB) extractTableName(query string, field string) string {
|
||||
cleanField := cleanIdentifier(field)
|
||||
parts := strings.Split(cleanField, ".")
|
||||
if len(parts) > 1 {
|
||||
alias := parts[0]
|
||||
reg := regexp.MustCompile(fmt.Sprintf(`(?i)FROM\s+(%s)\s+(?:AS\s+)?["\` + "`" + `]?%s["\` + "`" + `]?|JOIN\s+(%s)\s+(?:AS\s+)?["\` + "`" + `]?%s["\` + "`" + `]?`, identifierRegex, alias, identifierRegex, alias))
|
||||
match := reg.FindStringSubmatch(query)
|
||||
if len(match) > 1 {
|
||||
if match[1] != "" {
|
||||
return cleanIdentifier(match[1])
|
||||
}
|
||||
return cleanIdentifier(match[2])
|
||||
}
|
||||
return alias
|
||||
}
|
||||
reg := regexp.MustCompile(`(?i)FROM\s+(` + identifierRegex + `)`)
|
||||
match := reg.FindStringSubmatch(query)
|
||||
if len(match) > 1 {
|
||||
return cleanIdentifier(match[1])
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (db *DB) Insert(table string, data any) *ExecResult {
|
||||
query, values := db.MakeInsertSql(table, data, false)
|
||||
r := baseExec(db.conn, nil, query, values...)
|
||||
r.logger = db.logger
|
||||
if r.Error != nil {
|
||||
db.logger.LogQueryError(r.Error.Error(), query, values, r.usedTime)
|
||||
} else {
|
||||
if db.Config.LogSlow > 0 && r.usedTime >= float32(db.Config.LogSlow.TimeDuration()/time.Millisecond) {
|
||||
db.logger.LogQuery(query, values, r.usedTime)
|
||||
}
|
||||
}
|
||||
return r
|
||||
return db.Exec(query, values...)
|
||||
}
|
||||
|
||||
func (db *DB) Replace(table string, data any) *ExecResult {
|
||||
query, values := db.MakeInsertSql(table, data, true)
|
||||
r := baseExec(db.conn, nil, query, values...)
|
||||
r.logger = db.logger
|
||||
if r.Error != nil {
|
||||
db.logger.LogQueryError(r.Error.Error(), query, values, r.usedTime)
|
||||
} else {
|
||||
if db.Config.LogSlow > 0 && r.usedTime >= float32(db.Config.LogSlow.TimeDuration()/time.Millisecond) {
|
||||
db.logger.LogQuery(query, values, r.usedTime)
|
||||
}
|
||||
}
|
||||
return r
|
||||
return db.Exec(query, values...)
|
||||
}
|
||||
|
||||
func (db *DB) Update(table string, data any, conditions string, args ...any) *ExecResult {
|
||||
query, values := db.MakeUpdateSql(table, data, conditions, args...)
|
||||
r := baseExec(db.conn, nil, query, values...)
|
||||
r.logger = db.logger
|
||||
if r.Error != nil {
|
||||
db.logger.LogQueryError(r.Error.Error(), query, values, r.usedTime)
|
||||
} else {
|
||||
if db.Config.LogSlow > 0 && r.usedTime >= float32(db.Config.LogSlow.TimeDuration()/time.Millisecond) {
|
||||
db.logger.LogQuery(query, values, r.usedTime)
|
||||
}
|
||||
}
|
||||
return r
|
||||
return db.Exec(query, values...)
|
||||
}
|
||||
|
||||
func (db *DB) Delete(table string, conditions string, args ...any) *ExecResult {
|
||||
@ -712,16 +986,7 @@ func (db *DB) Delete(table string, conditions string, args ...any) *ExecResult {
|
||||
conditions = " where " + conditions
|
||||
}
|
||||
query := fmt.Sprintf("delete from %s%s", db.Quote(table), conditions)
|
||||
r := baseExec(db.conn, nil, query, args...)
|
||||
r.logger = db.logger
|
||||
if r.Error != nil {
|
||||
db.logger.LogQueryError(r.Error.Error(), query, args, r.usedTime)
|
||||
} else {
|
||||
if db.Config.LogSlow > 0 && r.usedTime >= float32(db.Config.LogSlow.TimeDuration()/time.Millisecond) {
|
||||
db.logger.LogQuery(query, args, r.usedTime)
|
||||
}
|
||||
}
|
||||
return r
|
||||
return db.Exec(query, args...)
|
||||
}
|
||||
|
||||
// Shadow delete
|
||||
@ -755,7 +1020,7 @@ func (db *DB) getTable(table string) *TableStruct {
|
||||
var query string
|
||||
if db.Config.Type == "mysql" {
|
||||
query = "SELECT COLUMN_NAME, DATA_TYPE, CHARACTER_MAXIMUM_LENGTH, COLUMN_KEY FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?"
|
||||
res := db.Query(query, db.Config.DB, table)
|
||||
res := db.rawQuery(query, db.Config.DB, table)
|
||||
rows := res.MapResults()
|
||||
for _, row := range rows {
|
||||
col := cast.String(row["COLUMN_NAME"])
|
||||
@ -767,14 +1032,14 @@ func (db *DB) getTable(table string) *TableStruct {
|
||||
if col == "autoVersion" {
|
||||
ts.VersionField = "autoVersion"
|
||||
}
|
||||
if (colKey == "PRI" || colKey == "UNI") && strings.ToLower(dataType) == "char" && (charLen == 8 || charLen == 10 || charLen == 12 || charLen == 14) {
|
||||
if (colKey == "PRI" || colKey == "UNI") && strings.ToLower(dataType) == "char" && (charLen >= 8 && charLen <= 16) {
|
||||
ts.IdField = col
|
||||
ts.IdSize = charLen
|
||||
}
|
||||
}
|
||||
} else if db.Config.Type == "postgres" || db.Config.Type == "pgx" {
|
||||
query = "SELECT column_name, data_type, character_maximum_length FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = ?"
|
||||
res := db.Query(query, table)
|
||||
res := db.rawQuery(query, table)
|
||||
rows := res.MapResults()
|
||||
for _, row := range rows {
|
||||
col := cast.String(row["column_name"])
|
||||
@ -787,15 +1052,15 @@ func (db *DB) getTable(table string) *TableStruct {
|
||||
}
|
||||
// PostgreSQL PK/Unique check is complex, we use column name 'id' and char type as a heuristic or check constraints if needed.
|
||||
// To keep it simple and efficient as requested:
|
||||
if (col == "id" || col == "ID") && (strings.Contains(strings.ToLower(dataType), "char")) && (charLen == 8 || charLen == 10 || charLen == 12 || charLen == 14) {
|
||||
if (col == "id" || col == "ID") && (strings.Contains(strings.ToLower(dataType), "char")) && (charLen >= 8 && charLen <= 16) {
|
||||
ts.IdField = col
|
||||
ts.IdSize = charLen
|
||||
}
|
||||
}
|
||||
} else if isFileDB(db.Config.Type) {
|
||||
// For SQLite
|
||||
query = fmt.Sprintf("PRAGMA table_info(%s)", db.Quote(table))
|
||||
res := db.Query(query)
|
||||
query := fmt.Sprintf("PRAGMA table_info(%s)", db.Quote(table))
|
||||
res := db.rawQuery(query)
|
||||
rows := res.MapResults()
|
||||
for _, row := range rows {
|
||||
colName := cast.String(row["name"])
|
||||
@ -814,7 +1079,7 @@ func (db *DB) getTable(table string) *TableStruct {
|
||||
if charLen == 0 {
|
||||
fmt.Sscanf(colType, "CHARACTER(%d)", &charLen)
|
||||
}
|
||||
if charLen == 8 || charLen == 10 || charLen == 12 || charLen == 14 {
|
||||
if charLen >= 8 && charLen <= 16 {
|
||||
ts.IdField = colName
|
||||
ts.IdSize = charLen
|
||||
}
|
||||
@ -826,19 +1091,19 @@ func (db *DB) getTable(table string) *TableStruct {
|
||||
shadowTable := table + "_deleted"
|
||||
if db.Config.Type == "mysql" {
|
||||
query = "SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?"
|
||||
res := db.Query(query, db.Config.DB, shadowTable)
|
||||
res := db.rawQuery(query, db.Config.DB, shadowTable)
|
||||
if res.StringOnR1C1() != "" {
|
||||
ts.HasShadowTable = true
|
||||
}
|
||||
} else if db.Config.Type == "postgres" || db.Config.Type == "pgx" {
|
||||
query = "SELECT table_name FROM information_schema.tables WHERE table_schema = current_schema() AND table_name = ?"
|
||||
res := db.Query(query, shadowTable)
|
||||
res := db.rawQuery(query, shadowTable)
|
||||
if res.StringOnR1C1() != "" {
|
||||
ts.HasShadowTable = true
|
||||
}
|
||||
} else if isFileDB(db.Config.Type) {
|
||||
query = "SELECT name FROM sqlite_master WHERE type='table' AND name=?"
|
||||
res := db.Query(query, shadowTable)
|
||||
res := db.rawQuery(query, shadowTable)
|
||||
if res.StringOnR1C1() != "" {
|
||||
ts.HasShadowTable = true
|
||||
}
|
||||
|
||||
@ -10,6 +10,7 @@ import (
|
||||
|
||||
"apigo.cc/go/cast"
|
||||
"apigo.cc/go/db"
|
||||
"apigo.cc/go/file"
|
||||
"apigo.cc/go/shell"
|
||||
|
||||
_ "apigo.cc/go/db/mysql"
|
||||
@ -18,7 +19,7 @@ import (
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
code := m.Run()
|
||||
os.Remove("test.db")
|
||||
file.Remove("test.db")
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
|
||||
115
FTS_test.go
Normal file
115
FTS_test.go
Normal file
@ -0,0 +1,115 @@
|
||||
package db_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"apigo.cc/go/db"
|
||||
"apigo.cc/go/file"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
func TestAutonomousFTS(t *testing.T) {
|
||||
dbPath := "test_fts.db"
|
||||
dbInst := db.GetDB("sqlite://"+dbPath, nil)
|
||||
defer file.Remove(dbPath)
|
||||
defer dbInst.Exec("DROP TABLE IF EXISTS fts_test")
|
||||
defer dbInst.Exec("DROP TABLE IF EXISTS fts_test_fts")
|
||||
|
||||
schema := `== Default ==
|
||||
fts_test
|
||||
id AI
|
||||
title TI // Fulltext title
|
||||
content TI // Fulltext content
|
||||
status i
|
||||
`
|
||||
err := dbInst.Sync(schema)
|
||||
if err != nil {
|
||||
t.Fatal("Sync error:", err)
|
||||
}
|
||||
|
||||
// 1. Verify schema
|
||||
row := dbInst.Query("SELECT \"sql\" FROM \"sqlite_master\" WHERE \"type\"='table' AND \"name\"='fts_test'").MapOnR1()
|
||||
sqlStr := ""
|
||||
if row["sql"] != nil {
|
||||
sqlStr = row["sql"].(string)
|
||||
}
|
||||
if !strings.Contains(sqlStr, "title_tokens") || !strings.Contains(sqlStr, "content_tokens") {
|
||||
t.Fatalf("Shadow columns missing in main table: %s", sqlStr)
|
||||
}
|
||||
|
||||
row = dbInst.Query("SELECT \"name\" FROM \"sqlite_master\" WHERE \"type\"='table' AND \"name\"='fts_test_fts'").MapOnR1()
|
||||
if row["name"] == nil {
|
||||
t.Fatal("FTS virtual table missing")
|
||||
}
|
||||
|
||||
// 2. Test Insert
|
||||
dbInst.Insert("fts_test", map[string]any{
|
||||
"title": "你好世界",
|
||||
"content": "这是一段测试文本",
|
||||
"status": 1,
|
||||
})
|
||||
|
||||
// Check if tokens are populated in main table
|
||||
row = dbInst.Query("SELECT title_tokens, content_tokens FROM fts_test WHERE id=1").MapOnR1()
|
||||
if row["title_tokens"] == nil || row["title_tokens"] == "" {
|
||||
t.Fatal("Tokens not populated in main table")
|
||||
}
|
||||
|
||||
// Check if tokens are in FTS table
|
||||
row = dbInst.Query("SELECT * FROM fts_test_fts").MapOnR1()
|
||||
if row["title_tokens"] == nil || row["title_tokens"] == "" {
|
||||
t.Fatal("Tokens not populated in FTS table")
|
||||
}
|
||||
|
||||
// 3. Test Query Interception (LIKE -> FTS)
|
||||
// Searching for "世界" should match "你好世界"
|
||||
res := dbInst.Query("SELECT * FROM fts_test WHERE title LIKE ?", "%世界%")
|
||||
list := res.MapResults()
|
||||
if len(list) != 1 {
|
||||
t.Fatalf("Query failed to find match via FTS redirection, found %d", len(list))
|
||||
}
|
||||
|
||||
// 4. Test Update
|
||||
dbInst.Update("fts_test", map[string]any{"title": "更新后的标题"}, "id=?", 1)
|
||||
row = dbInst.Query("SELECT title_tokens FROM fts_test WHERE id=1").MapOnR1()
|
||||
if !strings.Contains(row["title_tokens"].(string), "更新") {
|
||||
t.Fatalf("Tokens not updated: %v", row["title_tokens"])
|
||||
}
|
||||
|
||||
// 5. Test Multiple Fields & Alias
|
||||
dbInst.Insert("fts_test", map[string]any{
|
||||
"title": "测试标题",
|
||||
"content": "北京大学是一个好学校",
|
||||
"status": 1,
|
||||
})
|
||||
|
||||
// Search in content using alias
|
||||
res = dbInst.Query("SELECT t.title FROM fts_test AS t WHERE t.content LIKE ?", "%北京大学%")
|
||||
list = res.MapResults()
|
||||
if len(list) != 1 {
|
||||
t.Fatalf("Alias query failed, found %d", len(list))
|
||||
}
|
||||
|
||||
// 6. Test Hardcoded Literals
|
||||
res = dbInst.Query("SELECT * FROM fts_test WHERE title LIKE '%标题%'")
|
||||
list = res.MapResults()
|
||||
if len(list) != 2 {
|
||||
t.Fatalf("Hardcoded literal query failed, found %d", len(list))
|
||||
}
|
||||
|
||||
// 7. Test Various Identifier Styles
|
||||
styles := []string{
|
||||
"SELECT * FROM fts_test WHERE `title` LIKE ?",
|
||||
"SELECT * FROM fts_test WHERE \"title\" LIKE ?",
|
||||
"SELECT * FROM fts_test WHERE 'title' LIKE ?",
|
||||
"SELECT * FROM fts_test WHERE `fts_test`.`title` LIKE ?",
|
||||
}
|
||||
for _, sql := range styles {
|
||||
res = dbInst.Query(sql, "%测试%")
|
||||
list = res.MapResults()
|
||||
if len(list) != 1 {
|
||||
t.Errorf("Style failed: %s, found %d", sql, len(list))
|
||||
}
|
||||
}
|
||||
}
|
||||
10
Log.go
10
Log.go
@ -6,11 +6,11 @@ import (
|
||||
)
|
||||
|
||||
type DBLog struct {
|
||||
DbType string `log:"pos:6,color:blue"`
|
||||
Dsn string `log:"pos:7,color:gray,withoutkey:true"`
|
||||
Query string `log:"pos:8,color:cyan"`
|
||||
QueryArgs string `log:"pos:9,color:gray"`
|
||||
UsedTime float32 `log:"pos:10,format:%.2fms"`
|
||||
DbType string `log:"pos:7,color:blue"`
|
||||
Dsn string `log:"pos:8,color:gray,withoutkey:true"`
|
||||
Query string `log:"pos:9,color:cyan"`
|
||||
QueryArgs string `log:"pos:10,color:gray"`
|
||||
UsedTime float32 `log:"pos:11,format:%.2fms"`
|
||||
}
|
||||
|
||||
func (l *DBLog) Reset() {
|
||||
|
||||
36
Result.go
36
Result.go
@ -2,7 +2,6 @@ package db
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
@ -415,7 +414,7 @@ func (r *QueryResult) makeResults(results any, rows *sql.Rows) error {
|
||||
if field.Type().AssignableTo(val.Type()) {
|
||||
field.Set(val.Addr())
|
||||
} else if val.Type().String() == "string" {
|
||||
strVal := fixValue(col.DatabaseTypeName(), val)
|
||||
strVal := fixValue(col.Name(), col.DatabaseTypeName(), val)
|
||||
field.Set(reflect.New(field.Type().Elem()))
|
||||
field.Elem().SetString(cast.String(strVal.Interface()))
|
||||
} else if strings.Contains(field.Type().String(), "uint") {
|
||||
@ -436,7 +435,7 @@ func (r *QueryResult) makeResults(results any, rows *sql.Rows) error {
|
||||
if s, ok := val.Interface().(string); ok {
|
||||
storedValue := new(any)
|
||||
if s != "" {
|
||||
_ = json.Unmarshal([]byte(s), storedValue)
|
||||
cast.UnmarshalJSON([]byte(s), storedValue)
|
||||
}
|
||||
cast.Convert(convertedObject.Interface(), storedValue)
|
||||
field.Set(convertedObject.Elem())
|
||||
@ -446,12 +445,12 @@ func (r *QueryResult) makeResults(results any, rows *sql.Rows) error {
|
||||
}
|
||||
} else if field.Type().AssignableTo(val.Type()) {
|
||||
if val.Kind() == reflect.String {
|
||||
field.Set(fixValue(col.DatabaseTypeName(), val))
|
||||
field.Set(fixValue(col.Name(), col.DatabaseTypeName(), val))
|
||||
} else {
|
||||
field.Set(val)
|
||||
}
|
||||
} else if val.Type().String() == "string" {
|
||||
field.Set(fixValue(col.DatabaseTypeName(), val))
|
||||
field.Set(fixValue(col.Name(), col.DatabaseTypeName(), val))
|
||||
} else if strings.Contains(val.Type().String(), "int") {
|
||||
field.SetInt(val.Int())
|
||||
} else if strings.Contains(val.Type().String(), "float") {
|
||||
@ -471,9 +470,9 @@ func (r *QueryResult) makeResults(results any, rows *sql.Rows) error {
|
||||
for colIndex, col := range colTypes {
|
||||
valuePtr := reflect.ValueOf(scanValues[colIndex]).Elem()
|
||||
if !valuePtr.IsNil() {
|
||||
data.SetMapIndex(reflect.ValueOf(col.Name()), fixValue(col.DatabaseTypeName(), valuePtr.Elem()))
|
||||
data.SetMapIndex(reflect.ValueOf(col.Name()), fixValue(col.Name(), col.DatabaseTypeName(), valuePtr.Elem()))
|
||||
} else {
|
||||
data.SetMapIndex(reflect.ValueOf(col.Name()), fixValue(col.DatabaseTypeName(), reflect.New(rowType.Elem()).Elem()))
|
||||
data.SetMapIndex(reflect.ValueOf(col.Name()), fixValue(col.Name(), col.DatabaseTypeName(), reflect.New(rowType.Elem()).Elem()))
|
||||
}
|
||||
}
|
||||
} else if rowType.Kind() == reflect.Slice {
|
||||
@ -481,15 +480,15 @@ func (r *QueryResult) makeResults(results any, rows *sql.Rows) error {
|
||||
for colIndex, col := range colTypes {
|
||||
valuePtr := reflect.ValueOf(scanValues[colIndex]).Elem()
|
||||
if !valuePtr.IsNil() {
|
||||
data.Index(colIndex).Set(fixValue(col.DatabaseTypeName(), valuePtr.Elem()))
|
||||
data.Index(colIndex).Set(fixValue(col.Name(), col.DatabaseTypeName(), valuePtr.Elem()))
|
||||
} else {
|
||||
data.Index(colIndex).Set(fixValue(col.DatabaseTypeName(), reflect.New(rowType.Elem()).Elem()))
|
||||
data.Index(colIndex).Set(fixValue(col.Name(), col.DatabaseTypeName(), reflect.New(rowType.Elem()).Elem()))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
valuePtr := reflect.ValueOf(scanValues[0]).Elem()
|
||||
if !valuePtr.IsNil() {
|
||||
data = fixValue(colTypes[0].DatabaseTypeName(), valuePtr.Elem())
|
||||
data = fixValue(colTypes[0].Name(), colTypes[0].DatabaseTypeName(), valuePtr.Elem())
|
||||
}
|
||||
}
|
||||
|
||||
@ -511,15 +510,15 @@ func (r *QueryResult) makeResults(results any, rows *sql.Rows) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func fixValue(colType string, v reflect.Value) reflect.Value {
|
||||
func fixValue(colName string, colType string, v reflect.Value) reflect.Value {
|
||||
if v.Kind() == reflect.String {
|
||||
str := v.String()
|
||||
switch colType {
|
||||
case "DATE":
|
||||
switch {
|
||||
case strings.Contains(colType, "DATE"):
|
||||
if len(str) >= 10 && str[4] == '-' && str[7] == '-' {
|
||||
return reflect.ValueOf(str[:10])
|
||||
}
|
||||
case "DATETIME":
|
||||
case strings.Contains(colType, "DATETIME"):
|
||||
if len(str) >= 19 && str[10] == 'T' && str[4] == '-' && str[7] == '-' && str[13] == ':' && str[16] == ':' {
|
||||
str = strings.TrimRight(str, "Z")
|
||||
if len(str) > 19 && str[19] == '.' {
|
||||
@ -527,13 +526,20 @@ func fixValue(colType string, v reflect.Value) reflect.Value {
|
||||
}
|
||||
return reflect.ValueOf(str[:10] + " " + str[11:19])
|
||||
}
|
||||
case "TIME":
|
||||
case strings.Contains(colType, "TIME"):
|
||||
if len(str) >= 8 && str[2] == ':' && str[4] == ':' {
|
||||
if len(str) >= 15 && str[8] == '.' {
|
||||
return reflect.ValueOf(str[0:15])
|
||||
}
|
||||
return reflect.ValueOf(str[0:8])
|
||||
}
|
||||
case strings.Contains(colType, "JSON"):
|
||||
if str != "" && (str[0] == '{' || str[0] == '[') {
|
||||
var out any
|
||||
if err := cast.UnmarshalJSON([]byte(str), &out); err == nil {
|
||||
return reflect.ValueOf(out)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return v
|
||||
|
||||
110
Schema.go
110
Schema.go
@ -119,6 +119,9 @@ func ParseField(line string) TableField {
|
||||
field.Type = "middleint unsigned"
|
||||
case "t":
|
||||
field.Type = "text"
|
||||
case "o":
|
||||
field.Type = "json"
|
||||
field.IsObject = true
|
||||
case "bb":
|
||||
field.Type = "blob"
|
||||
default:
|
||||
@ -199,7 +202,23 @@ func ParseSchema(desc string) []*SchemaGroup {
|
||||
if field.IsVersion {
|
||||
currentTable.VersionField = field.Name
|
||||
}
|
||||
currentTable.Fields = append(currentTable.Fields, field)
|
||||
if field.Index == "fulltext" {
|
||||
// 保持原字段,但移除其索引标记,由影子列承担索引
|
||||
field.Index = ""
|
||||
currentTable.Fields = append(currentTable.Fields, field)
|
||||
|
||||
// 隐式追加影子列
|
||||
tokensField := TableField{
|
||||
Name: field.Name + "_tokens",
|
||||
Type: "text",
|
||||
Null: "NULL",
|
||||
Index: "fulltext",
|
||||
Comment: "FTS tokens for " + field.Name,
|
||||
}
|
||||
currentTable.Fields = append(currentTable.Fields, tokensField)
|
||||
} else {
|
||||
currentTable.Fields = append(currentTable.Fields, field)
|
||||
}
|
||||
}
|
||||
}
|
||||
return groups
|
||||
@ -226,6 +245,9 @@ func (field *TableField) Parse(tableType string) {
|
||||
}
|
||||
} else if tableType == "pg" || tableType == "pgsql" || tableType == "postgres" {
|
||||
typ := field.Type
|
||||
if typ == "json" {
|
||||
typ = "jsonb"
|
||||
}
|
||||
if field.Extra == "AUTO_INCREMENT" {
|
||||
if strings.Contains(typ, "bigint") {
|
||||
typ = "bigserial"
|
||||
@ -272,7 +294,7 @@ func (db *DB) Sync(desc string) error {
|
||||
for _, group := range groups {
|
||||
for _, table := range group.Tables {
|
||||
db.tablesLock.Lock()
|
||||
db.tables[table.Name] = table
|
||||
delete(db.tables, table.Name)
|
||||
db.tablesLock.Unlock()
|
||||
|
||||
err := db.CheckTable(table)
|
||||
@ -285,6 +307,8 @@ func (db *DB) Sync(desc string) error {
|
||||
return outErr
|
||||
}
|
||||
|
||||
|
||||
|
||||
// CheckTable 检查并同步单个表结构
|
||||
func (db *DB) CheckTable(table *TableStruct) error {
|
||||
fieldSets := make([]string, 0)
|
||||
@ -292,6 +316,7 @@ func (db *DB) CheckTable(table *TableStruct) error {
|
||||
keySets := make([]string, 0)
|
||||
keySetBy := make(map[string]string)
|
||||
keySetFields := make(map[string]string)
|
||||
ftsFields := make([]string, 0)
|
||||
|
||||
isPostgres := db.Config.Type == "pg" || db.Config.Type == "pgsql" || db.Config.Type == "postgres"
|
||||
|
||||
@ -332,9 +357,19 @@ func (db *DB) CheckTable(table *TableStruct) error {
|
||||
keySetBy[keyName] = keySet
|
||||
}
|
||||
case "fulltext":
|
||||
if !strings.HasPrefix(db.Config.Type, "sqlite") && db.Config.Type != "chai" && !isPostgres {
|
||||
keyName := fmt.Sprint("tk_", table.Name, "_", field.Name)
|
||||
keySet := fmt.Sprintf("FULLTEXT KEY "+db.Quote("%s")+" ("+db.Quote("%s")+") COMMENT '%s'", keyName, field.Name, field.Comment)
|
||||
ftsFields = append(ftsFields, field.Name)
|
||||
keyName := fmt.Sprint("tk_", table.Name, "_", field.Name)
|
||||
keySet := ""
|
||||
if isPostgres {
|
||||
// 使用 simple 分词器,配合应用层的分词结果
|
||||
keySet = fmt.Sprintf("CREATE INDEX \"%s\" ON \"%s\" USING GIN (to_tsvector('simple', \"%s\"))", keyName, table.Name, field.Name)
|
||||
} else if !strings.HasPrefix(db.Config.Type, "sqlite") && db.Config.Type != "chai" {
|
||||
keySet = fmt.Sprintf("FULLTEXT KEY "+db.Quote("%s")+" ("+db.Quote("%s")+") COMMENT '%s'", keyName, field.Name, field.Comment)
|
||||
} else {
|
||||
// SQLite 使用 FTS5,这里不生成普通索引
|
||||
keySet = ""
|
||||
}
|
||||
if keySet != "" {
|
||||
keySets = append(keySets, keySet)
|
||||
keySetBy[keyName] = keySet
|
||||
}
|
||||
@ -391,34 +426,40 @@ func (db *DB) CheckTable(table *TableStruct) error {
|
||||
tmpFields := []struct {
|
||||
Name string
|
||||
Type string
|
||||
Notnull bool
|
||||
Notnull int
|
||||
Dflt_value any
|
||||
Pk bool
|
||||
Pk int
|
||||
}{}
|
||||
db.Query("PRAGMA table_info(" + db.Quote(table.Name) + ")").To(&tmpFields)
|
||||
if err := db.Query("PRAGMA table_info(" + db.Quote(table.Name) + ")").To(&tmpFields); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, f := range tmpFields {
|
||||
oldFieldList = append(oldFieldList, &tableFieldDesc{
|
||||
Field: f.Name,
|
||||
Type: f.Type,
|
||||
Null: cast.If(f.Notnull, "NO", "YES"),
|
||||
Key: cast.If(f.Pk, "PRI", ""),
|
||||
Null: cast.If(f.Notnull != 0, "NO", "YES"),
|
||||
Key: cast.If(f.Pk != 0, "PRI", ""),
|
||||
Default: cast.String(f.Dflt_value),
|
||||
})
|
||||
}
|
||||
tmpIndexes := []struct {
|
||||
Name string
|
||||
Unique bool
|
||||
Unique int
|
||||
Origin string
|
||||
Partial int
|
||||
}{}
|
||||
db.Query("PRAGMA index_list(" + db.Quote(table.Name) + ")").To(&tmpIndexes)
|
||||
if err := db.Query("PRAGMA index_list(" + db.Quote(table.Name) + ")").To(&tmpIndexes); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, i := range tmpIndexes {
|
||||
tmpIndexInfo := []struct {
|
||||
Name string
|
||||
Seqno int
|
||||
Cid int
|
||||
}{}
|
||||
db.Query("PRAGMA index_info(" + db.Quote(i.Name) + ")").To(&tmpIndexInfo)
|
||||
if err := db.Query("PRAGMA index_info(" + db.Quote(i.Name) + ")").To(&tmpIndexInfo); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(tmpIndexInfo) > 0 {
|
||||
oldIndexInfos = append(oldIndexInfos, &tableKeyDesc{
|
||||
Key_name: i.Name,
|
||||
@ -482,6 +523,9 @@ func (db *DB) CheckTable(table *TableStruct) error {
|
||||
for keyId := range oldIndexes {
|
||||
if keyId != "PRIMARY" && !isPostgres && strings.ToLower(keySetFields[keyId]) != strings.ToLower(oldIndexes[keyId]) {
|
||||
if strings.HasPrefix(db.Config.Type, "sqlite") {
|
||||
if strings.HasPrefix(keyId, "sqlite_autoindex_") {
|
||||
continue
|
||||
}
|
||||
actions = append(actions, "DROP INDEX "+db.Quote(keyId))
|
||||
} else {
|
||||
actions = append(actions, "DROP KEY "+db.Quote(keyId))
|
||||
@ -640,13 +684,49 @@ func (db *DB) CheckTable(table *TableStruct) error {
|
||||
}
|
||||
}
|
||||
|
||||
if res.Error != nil {
|
||||
if res != nil && res.Error != nil {
|
||||
_ = tx.Rollback()
|
||||
return res.Error
|
||||
}
|
||||
_ = tx.Commit()
|
||||
}
|
||||
}
|
||||
|
||||
if len(ftsFields) > 0 && strings.HasPrefix(db.Config.Type, "sqlite") {
|
||||
ftsTableName := table.Name + "_fts"
|
||||
ftsInfo := db.Query("SELECT \"name\" FROM \"sqlite_master\" WHERE \"type\"='table' AND \"name\"='" + ftsTableName + "'").MapOnR1()
|
||||
if ftsInfo["name"] == nil {
|
||||
// 创建 FTS 虚拟表
|
||||
db.Exec(fmt.Sprintf("CREATE VIRTUAL TABLE \"%s\" USING fts5(%s, tokenize='unicode61')", ftsTableName, strings.Join(ftsFields, ", ")))
|
||||
|
||||
idField := "id"
|
||||
if len(pks) > 0 {
|
||||
idField = pks[0]
|
||||
}
|
||||
|
||||
// AI Trigger
|
||||
newFtsFields := make([]string, 0, len(ftsFields))
|
||||
for _, f := range ftsFields {
|
||||
newFtsFields = append(newFtsFields, "new."+f)
|
||||
}
|
||||
aiSql := fmt.Sprintf("CREATE TRIGGER IF NOT EXISTS \"%s_ai\" AFTER INSERT ON \"%s\" BEGIN INSERT INTO \"%s\"(rowid, %s) VALUES (new.%s, %s); END;",
|
||||
ftsTableName, table.Name, ftsTableName, strings.Join(ftsFields, ", "), idField, strings.Join(newFtsFields, ", "))
|
||||
db.Exec(aiSql)
|
||||
|
||||
// AD Trigger
|
||||
adSql := fmt.Sprintf("CREATE TRIGGER IF NOT EXISTS \"%s_ad\" AFTER DELETE ON \"%s\" BEGIN DELETE FROM \"%s\" WHERE rowid = old.%s; END;",
|
||||
ftsTableName, table.Name, ftsTableName, idField)
|
||||
db.Exec(adSql)
|
||||
|
||||
// AU Trigger
|
||||
updateSets := make([]string, 0, len(ftsFields))
|
||||
for _, f := range ftsFields {
|
||||
updateSets = append(updateSets, fmt.Sprintf("%s = new.%s", f, f))
|
||||
}
|
||||
auSql := fmt.Sprintf("CREATE TRIGGER IF NOT EXISTS \"%s_au\" AFTER UPDATE ON \"%s\" BEGIN UPDATE \"%s\" SET %s WHERE rowid = old.%s; END;",
|
||||
ftsTableName, table.Name, ftsTableName, strings.Join(updateSets, ", "), idField)
|
||||
db.Exec(auSql)
|
||||
}
|
||||
}
|
||||
SYNC_SHADOW:
|
||||
if table.ShadowDelete && !strings.HasSuffix(table.Name, "_deleted") {
|
||||
table.HasShadowTable = true
|
||||
|
||||
@ -1,17 +1,17 @@
|
||||
package db_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"apigo.cc/go/db"
|
||||
"apigo.cc/go/file"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
func TestSchemaSync(t *testing.T) {
|
||||
dbPath := "test_schema.db"
|
||||
dbInst := db.GetDB("sqlite://"+dbPath, nil)
|
||||
defer os.Remove(dbPath)
|
||||
defer file.Remove(dbPath)
|
||||
defer dbInst.Exec("DROP TABLE IF EXISTS test_table")
|
||||
defer dbInst.Exec("DROP TABLE IF EXISTS test_table_deleted")
|
||||
|
||||
@ -40,7 +40,7 @@ test_table SD // Test table with shadow delete
|
||||
func TestAutoDetectShadow(t *testing.T) {
|
||||
dbPath := "auto_detect.db"
|
||||
dbInst := db.GetDB("sqlite://"+dbPath, nil)
|
||||
defer os.Remove(dbPath)
|
||||
defer file.Remove(dbPath)
|
||||
defer dbInst.Exec("DROP TABLE IF EXISTS test_auto")
|
||||
defer dbInst.Exec("DROP TABLE IF EXISTS test_auto_deleted")
|
||||
|
||||
|
||||
10
Stmt.go
10
Stmt.go
@ -3,13 +3,13 @@ package db
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"apigo.cc/go/log"
|
||||
)
|
||||
|
||||
type Stmt struct {
|
||||
conn *sql.Stmt
|
||||
sqliteMu *sync.Mutex
|
||||
lastSql *string
|
||||
lastArgs []any
|
||||
Error error
|
||||
@ -17,6 +17,10 @@ type Stmt struct {
|
||||
}
|
||||
|
||||
func (stmt *Stmt) Exec(args ...any) *ExecResult {
|
||||
if stmt.sqliteMu != nil {
|
||||
stmt.sqliteMu.Lock()
|
||||
defer stmt.sqliteMu.Unlock()
|
||||
}
|
||||
stmt.lastArgs = args
|
||||
if stmt.conn == nil {
|
||||
return &ExecResult{Sql: stmt.lastSql, Args: stmt.lastArgs, usedTime: -1, logger: stmt.logger, Error: errors.New("operate on a bad connection")}
|
||||
@ -24,7 +28,7 @@ func (stmt *Stmt) Exec(args ...any) *ExecResult {
|
||||
startTime := time.Now()
|
||||
r, err := stmt.conn.Exec(args...)
|
||||
endTime := time.Now()
|
||||
usedTime := log.MakeUsedTime(startTime, endTime)
|
||||
usedTime := makeUsedTime(startTime, endTime)
|
||||
if err != nil {
|
||||
stmt.logger.LogQueryError(err.Error(), *stmt.lastSql, stmt.lastArgs, usedTime)
|
||||
return &ExecResult{Sql: stmt.lastSql, Args: stmt.lastArgs, usedTime: usedTime, logger: stmt.logger, Error: err}
|
||||
|
||||
18
TEST.md
18
TEST.md
@ -2,29 +2,35 @@
|
||||
|
||||
## 📊 概览
|
||||
- **模块**: `apigo.cc/go/db`
|
||||
- **总测试用例**: 5
|
||||
- **通过**: 5
|
||||
- **总测试用例**: 13
|
||||
- **通过**: 13
|
||||
- **失败**: 0
|
||||
- **编译状态**: 成功 (Success)
|
||||
- **测试日期**: 2026-05-03
|
||||
- **测试日期**: 2026-05-13
|
||||
|
||||
## ✅ 详细详情
|
||||
| 测试用例 | 状态 | 耗时 | 备注 |
|
||||
| :--- | :--- | :--- | :--- |
|
||||
| `TestTableProbing` | 通过 | 0.00s | 验证表结构探测 |
|
||||
| `TestMakeInsertSql` | 通过 | 0.00s | 验证 Struct 模型的 SQL 生成逻辑 |
|
||||
| `TestBaseSelect` | 通过 | 0.00s | 验证结果绑定 (Struct, Map, 基础类型) |
|
||||
| `TestInsertReplaceUpdateDelete` | 通过 | 0.01s | 验证 SQLite 下的 CRUD 基本操作 |
|
||||
| `TestTransaction` | 通过 | 0.03s | 验证事务隔离、回滚与提交 |
|
||||
| `TestAutonomousFTS` | 通过 | 0.01s | 验证全文搜索功能 |
|
||||
| `TestSchemaSync` | 通过 | 0.01s | 验证 DSL 同步、影子删除、版本号乐观锁及泛型 API |
|
||||
| `TestAutoDetectShadow` | 通过 | 0.00s | 验证影子表自动检测 |
|
||||
| `TestSmartDelete` | 通过 | 0.01s | 验证智能删除 (物理/影子) |
|
||||
| `TestGenericQuery` | 通过 | 0.00s | 验证泛型查询映射 |
|
||||
| `TestAutoRandomID` | 通过 | 0.01s | 验证 char(N) 主键的自动 ID 填充 |
|
||||
| `TestVersionControl` | 通过 | 0.00s | 验证版本控制递增 |
|
||||
|
||||
## 🚀 性能基准 (Benchmarks)
|
||||
| 基准测试 | 迭代次数 | 耗时 | 内存分配 | 备注 |
|
||||
| :--- | :--- | :--- | :--- | :--- |
|
||||
| `BenchmarkForPool` | 172009 | 7384 ns/op | 1224 B/op (34 allocs) | 验证 SQLite 下的查询绑定性能 |
|
||||
| `BenchmarkForPoolParallel` | 160250 | 6852 ns/op | 1296 B/op (35 allocs) | 验证高并发下的查询稳定性 |
|
||||
| `BenchmarkForPool` | 106807 | 12230 ns/op | - | 验证 SQLite 下的查询绑定性能 (v1.0.11) |
|
||||
| `BenchmarkForPoolParallel` | 86833 | 15723 ns/op | - | 验证高并发下的查询稳定性 (v1.0.11) |
|
||||
|
||||
## 🛠 环境
|
||||
- **OS**: darwin (macOS)
|
||||
- **Go Version**: 1.2x+
|
||||
- **Go Version**: 1.25.0
|
||||
- **Primary Driver**: modernc.org/sqlite
|
||||
|
||||
91
Tx.go
91
Tx.go
@ -5,12 +5,15 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Tx struct {
|
||||
conn *sql.Tx
|
||||
db *DB
|
||||
sqliteMu *sync.Mutex
|
||||
hasLock bool
|
||||
lastSql *string
|
||||
lastArgs []any
|
||||
Error error
|
||||
@ -29,6 +32,7 @@ func (tx *Tx) Quotes(texts []string) string {
|
||||
}
|
||||
|
||||
func (tx *Tx) Commit() error {
|
||||
defer tx.unlock()
|
||||
if tx.isCommittedOrRollbacked {
|
||||
return nil
|
||||
}
|
||||
@ -45,6 +49,7 @@ func (tx *Tx) Commit() error {
|
||||
}
|
||||
|
||||
func (tx *Tx) Rollback() error {
|
||||
defer tx.unlock()
|
||||
if tx.isCommittedOrRollbacked {
|
||||
return nil
|
||||
}
|
||||
@ -78,6 +83,7 @@ func (tx *Tx) CheckFinished() error {
|
||||
}
|
||||
|
||||
func (tx *Tx) Prepare(query string) *Stmt {
|
||||
tx.lock()
|
||||
tx.lastSql = &query
|
||||
r := basePrepare(nil, tx.conn, query)
|
||||
r.logger = tx.logger
|
||||
@ -88,9 +94,12 @@ func (tx *Tx) Prepare(query string) *Stmt {
|
||||
}
|
||||
|
||||
func (tx *Tx) Exec(query string, args ...any) *ExecResult {
|
||||
query, args = tx.db.rewriteFTS(query, args)
|
||||
args = flatArgs(args)
|
||||
tx.lock()
|
||||
tx.lastSql = &query
|
||||
tx.lastArgs = args
|
||||
r := baseExec(nil, tx.conn, query, args...)
|
||||
r := baseExecRaw(nil, tx.conn, query, args...)
|
||||
r.logger = tx.logger
|
||||
if r.Error != nil {
|
||||
tx.logger.LogQueryError(r.Error.Error(), *tx.lastSql, tx.lastArgs, r.usedTime)
|
||||
@ -103,9 +112,12 @@ func (tx *Tx) Exec(query string, args ...any) *ExecResult {
|
||||
}
|
||||
|
||||
func (tx *Tx) Query(query string, args ...any) *QueryResult {
|
||||
query, args = tx.db.rewriteFTS(query, args)
|
||||
args = flatArgs(args)
|
||||
// Query in Tx doesn't acquire lock unless it's already held by a previous write
|
||||
tx.lastSql = &query
|
||||
tx.lastArgs = args
|
||||
r := baseQuery(nil, tx.conn, query, args...)
|
||||
r := baseQueryRaw(nil, tx.conn, query, args...)
|
||||
r.logger = tx.logger
|
||||
if r.Error != nil {
|
||||
tx.logger.LogQueryError(r.Error.Error(), *tx.lastSql, tx.lastArgs, r.usedTime)
|
||||
@ -117,52 +129,20 @@ func (tx *Tx) Query(query string, args ...any) *QueryResult {
|
||||
return r
|
||||
}
|
||||
|
||||
|
||||
func (tx *Tx) Insert(table string, data any) *ExecResult {
|
||||
query, values := tx.MakeInsertSql(table, data, false)
|
||||
tx.lastSql = &query
|
||||
tx.lastArgs = values
|
||||
r := baseExec(nil, tx.conn, query, values...)
|
||||
r.logger = tx.logger
|
||||
if r.Error != nil {
|
||||
tx.logger.LogQueryError(r.Error.Error(), *tx.lastSql, tx.lastArgs, r.usedTime)
|
||||
} else {
|
||||
if tx.logSlow > 0 && r.usedTime >= float32(tx.logSlow/time.Millisecond) {
|
||||
tx.logger.LogQuery(*tx.lastSql, tx.lastArgs, r.usedTime)
|
||||
}
|
||||
}
|
||||
return r
|
||||
return tx.Exec(query, values...)
|
||||
}
|
||||
|
||||
func (tx *Tx) Replace(table string, data any) *ExecResult {
|
||||
query, values := tx.MakeInsertSql(table, data, true)
|
||||
tx.lastSql = &query
|
||||
tx.lastArgs = values
|
||||
r := baseExec(nil, tx.conn, query, values...)
|
||||
r.logger = tx.logger
|
||||
if r.Error != nil {
|
||||
tx.logger.LogQueryError(r.Error.Error(), *tx.lastSql, tx.lastArgs, r.usedTime)
|
||||
} else {
|
||||
if tx.logSlow > 0 && r.usedTime >= float32(tx.logSlow/time.Millisecond) {
|
||||
tx.logger.LogQuery(*tx.lastSql, tx.lastArgs, r.usedTime)
|
||||
}
|
||||
}
|
||||
return r
|
||||
return tx.Exec(query, values...)
|
||||
}
|
||||
|
||||
func (tx *Tx) Update(table string, data any, conditions string, args ...any) *ExecResult {
|
||||
query, values := tx.MakeUpdateSql(table, data, conditions, args...)
|
||||
tx.lastSql = &query
|
||||
tx.lastArgs = values
|
||||
r := baseExec(nil, tx.conn, query, values...)
|
||||
r.logger = tx.logger
|
||||
if r.Error != nil {
|
||||
tx.logger.LogQueryError(r.Error.Error(), *tx.lastSql, tx.lastArgs, r.usedTime)
|
||||
} else {
|
||||
if tx.logSlow > 0 && r.usedTime >= float32(tx.logSlow/time.Millisecond) {
|
||||
tx.logger.LogQuery(*tx.lastSql, tx.lastArgs, r.usedTime)
|
||||
}
|
||||
}
|
||||
return r
|
||||
return tx.Exec(query, values...)
|
||||
}
|
||||
|
||||
func (tx *Tx) Delete(table string, conditions string, args ...any) *ExecResult {
|
||||
@ -185,24 +165,31 @@ func (tx *Tx) Delete(table string, conditions string, args ...any) *ExecResult {
|
||||
colList = " select *"
|
||||
}
|
||||
moveQuery := fmt.Sprintf("insert into %s%s from %s%s", tx.Quote(table+"_deleted"), colList, tx.Quote(table), where)
|
||||
r := baseExec(nil, tx.conn, moveQuery, args...)
|
||||
// Use Exec to handle locking
|
||||
r := tx.Exec(moveQuery, args...)
|
||||
if r.Error != nil {
|
||||
tx.logger.LogQueryError(r.Error.Error(), moveQuery, args, r.usedTime)
|
||||
return r
|
||||
}
|
||||
}
|
||||
|
||||
query := fmt.Sprintf("delete from %s%s", tx.Quote(table), where)
|
||||
tx.lastSql = &query
|
||||
tx.lastArgs = args
|
||||
r := baseExec(nil, tx.conn, query, args...)
|
||||
r.logger = tx.logger
|
||||
if r.Error != nil {
|
||||
tx.logger.LogQueryError(r.Error.Error(), *tx.lastSql, tx.lastArgs, r.usedTime)
|
||||
} else {
|
||||
if tx.logSlow > 0 && r.usedTime >= float32(tx.logSlow/time.Millisecond) {
|
||||
tx.logger.LogQuery(*tx.lastSql, tx.lastArgs, r.usedTime)
|
||||
}
|
||||
}
|
||||
return r
|
||||
return tx.Exec(query, args...)
|
||||
}
|
||||
|
||||
func (tx *Tx) lock() {
|
||||
if tx.sqliteMu == nil || tx.hasLock {
|
||||
return
|
||||
}
|
||||
tx.sqliteMu.Lock()
|
||||
tx.hasLock = true
|
||||
}
|
||||
|
||||
func (tx *Tx) unlock() {
|
||||
if tx.sqliteMu == nil || !tx.hasLock {
|
||||
return
|
||||
}
|
||||
tx.sqliteMu.Unlock()
|
||||
tx.hasLock = false
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
package db_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"apigo.cc/go/db"
|
||||
"apigo.cc/go/file"
|
||||
"apigo.cc/go/log"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
@ -14,7 +14,7 @@ func TestSmartDelete(t *testing.T) {
|
||||
|
||||
dbPath := "./test_smart_delete.db"
|
||||
dbName := "test_delete"
|
||||
os.Remove(dbPath)
|
||||
file.Remove(dbPath)
|
||||
|
||||
db.SetConfigForTest(dbName, &db.Config{
|
||||
Type: "sqlite",
|
||||
@ -27,7 +27,7 @@ func TestSmartDelete(t *testing.T) {
|
||||
}
|
||||
defer func() {
|
||||
dbInst.Destroy()
|
||||
os.Remove(dbPath)
|
||||
file.Remove(dbPath)
|
||||
}()
|
||||
|
||||
// Create table and shadow table
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
package db_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
"apigo.cc/go/cast"
|
||||
"apigo.cc/go/db"
|
||||
"apigo.cc/go/file"
|
||||
"apigo.cc/go/log"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
@ -12,7 +12,7 @@ import (
|
||||
func TestGenericQuery(t *testing.T) {
|
||||
db.ResetAllForTest()
|
||||
dbPath := "./test_generic.db"
|
||||
os.Remove(dbPath)
|
||||
file.Remove(dbPath)
|
||||
|
||||
db.SetConfigForTest("test_generic", &db.Config{Type: "sqlite", Host: dbPath})
|
||||
dbInst := db.GetDB("test_generic", log.DefaultLogger)
|
||||
@ -21,7 +21,7 @@ func TestGenericQuery(t *testing.T) {
|
||||
}
|
||||
defer func() {
|
||||
dbInst.Destroy()
|
||||
os.Remove(dbPath)
|
||||
file.Remove(dbPath)
|
||||
}()
|
||||
|
||||
r := dbInst.Query("SELECT 1 as num, 'hello' as str")
|
||||
|
||||
30
go.mod
30
go.mod
@ -3,15 +3,15 @@ module apigo.cc/go/db
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
apigo.cc/go/cast v1.2.8
|
||||
apigo.cc/go/config v1.0.6
|
||||
apigo.cc/go/crypto v1.0.5
|
||||
apigo.cc/go/id v1.0.5
|
||||
apigo.cc/go/log v1.1.13
|
||||
apigo.cc/go/rand v1.0.5
|
||||
apigo.cc/go/redis v1.0.8
|
||||
apigo.cc/go/safe v1.0.5
|
||||
apigo.cc/go/shell v1.0.5
|
||||
apigo.cc/go/cast v1.3.3
|
||||
apigo.cc/go/config v1.3.1
|
||||
apigo.cc/go/crypto v1.3.1
|
||||
apigo.cc/go/file v1.3.2
|
||||
apigo.cc/go/id v1.3.1
|
||||
apigo.cc/go/log v1.3.4
|
||||
apigo.cc/go/redis v1.3.2
|
||||
apigo.cc/go/safe v1.3.1
|
||||
apigo.cc/go/shell v1.3.1
|
||||
github.com/go-sql-driver/mysql v1.10.0
|
||||
github.com/jackc/pgx/v5 v5.9.2
|
||||
github.com/mitchellh/mapstructure v1.5.0
|
||||
@ -19,11 +19,11 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
apigo.cc/go/encoding v1.0.5 // indirect
|
||||
apigo.cc/go/file v1.0.6 // indirect
|
||||
apigo.cc/go/encoding v1.3.1 // indirect
|
||||
apigo.cc/go/rand v1.3.1 // indirect
|
||||
filippo.io/edwards25519 v1.2.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/gomodule/redigo v1.9.3 // indirect
|
||||
github.com/gomodule/redigo v2.0.0+incompatible // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
@ -31,10 +31,10 @@ require (
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
golang.org/x/crypto v0.50.0 // indirect
|
||||
golang.org/x/crypto v0.51.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/sys v0.43.0 // indirect
|
||||
golang.org/x/text v0.36.0 // indirect
|
||||
golang.org/x/sys v0.44.0 // indirect
|
||||
golang.org/x/text v0.37.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
modernc.org/libc v1.72.0 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
|
||||
68
go.sum
68
go.sum
@ -1,25 +1,25 @@
|
||||
apigo.cc/go/cast v1.2.8 h1:plb676DH2TjYljzf8OEMGT6lIhmZ/xaxEFfs0kDOiSI=
|
||||
apigo.cc/go/cast v1.2.8/go.mod h1:lGlwImiOvHxG7buyMWhFzcdvQzmSaoKbmr7bcDfUpHk=
|
||||
apigo.cc/go/config v1.0.6 h1:32nOCr+8AkGFnKuythCjHPOjxilg6SOlSWXKTkNtx6I=
|
||||
apigo.cc/go/config v1.0.6/go.mod h1:nX+nLKZTP6Xton9Gt/9XsTh0d1sQ+Qkwysgyjq/k4R0=
|
||||
apigo.cc/go/crypto v1.0.5 h1:uJyO9X5O9OAnGGy37fuxdC18uZkOB1PyOYSFZiaDY5Q=
|
||||
apigo.cc/go/crypto v1.0.5/go.mod h1:c7a7sY2Yv9/531WS72L3tmB7OqdY3IUWIS1Jhv0Pfgc=
|
||||
apigo.cc/go/encoding v1.0.5 h1:a2XbXyd8D2gKo1ekXn/pt5adltWbIfdJCMhaF2uvzF0=
|
||||
apigo.cc/go/encoding v1.0.5/go.mod h1:V5CgT7rBbCxy+uCU20q0ptcNNRSgMtpA8cNOs6r8IeI=
|
||||
apigo.cc/go/file v1.0.6 h1:kyrPJ+oqC0DtYubX2aI+3QIVoDAPkRiYyBwd1F0cBlA=
|
||||
apigo.cc/go/file v1.0.6/go.mod h1:AOw8+3q1fmCZpBWpBfUSSb+Q6Li3W9jH1EktQXmFhVg=
|
||||
apigo.cc/go/id v1.0.5 h1:23YkR7oklSA69gthYlu8zl/kpIkeIoEYxi1f1Sz5l3A=
|
||||
apigo.cc/go/id v1.0.5/go.mod h1:ZaYLIyrJvkf3j7J8a0lnKywSAHljaczWxU0x2HmQDzg=
|
||||
apigo.cc/go/log v1.1.13 h1:ZABeVA9DxhdneLqHrYEc+6YijgoygG8eEsgDxYDzpDc=
|
||||
apigo.cc/go/log v1.1.13/go.mod h1:eabuI2SynGNgo5FXPbGgQtyxjp94wT643XzjYhEIP3A=
|
||||
apigo.cc/go/rand v1.0.5 h1:AkUoWr0SELgeDmRjLEDjOIp29nXdzqQQvmGRIHpTN7U=
|
||||
apigo.cc/go/rand v1.0.5/go.mod h1:mZ/4Soa3bk+XvDaqPWJuUe1bfEi4eThBj1XmEAuYxsk=
|
||||
apigo.cc/go/redis v1.0.8 h1:uNpjPiS9zlcaGxacSMtNUYaN3xIfeNk030pZFDSQcRk=
|
||||
apigo.cc/go/redis v1.0.8/go.mod h1:gEgnzhrrlZHL6XzsKEG+zR2y6l/eWIbwdT1dbhbG/7g=
|
||||
apigo.cc/go/safe v1.0.5 h1:yZJLhpMntJrtqU/ev0UlyOoHu/cLrnnGUO4aHyIZcwE=
|
||||
apigo.cc/go/safe v1.0.5/go.mod h1:i9xnh7reJIFPauLnlzuIDgvrQvhjxpFlpVh3O6ulWd0=
|
||||
apigo.cc/go/shell v1.0.5 h1:bmvUTJGe1GwsHAy42v3iaoK40PoBC7Xq1aMCYxUZmtg=
|
||||
apigo.cc/go/shell v1.0.5/go.mod h1:sx/nYw5CihHWmo5JHkaZUbmMYXNHx8swzArbQCUGHjc=
|
||||
apigo.cc/go/cast v1.3.3 h1:aln5eDR5DZVWVzZ/y5SJh1gQNgWv2sT82I25NaO9g34=
|
||||
apigo.cc/go/cast v1.3.3/go.mod h1:lGlwImiOvHxG7buyMWhFzcdvQzmSaoKbmr7bcDfUpHk=
|
||||
apigo.cc/go/config v1.3.1 h1:wZzUh4oL+fGD6SayVgX6prLPMsniM25etWFcEH8XzIE=
|
||||
apigo.cc/go/config v1.3.1/go.mod h1:7KHz/1WmtBLM762Lln/TaXh2dmlMvJTLhnlk33zbS3U=
|
||||
apigo.cc/go/crypto v1.3.1 h1:ulQ2zX9bUWirk0sEacx1Srsjs2Jow7HlZq7ED7msNcg=
|
||||
apigo.cc/go/crypto v1.3.1/go.mod h1:SwHlBFDPddttWgFFtzsEMla8CM/rcFy9nvdsJjW4CIs=
|
||||
apigo.cc/go/encoding v1.3.1 h1:y8O58KYAyulkThg1O2ji2BqjnFoSvk42sit9I3z+K7Y=
|
||||
apigo.cc/go/encoding v1.3.1/go.mod h1:xAJk5b83VZ31mXMTnyp0dfMoBKfT/AHDn0u+cQfojgY=
|
||||
apigo.cc/go/file v1.3.2 h1:pu4oiDyiqgj3/eykfnJf+/6+A9v/Z0b3ClP5XK+lwG4=
|
||||
apigo.cc/go/file v1.3.2/go.mod h1:vci4h0Pz94mV6dkniQkuyBYERVYeq7/LX4jJVuCg9hs=
|
||||
apigo.cc/go/id v1.3.1 h1:pkqi6VeWyQoHuIu0Zbx/RRxIAdM61Js0j6cY1M9XVCk=
|
||||
apigo.cc/go/id v1.3.1/go.mod h1:P2/vl3tyW3US+ayOFSMoPIOCulNLBngNYPhXJC/Z7J4=
|
||||
apigo.cc/go/log v1.3.4 h1:UT8Neb9r4QjjbCFbTzw+ZeTxd+DmdmR5gNExeR4Cj+g=
|
||||
apigo.cc/go/log v1.3.4/go.mod h1:/Q/2r51xWSsrS4QN5U9jLiTw8n6qNC8kG9nuVHweY20=
|
||||
apigo.cc/go/rand v1.3.1 h1:7FvsI6PtQ5XrWER0dTiLVo0p7GIxRidT/TBKhVy93j8=
|
||||
apigo.cc/go/rand v1.3.1/go.mod h1:mZ/4Soa3bk+XvDaqPWJuUe1bfEi4eThBj1XmEAuYxsk=
|
||||
apigo.cc/go/redis v1.3.2 h1:iUWL/CHHnfonz0dJq6/V4IG3QuXBoHA2L1xnoGEbNEQ=
|
||||
apigo.cc/go/redis v1.3.2/go.mod h1:/k5wcfAzB9jrfd9otabio9CPUxEsLPgEs4oggBG5sbs=
|
||||
apigo.cc/go/safe v1.3.1 h1:irTCqPAC97gGsX/Lw5AzLelDt1xXLEZIAaVhLELWe9Q=
|
||||
apigo.cc/go/safe v1.3.1/go.mod h1:XdOpBhN2vkImalaykYXXmEpczqWa1y3ah6/Q72cdRqE=
|
||||
apigo.cc/go/shell v1.3.1 h1:M8oD0b2HcJuCC6frQFx11b3UTcTx3lATX8XK+YXSVm8=
|
||||
apigo.cc/go/shell v1.3.1/go.mod h1:ZMdJjpCpWdvsHKUXlelh/AxsV/nWdkH/k3lISfzMdUw=
|
||||
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
|
||||
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@ -29,8 +29,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw=
|
||||
github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
|
||||
github.com/gomodule/redigo v1.9.3 h1:dNPSXeXv6HCq2jdyWfjgmhBdqnR6PRO3m/G05nvpPC8=
|
||||
github.com/gomodule/redigo v1.9.3/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw=
|
||||
github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
|
||||
github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
@ -66,19 +66,19 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
|
||||
golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
|
||||
golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
|
||||
golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
|
||||
golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
|
||||
golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
|
||||
golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM=
|
||||
golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
|
||||
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
|
||||
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
|
||||
golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
|
||||
golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
|
||||
golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
|
||||
golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
||||
golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c=
|
||||
golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
|
||||
@ -5,13 +5,14 @@ import (
|
||||
"testing"
|
||||
|
||||
"apigo.cc/go/db"
|
||||
"apigo.cc/go/file"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
func TestAutoRandomID(t *testing.T) {
|
||||
dbPath := "id_test.db"
|
||||
dbset := "sqlite://" + dbPath
|
||||
defer os.Remove(dbPath)
|
||||
defer file.Remove(dbPath)
|
||||
|
||||
dbInst := db.GetDB(dbset, nil)
|
||||
// Create table with char(12) primary key
|
||||
|
||||
@ -1,16 +1,16 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"apigo.cc/go/file"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
func TestTableProbing(t *testing.T) {
|
||||
ResetAllForTest()
|
||||
dbPath := "./test_probing.db"
|
||||
os.Remove(dbPath)
|
||||
file.Remove(dbPath)
|
||||
SetConfigForTest("test_probing", &Config{Type: "sqlite", Host: dbPath})
|
||||
dbInst := GetDB("test_probing", nil)
|
||||
if dbInst == nil {
|
||||
@ -18,7 +18,7 @@ func TestTableProbing(t *testing.T) {
|
||||
}
|
||||
defer func() {
|
||||
dbInst.Destroy()
|
||||
os.Remove(dbPath)
|
||||
file.Remove(dbPath)
|
||||
}()
|
||||
|
||||
|
||||
|
||||
74
tokenize.go
Normal file
74
tokenize.go
Normal file
@ -0,0 +1,74 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
var punctuationReg = regexp.MustCompile(`[^\p{L}\p{N}]+`)
|
||||
|
||||
// BigramTokenize 将文本进行二元分词,用于全文检索影子列
|
||||
// 规则:
|
||||
// 1. 移除非字母数字的标点符号,按空格/标点初步切分块。
|
||||
// 2. 对每个块内的 CJK(中日韩)字符,使用滑动窗口进行 2-gram 切分。
|
||||
// 3. 对于块内的非 CJK(英文、数字等)字符,按单词整体保留。
|
||||
func BigramTokenize(text string) string {
|
||||
if text == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
// 1. 初步切分,按非字母数字字符分割
|
||||
chunks := punctuationReg.Split(text, -1)
|
||||
var allTokens []string
|
||||
|
||||
for _, chunk := range chunks {
|
||||
if chunk == "" {
|
||||
continue
|
||||
}
|
||||
runes := []rune(chunk)
|
||||
length := len(runes)
|
||||
|
||||
var currentWord []rune
|
||||
for i := 0; i < length; i++ {
|
||||
r := runes[i]
|
||||
if isCJK(r) {
|
||||
// 遇到中文字符,先冲刷掉之前的英文单词
|
||||
if len(currentWord) > 0 {
|
||||
allTokens = append(allTokens, string(currentWord))
|
||||
currentWord = nil
|
||||
}
|
||||
// 1-gram
|
||||
allTokens = append(allTokens, string(r))
|
||||
// 2-gram
|
||||
if i < length-1 && isCJK(runes[i+1]) {
|
||||
allTokens = append(allTokens, string(runes[i:i+2]))
|
||||
}
|
||||
} else {
|
||||
// 累积英文/数字
|
||||
currentWord = append(currentWord, r)
|
||||
}
|
||||
}
|
||||
// 循环结束,冲刷最后一个单词
|
||||
if len(currentWord) > 0 {
|
||||
allTokens = append(allTokens, string(currentWord))
|
||||
}
|
||||
}
|
||||
|
||||
// 4. 去重,减小索引体积
|
||||
tokenMap := make(map[string]bool)
|
||||
var uniqueTokens []string
|
||||
for _, t := range allTokens {
|
||||
if !tokenMap[t] {
|
||||
tokenMap[t] = true
|
||||
uniqueTokens = append(uniqueTokens, t)
|
||||
}
|
||||
}
|
||||
|
||||
return strings.Join(uniqueTokens, " ")
|
||||
}
|
||||
|
||||
func isCJK(r rune) bool {
|
||||
return unicode.Is(unicode.Han, r) ||
|
||||
unicode.In(r, unicode.Hiragana, unicode.Katakana, unicode.Hangul)
|
||||
}
|
||||
12
utils.go
Normal file
12
utils.go
Normal file
@ -0,0 +1,12 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func runShell(command string) string {
|
||||
cmd := exec.Command("bash", "-c", command)
|
||||
out, _ := cmd.CombinedOutput()
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
@ -1,11 +1,11 @@
|
||||
package db_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"apigo.cc/go/db"
|
||||
"apigo.cc/go/file"
|
||||
"apigo.cc/go/log"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
@ -13,7 +13,7 @@ import (
|
||||
func TestVersionControl(t *testing.T) {
|
||||
db.ResetAllForTest()
|
||||
dbPath := "./test_version.db"
|
||||
os.Remove(dbPath)
|
||||
file.Remove(dbPath)
|
||||
db.SetConfigForTest("test_version", &db.Config{Type: "sqlite", Host: dbPath})
|
||||
dbInst := db.GetDB("test_version", log.DefaultLogger)
|
||||
if dbInst == nil {
|
||||
@ -21,7 +21,7 @@ func TestVersionControl(t *testing.T) {
|
||||
}
|
||||
defer func() {
|
||||
dbInst.Destroy()
|
||||
os.Remove(dbPath)
|
||||
file.Remove(dbPath)
|
||||
}()
|
||||
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user