From 529feee4da6abad8692c5849645096e59bd6596b Mon Sep 17 00:00:00 2001 From: K Date: Sat, 13 Jan 2024 06:18:12 +0800 Subject: [PATCH] :pill: add cache for long text query; :bug: fixed #1, #23, #24 --- cmd/kd.go | 3 +-- internal/cache/cache.go | 50 ++++++++++++++++++++++++++++++++++++++++ internal/cache/utils.go | 4 ++++ internal/client.go | 20 +++++++--------- internal/query/online.go | 16 ++++++++++--- internal/query/output.go | 13 ++++++++++- internal/query/query.go | 7 +++++- pkg/file.go | 1 + plan.md | 5 +--- 9 files changed, 96 insertions(+), 23 deletions(-) diff --git a/cmd/kd.go b/cmd/kd.go index f8d7825..7c69279 100644 --- a/cmd/kd.go +++ b/cmd/kd.go @@ -24,7 +24,7 @@ import ( "go.uber.org/zap" ) -var VERSION = "v0.0.6" +var VERSION = "v0.0.7" func showPrompt() { exename, err := pkg.GetExecutableBasename() @@ -185,7 +185,6 @@ func flagStatus(*cli.Context, bool) error { if err == nil { fmt.Printf(" Binary地址：%s\n", kdpath) } - return err } diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 0ec1e85..220627a 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -5,9 +5,11 @@ import ( "compress/zlib" "encoding/json" "errors" + "fmt" "io" "os" "path/filepath" + "time" "github.com/Karmenzind/kd/internal/model" "github.com/Karmenzind/kd/pkg" @@ -76,6 +78,54 @@ func UpdateQueryCache(r *model.Result) (err error) { } // ----------------------------------------------------------------------------- +// Long Text query cache +// ----------------------------------------------------------------------------- + +type LongTextData struct { + Result string `json:"r"` + AccessTS int64 `json:"a"` + CreateTS int64 `json:"c"` +} + +func GetLongTextCache(r *model.Result) (err error) { + var m map[string]LongTextData + if pkg.IsPathExists(LONG_TEXT_CACHE_FILE) { + err = pkg.LoadJson(LONG_TEXT_CACHE_FILE, &m) + if err != nil { + return err + } + if res, ok := m[r.Query]; ok { + r.MachineTrans = res.Result + zap.S().Debugf("Got cached '%s'", r.Query) + (&res).AccessTS = time.Now().Unix() + m[r.Query] = res + go pkg.SaveJson(LONG_TEXT_CACHE_FILE, &m) + return + } else { + return fmt.Errorf("no cache for %s", r.Query) + } + } + return +} + +func UpdateLongTextCache(r *model.Result) (err error) { + var m map[string]LongTextData + if pkg.IsPathExists(LONG_TEXT_CACHE_FILE) { + err = pkg.LoadJson(LONG_TEXT_CACHE_FILE, &m) + if err != nil { + return err + } + } else { + m = map[string]LongTextData{} + } + now := time.Now().Unix() + m[r.Query] = LongTextData{r.MachineTrans, now, now} + err = pkg.SaveJson(LONG_TEXT_CACHE_FILE, m) + return err +} + +// ----------------------------------------------------------------------------- +// deprecated // JSON version // ----------------------------------------------------------------------------- diff --git a/internal/cache/utils.go b/internal/cache/utils.go index 14ea62a..acd2898 100644 --- a/internal/cache/utils.go +++ b/internal/cache/utils.go @@ -3,6 +3,7 @@ package cache import ( "fmt" "os" + "path/filepath" "github.com/Karmenzind/kd/internal/run" d "github.com/Karmenzind/kd/pkg/decorate" @@ -13,6 +14,8 @@ var CACHE_WORDS_PATH = run.CACHE_WORDS_PATH var CACHE_RUN_PATH = run.CACHE_RUN_PATH var CACHE_STAT_DIR_PATH = run.CACHE_STAT_DIR_PATH +var LONG_TEXT_CACHE_FILE string + func init() { for _, directory := range []string{ CACHE_ROOT_PATH, @@ -25,4 +28,5 @@ func init() { d.EchoFatal(fmt.Sprintf("Failed to create %s", directory)) } } + LONG_TEXT_CACHE_FILE = filepath.Join(CACHE_ROOT_PATH, "long_text_results.json") } diff --git a/internal/client.go b/internal/client.go index 05b22a8..f2b5dfa 100644 --- a/internal/client.go +++ b/internal/client.go @@ -66,8 +66,10 @@ func Query(query string, noCache bool, longText bool) (r *model.Result, err erro daemonRunning := make(chan bool) go ensureDaemon(daemonRunning) - core.WG.Add(1) - go cache.CounterIncr(query, r.History) + if !longText { + core.WG.Add(1) + go cache.CounterIncr(query, r.History) + } // any valid char if m, _ := regexp.MatchString("^[a-zA-Z0-9\u4e00-\u9fa5]", query); !m { @@ -91,18 +93,12 @@ func Query(query string, noCache bool, longText bool) (r *model.Result, err erro inNotFound = true } r.Initialize() + } - if !noCache { - cacheErr := q.FetchCached(r) - if cacheErr != nil { - zap.S().Warnf("[cache] Query error: %s", cacheErr) - } - if r.Found { - return - } - _ = err + if !noCache { + if cacheErr := q.FetchCached(r); cacheErr == nil && r.Found { + return } - } if <-daemonRunning { diff --git a/internal/query/online.go b/internal/query/online.go index 4abe2dd..f77901d 100644 --- a/internal/query/online.go +++ b/internal/query/online.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "os" + "path/filepath" "strconv" "strings" "time" @@ -13,6 +14,7 @@ import ( "github.com/Karmenzind/kd/config" "github.com/Karmenzind/kd/internal/cache" "github.com/Karmenzind/kd/internal/model" + "github.com/Karmenzind/kd/internal/run" "github.com/Karmenzind/kd/pkg" "github.com/anaskhan96/soup" "go.uber.org/zap" @@ -66,9 +68,16 @@ func requestYoudao(r *model.Result) (body []byte, err error) { zap.S().Debugf("[http-get] detail: header %+v", url, len(body), resp.Header) } if config.Cfg.Debug { - errW := os.WriteFile(fmt.Sprintf("/home/k/Workspace/kd/data/%s.html", r.Query), (body), 0666) - if errW != nil { - zap.S().Warnf("Failed to write file '%s': %s", r.Query, errW) + htmlDir := filepath.Join(run.CACHE_ROOT_PATH, "html") + errW := os.MkdirAll(htmlDir, os.ModePerm) + if errW == nil { + htmlPath := filepath.Join(htmlDir, fmt.Sprintf("%s.html", r.Query)) + errW = os.WriteFile(htmlPath, body, 0666) + } + if errW == nil { + zap.S().Debugf("Saved '%s.html'", r.Query) + } else { + zap.S().Warnf("Failed to save html data for '%s': %s", r.Query, errW) } } return @@ -102,6 +111,7 @@ func FetchOnline(r *model.Result) (err error) { yr.parseMachineTrans() if r.MachineTrans != "" { r.Found = true + go cache.UpdateLongTextCache(r) } return } diff --git a/internal/query/output.go b/internal/query/output.go index f984cb6..ea34968 100644 --- a/internal/query/output.go +++ b/internal/query/output.go @@ -150,7 +150,18 @@ func displayExample(item []string, tab string, onlyEN bool, isEN bool) string { if onlyEN { r = d.EgEn(item[0]) } else { - r = fmt.Sprintf("%s %s", d.EgEn(item[0]), d.EgCh(item[1])) + var rh string + if len(item) >= 3 { + // if strings.ToLower(item[2]) != "youdao" { + // rh = d.EgCh(item[1]) + d.EgCh(item[2]) + // } + rh = d.EgCh(item[1]) + d.EgCh(item[2]) + } + if rh == "" { + rh = d.EgCh(item[1]) + } + r = fmt.Sprintf("%s %s", d.EgEn(item[0]), rh) + // r = fmt.Sprintf("%s %s", d.EgEn(item[0]), d.EgCh(item[1])) } case "au": // TODO 增加来源渲染 diff --git a/internal/query/query.go b/internal/query/query.go index f619178..03d951f 100644 --- a/internal/query/query.go +++ b/internal/query/query.go @@ -23,11 +23,16 @@ TODO: */ func FetchCached(r *model.Result) (err error) { - err = cache.GetCachedQuery(r) + if r.IsLongText { + err = cache.GetLongTextCache(r) + } else { + err = cache.GetCachedQuery(r) + } if err == nil { r.Found = true return } + zap.S().Debugf("[cache] Query error: %s", err) r.Found = false return } diff --git a/pkg/file.go b/pkg/file.go index 58ad817..d6cdcfd 100644 --- a/pkg/file.go +++ b/pkg/file.go @@ -8,6 +8,7 @@ import ( "go.uber.org/zap" ) + func IsPathExists(p string) bool { if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) { return false diff --git a/plan.md b/plan.md index 063eec6..b63b5a5 100644 --- a/plan.md +++ b/plan.md @@ -5,8 +5,6 @@ - 读文件加锁 - 指定port - release增加version，aur判断此文件 -- 长句查询 (另外缓存) -- notfound counter 忽略longtext - gitea镜像 ## short-term @@ -25,7 +23,6 @@ - 加入词库设置，供选择词库大小 ## low priority -- 一键安装脚本 - --update下载之后缓存，避免重复下载 - cli替换为cobra - source数据 分为base-sourse & web-source @@ -34,8 +31,8 @@ - 检测配置保存时间变化的基础上再加上内容判断？ - not found list记录查询时间，超时删除 - not found和索引都加入服务端缓存，benchmark比较直接查本地和tcp通信的速度 - - AUR收尾工作 +- move default log to cache dir # BUG