From d62eb76434df17d567817f45db3d704c31fd8a2f Mon Sep 17 00:00:00 2001 From: shaoying Date: Tue, 21 Jan 2020 20:03:22 +0800 Subject: [PATCH] add save --- base/web/web.go | 4 ++- misc/input/input.go | 88 ++++++++++++++++++++++++++------------------- type.go | 28 ++++++++------- 3 files changed, 70 insertions(+), 50 deletions(-) diff --git a/base/web/web.go b/base/web/web.go index 54608524..3b55f215 100644 --- a/base/web/web.go +++ b/base/web/web.go @@ -378,7 +378,9 @@ var Index = &ice.Context{Name: "web", Help: "网络模块", "cmd", []interface{}{ice.CLI_SYSTEM, "ice.sh", "start", ice.WEB_SPACE, "connect"}, )}, ice.WEB_FAVOR: {Name: "favor", Help: "收藏夹", Value: kit.Data(kit.MDB_SHORT, kit.MDB_NAME)}, - ice.WEB_CACHE: {Name: "cache", Help: "缓存池", Value: kit.Data(kit.MDB_SHORT, "text", "path", "var/file", "store", "var/data", "limit", "30", "least", "10")}, + ice.WEB_CACHE: {Name: "cache", Help: "缓存池", Value: kit.Data( + kit.MDB_SHORT, "text", "path", "var/file", "store", "var/data", "limit", "30", "least", "10", "fsize", "100000", + )}, ice.WEB_STORY: {Name: "story", Help: "故事会", Value: kit.Dict( kit.MDB_META, kit.Dict(kit.MDB_SHORT, "data"), "head", kit.Data(kit.MDB_SHORT, "story"), diff --git a/misc/input/input.go b/misc/input/input.go index fde98d70..3aa1e92c 100644 --- a/misc/input/input.go +++ b/misc/input/input.go @@ -4,10 +4,10 @@ import ( "bufio" "bytes" "encoding/csv" + "fmt" "github.com/shylinux/icebergs" "github.com/shylinux/icebergs/base/cli" "github.com/shylinux/toolkits" - "math/rand" "os" "path" "strings" @@ -17,28 +17,32 @@ var Index = &ice.Context{Name: "input", Help: "输入法", Caches: map[string]*ice.Cache{}, Configs: map[string]*ice.Config{ "input": {Name: "input", Help: "输入法", Value: kit.Data( - "store", "var/input/", "limit", "2000", "least", "1000", - kit.MDB_SHORT, "code", + "store", "var/input/", "limit", "2000", "least", "1000", "fsize", "100000", + "field", kit.Dict("file", 0, "line", 1, "code", 2, "id", 3, "text", 4, "time", 5, "weight", 6), )}, }, Commands: map[string]*ice.Command{ ice.ICE_INIT: {Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { m.Cmd(ice.CTX_CONFIG, "load", "input.json") }}, - ice.ICE_EXIT: {Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { m.Cmd(ice.CTX_CONFIG, "save", "input.json", "cli.input.input") }}, - "input": {Name: "input load|list", Help: "输入法", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { - count := kit.Int(m.Conf("input", "meta.count")) + "input": {Name: "input load|list|push|save", Help: "输入法", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { if len(arg) == 0 { - arg = append(arg, "list", kit.Format(count-rand.Intn(count))) + arg = append(arg, "list") } switch arg[0] { case "load": // 加载词库 + lib := kit.Select(path.Base(arg[1]), arg, 2) + m.Option("cache.fsize", m.Conf("input", "meta.fsize")) + m.Option("cache.limit", m.Conf("input", "meta.limit")) + m.Option("cache.least", m.Conf("input", "meta.least")) + m.Assert(os.RemoveAll(m.Option("cache.store", path.Join(m.Conf("input", "meta.store"), lib)))) + m.Conf("input", lib, "") if f, e := os.Open(arg[1]); m.Assert(e) { bio := bufio.NewScanner(f) for bio.Scan() { @@ -46,22 +50,52 @@ var Index = &ice.Context{Name: "input", Help: "输入法", continue } line := kit.Split(bio.Text(), " \t") - m.Grow("input", nil, kit.Dict( + if line[2] == "0" { + continue + } + m.Grow("input", lib, kit.Dict( "text", line[0], "code", line[1], "weight", line[2], )) } + m.Option("cache.limit", 0) + m.Option("cache.least", 0) + n := m.Grow("input", lib, kit.Dict( + "text", "成功", "code", "z", "weight", "0", + )) + m.Echo("%s: %d", lib, n) } case "push": - m.Rich("input", nil, kit.Dict( - "id", "0", "text", arg[1], "code", arg[2], "weight", kit.Select("99990000", arg, 3), + // 添加词汇 + lib := kit.Select("person", arg, 3) + m.Option("cache.limit", 0) + m.Option("cache.least", 0) + m.Option("cache.store", path.Join(m.Conf("input", "meta.store"), lib)) + n := m.Grow("input", lib, kit.Dict( + "text", arg[1], "code", arg[2], "weight", kit.Select("99990000", arg, 4), )) + m.Echo("%s: %d", lib, n) case "list": // 词汇列表 - m.Option("cache.offend", kit.Select("0", arg, 1)) - m.Option("cache.limit", kit.Select("10", arg, 2)) - m.Grows("input", nil, "", "", func(index int, value map[string]interface{}) { + lib := kit.Select("person", arg, 1) + m.Option("cache.offend", kit.Select("0", arg, 2)) + m.Option("cache.limit", kit.Select("10", arg, 3)) + m.Grows("input", lib, "", "", func(index int, value map[string]interface{}) { m.Push("", value, []string{"id", "code", "text", "weight"}) }) + case "save": + // 导出词库 + m.Option("cache.offend", 0) + m.Option("cache.limit", 1000000) + if f, p, e := kit.Create(kit.Select(arg[1], arg, 2)); m.Assert(e) { + defer f.Close() + n := 0 + m.Grows("input", arg[1], "", "", func(index int, value map[string]interface{}) { + n++ + fmt.Fprintf(f, "%s %s %s\n", value["text"], value["code"], value["weight"]) + }) + m.Log(ice.LOG_EXPORT, "%s: %d", p, n) + m.Echo("%s: %d", p, n) + } } }}, "match": {Name: "match [word [method]]", Help: "五笔字码", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { @@ -71,11 +105,6 @@ var Index = &ice.Context{Name: "input", Help: "输入法", return } - m.Info("add %s: %s", arg[0], kit.Hashs(arg[0])) - m.Richs("input", nil, arg[0], func(key string, value map[string]interface{}) { - m.Push(key, value, []string{"id", "code", "text", "weight"}) - }) - // 搜索方法 method := "word" if len(arg) > 1 { @@ -87,25 +116,9 @@ var Index = &ice.Context{Name: "input", Help: "输入法", arg[0] = "^" + arg[0] + "," } - // 字段列表 - field := m.Confm("input", "meta.field") - if field == nil { - field = map[string]interface{}{} - head := []string{} - if f, e := os.Open(path.Join(m.Conf("input", "meta.store"), "input.csv")); m.Assert(e) { - defer f.Close() - - bio := csv.NewReader(f) - head, e = bio.Read() - } - for i, k := range head { - field[k] = i - } - m.Conf("input", "meta.field", field) - } - // 搜索词汇 - bio := csv.NewReader(bytes.NewBufferString(m.Cmdx(ice.CLI_SYSTEM, "grep", "-rh", arg[0], m.Conf("input", "meta.store")))) + field := m.Confm("input", "meta.field") + bio := csv.NewReader(bytes.NewBufferString(strings.Replace(m.Cmdx(ice.CLI_SYSTEM, "grep", "-rn", arg[0], m.Conf("input", "meta.store")), ":", ",", -1))) for i := 0; i < kit.Int(kit.Select("100", arg, 2)); i++ { if line, e := bio.Read(); e != nil { break @@ -118,8 +131,9 @@ var Index = &ice.Context{Name: "input", Help: "输入法", "id", line[kit.Int(field["id"])], "weight", line[kit.Int(field["weight"])], ) } + // 输出词汇 + m.Push("file", path.Base(line[kit.Int(field["file"])])) for _, k := range []string{"id", "code", "text", "weight"} { - // 输出词汇 m.Push(k, line[kit.Int(field[k])]) } } diff --git a/type.go b/type.go index da45f608..4437584f 100644 --- a/type.go +++ b/type.go @@ -1043,13 +1043,13 @@ func (m *Message) Grow(key string, chain interface{}, data interface{}) int { meta["count"] = id // 保存数据 - if len(list) >= kit.Int(kit.Select(m.Conf(WEB_CACHE, Meta("limit")), meta["limit"])) { - least := kit.Int(kit.Select(m.Conf(WEB_CACHE, Meta("least")), meta["least"])) + if len(list) >= kit.Int(kit.Select(m.Conf(WEB_CACHE, "meta.limit"), kit.Select(kit.Format(meta["limit"]), m.Option("cache.limit")))) { + least := kit.Int(kit.Select(m.Conf(WEB_CACHE, "meta.least"), kit.Select(kit.Format(meta["least"]), m.Option("cache.least")))) // 创建文件 - name := path.Join(kit.Select(m.Conf(WEB_CACHE, Meta("store")), meta["store"]), kit.Keys(key, chain, "csv")) + name := path.Join(kit.Select(m.Conf(WEB_CACHE, Meta("store")), kit.Select(kit.Format(meta["store"]), m.Option("cache.store"))), kit.Keys(key, chain, "csv")) if s, e := os.Stat(name); e == nil { - if s.Size() > 100000 { + if s.Size() > kit.Int64(kit.Select(m.Conf(WEB_CACHE, "meta.fsize"), kit.Select(kit.Format(meta["fsize"]), m.Option("cache.fsize")))) { name = strings.Replace(name, ".csv", fmt.Sprintf("_%d.csv", kit.Int(meta["offset"])), -1) } } @@ -1084,13 +1084,17 @@ func (m *Message) Grow(key string, chain interface{}, data interface{}) int { count := len(list) - least offset := kit.Int(meta["offset"]) record, _ := meta["record"].([]interface{}) - meta["record"] = append(record, map[string]interface{}{ - "time": m.Time(), - "offset": offset, - "position": s.Size(), - "count": count, - "file": name, - }) + if len(record) > 0 && kit.Format(kit.Value(record, kit.Keys(len(record)-1, "file"))) == name && count < 10 { + kit.Value(record, kit.Keys(len(record)-1, "count"), kit.Int(kit.Value(record, kit.Keys(len(record)-1, "count")))+count) + } else { + meta["record"] = append(record, map[string]interface{}{ + "time": m.Time(), + "offset": offset, + "position": s.Size(), + "count": count, + "file": name, + }) + } // 保存数据 for i, v := range list { @@ -1126,7 +1130,7 @@ func (m *Message) Grows(key string, chain interface{}, match string, value strin } meta, ok := cache[kit.MDB_META].(map[string]interface{}) list, ok := cache[kit.MDB_LIST].([]interface{}) - if !ok || len(list) == 0 { + if !ok { return nil }