diff --git a/base.go b/base.go index 2c7405b2..a0215be8 100644 --- a/base.go +++ b/base.go @@ -2,6 +2,8 @@ package ice import ( "github.com/shylinux/toolkits" + "github.com/shylinux/toolkits/conf" + "github.com/shylinux/toolkits/log" "fmt" "os" @@ -132,6 +134,8 @@ func Run(arg ...string) string { Index.root = Index Pulse.root = Pulse + log.Init(conf.New(nil)) + if len(arg) == 0 { arg = os.Args[1:] } diff --git a/base/nfs/nfs.go b/base/nfs/nfs.go index e921478e..7a84fb1a 100644 --- a/base/nfs/nfs.go +++ b/base/nfs/nfs.go @@ -17,8 +17,12 @@ import ( "strings" ) -func dir(m *ice.Message, root string, name string, level int, deep bool, dir_type string, dir_reg *regexp.Regexp, fields []string, format string) { +const ( + FILE = "file" + TRASH = "trash" +) +func _file_list(m *ice.Message, root string, name string, level int, deep bool, dir_type string, dir_reg *regexp.Regexp, fields []string) { if fs, e := ioutil.ReadDir(path.Join(root, name)); e != nil { if f, e := os.Open(path.Join(root, name)); e == nil { defer f.Close() @@ -49,7 +53,7 @@ func dir(m *ice.Message, root string, name string, level int, deep bool, dir_typ for _, field := range fields { switch field { case "time": - m.Push("time", f.ModTime().Format(format)) + m.Push("time", f.ModTime().Format(ice.ICE_TIME)) case "type": if m.Assert(e) && f.IsDir() { m.Push("type", "dir") @@ -131,42 +135,25 @@ func dir(m *ice.Message, root string, name string, level int, deep bool, dir_typ } } if f.IsDir() && deep { - dir(m, root, p, level+1, deep, dir_type, dir_reg, fields, format) + _file_list(m, root, p, level+1, deep, dir_type, dir_reg, fields) } } } } -func travel(m *ice.Message, root string, name string, cb func(name string)) { - if fs, e := ioutil.ReadDir(path.Join(root, name)); e != nil { - cb(name) - } else { - for _, f := range fs { - if f.Name() == "." || f.Name() == ".." { - continue - } - if strings.HasPrefix(f.Name(), ".") { - continue - } - - p := path.Join(root, name, f.Name()) - if f, e = os.Lstat(p); e != nil { - m.Log("info", "%s", e) - continue - } else if (f.Mode()&os.ModeSymlink) != 0 && f.IsDir() { - continue - } - if f.IsDir() { - travel(m, root, path.Join(name, f.Name()), cb) - cb(path.Join(name, f.Name())) - } else { - cb(path.Join(name, f.Name())) +func _file_show(m *ice.Message, name string) { + if f, e := os.OpenFile(name, os.O_RDONLY, 0777); m.Assert(e) { + defer f.Close() + if s, e := f.Stat(); m.Assert(e) { + buf := make([]byte, s.Size()) + if n, e := f.Read(buf); m.Assert(e) { + m.Log_IMPORT("file", name, "size", n) + m.Echo(string(buf[:n])) } } } } - -func _nfs_save(m *ice.Message, file string, text ...string) { - if f, p, e := kit.Create(file); m.Assert(e) { +func _file_save(m *ice.Message, name string, text ...string) { + if f, p, e := kit.Create(name); m.Assert(e) { defer f.Close() for _, v := range text { if n, e := f.WriteString(v); m.Assert(e) { @@ -176,155 +163,74 @@ func _nfs_save(m *ice.Message, file string, text ...string) { m.Echo(p) } } -func Save(m *ice.Message, file string, text ...string) { - _nfs_save(m, file, text...) +func _file_copy(m *ice.Message, name string, from ...string) { + if f, p, e := kit.Create(name); m.Assert(e) { + defer f.Close() + for _, v := range from { + if s, e := os.Open(v); !m.Warn(e != nil, "%s", e) { + defer s.Close() + if n, e := io.Copy(f, s); !m.Warn(e != nil, "%s", e) { + m.Log_IMPORT("file", p, "size", n) + } + } + } + } +} +func _file_link(m *ice.Message, name string, from string) { + m.Cmd("nfs.trash", name) + os.MkdirAll(path.Dir(name), 0777) + os.Link(from, name) +} +func _file_trash(m *ice.Message, name string, from ...string) { + if s, e := os.Stat(name); e == nil { + if s.IsDir() { + name := path.Base(name) + ".tar.gz" + m.Cmd(ice.CLI_SYSTEM, "tar", "zcf", name, name) + } + + if f, e := os.Open(name); m.Assert(e) { + defer f.Close() + + h := kit.Hashs(f) + p := path.Join(m.Conf("trash", "meta.path"), h[:2], h) + os.MkdirAll(path.Dir(p), 0777) + os.Rename(name, p) + + m.Cmd(ice.WEB_FAVOR, "trash", "bin", name, p) + } + } +} + +func FileSave(m *ice.Message, file string, text ...string) { + _file_save(m, file, text...) } var Index = &ice.Context{Name: "nfs", Help: "存储模块", - Caches: map[string]*ice.Cache{}, Configs: map[string]*ice.Config{ - "trash": {Name: "trash", Help: "trash", Value: kit.Data("path", "var/trash")}, + TRASH: {Name: "trash", Help: "删除", Value: kit.Data("path", "var/trash")}, }, Commands: map[string]*ice.Command{ - ice.ICE_INIT: {Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { - m.Cmd(ice.APP_SEARCH, "add", "dir", "base", m.AddCmd(&ice.Command{Name: "search word", Help: "搜索引擎", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { - switch arg[0] { - case "set": - m.Cmdy("nfs.dir", arg[5]) - return - } - - travel(m, "./", "", func(name string) { - if strings.Contains(name, arg[0]) { - s, e := os.Stat(name) - m.Assert(e) - m.Push("pod", m.Option(ice.MSG_USERPOD)) - m.Push("engine", "dir") - m.Push("favor", "file") - m.Push("id", kit.FmtSize(s.Size())) - m.Push("time", s.ModTime().Format(ice.ICE_TIME)) - m.Push("type", strings.TrimPrefix(path.Ext(name), ".")) - m.Push("name", path.Base(name)) - m.Push("text", name) - } - }) - }})) - m.Cmd(ice.APP_COMMEND, "add", "dir", "base", m.AddCmd(&ice.Command{Name: "commend word", Help: "推荐引擎", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { - switch arg[0] { - case "set": - m.Cmdy("nfs.dir", arg[5]) - return - } - - travel(m, "./", "", func(name string) { - score := 0 - m.Richs(ice.APP_COMMEND, "meta.user", m.Option(ice.MSG_USERNAME), func(key string, value map[string]interface{}) { - m.Grows(ice.APP_COMMEND, kit.Keys("meta.user", kit.MDB_HASH, key, "like"), "", "", func(index int, value map[string]interface{}) { - switch kit.Value(value, "extra.engine") { - case "dir": - if value["type"] == strings.TrimPrefix(path.Ext(name), ".") { - score += 1 - } - if value["name"] == path.Base(name) { - score += 2 - } - if value["text"] == name { - score += 3 - } - default: - } - }) - m.Grows(cmd, kit.Keys("meta.user", kit.MDB_HASH, key, "hate"), "", "", func(index int, value map[string]interface{}) { - switch kit.Value(value, "extra.engine") { - case "dir": - if value["type"] == strings.TrimPrefix(path.Ext(name), ".") { - score -= 1 - } - if value["name"] == path.Base(name) { - score -= 2 - } - if value["text"] == name { - score -= 3 - } - default: - } - }) - }) - - if s, e := os.Stat(name); e == nil { - m.Push("pod", m.Option(ice.MSG_USERPOD)) - m.Push("engine", "dir") - m.Push("favor", "file") - m.Push("id", kit.FmtSize(s.Size())) - m.Push("score", score) - m.Push("type", strings.TrimPrefix(path.Ext(name), ".")) - m.Push("name", path.Base(name)) - m.Push("text", name) - } - }) - }})) - }}, - - "dir": {Name: "dir name field auto", Help: "目录", List: kit.List( - kit.MDB_INPUT, "text", "name", "path", "action", "auto", - kit.MDB_INPUT, "button", "name", "查看", - kit.MDB_INPUT, "button", "name", "返回", "cb", "Last", - ), Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { + "dir": {Name: "dir path field...", Help: "目录", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { rg, _ := regexp.Compile(m.Option("dir_reg")) - dir(m, kit.Select("./", m.Option("dir_root")), kit.Select("", arg, 0), 0, m.Options("dir_deep"), kit.Select("both", m.Option("dir_type")), rg, - strings.Split(kit.Select("time size line path", arg, 1), " "), ice.ICE_TIME) + _file_list(m, kit.Select("./", m.Option("dir_root")), kit.Select("", arg, 0), + 0, m.Options("dir_deep"), kit.Select("both", m.Option("dir_type")), rg, + strings.Split(kit.Select("time size line path", strings.Join(arg[1:], " ")), " ")) }}, - "cat": {Name: "cat path", Help: "保存", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { - if f, e := os.OpenFile(arg[0], os.O_RDONLY, 0777); m.Assert(e) { - defer f.Close() - buf := make([]byte, 4096000) - if n, e := f.Read(buf); m.Assert(e) { - m.Log(ice.LOG_IMPORT, "%d: %s", n, arg[0]) - m.Echo(string(buf[:n])) - } - } + "cat": {Name: "cat file", Help: "查看", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { + _file_show(m, arg[0]) + }}, + "save": {Name: "save file text...", Help: "保存", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { + _file_save(m, arg[0], arg[1:]...) + }}, + "copy": {Name: "copy file from...", Help: "复制", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { + _file_copy(m, arg[0], arg[1:]...) + }}, + "link": {Name: "link file from", Help: "链接", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { + _file_link(m, arg[0], arg[1]) }}, - "save": {Name: "save path text...", Help: "保存", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { - _nfs_save(m, arg[0], arg[1:]...) - }}, - "copy": {Name: "copy path file...", Help: "保存", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { - if f, _, e := kit.Create(arg[0]); m.Assert(e) { - defer f.Close() - for _, v := range arg[1:] { - if s, e := os.Open(v); !m.Warn(e != nil, "%s", e) { - if n, e := io.Copy(f, s); m.Assert(e) { - m.Log(ice.LOG_IMPORT, "%d: %v", n, v) - } - } - } - } - }}, - "link": {Name: "link path file", Help: "链接", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { - m.Cmd("nfs.trash", arg[0]) - os.MkdirAll(path.Dir(arg[0]), 0777) - os.Link(arg[1], arg[0]) - }}, - - "trash": {Name: "trash file", Help: "保存", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { - if s, e := os.Stat(arg[0]); e == nil { - if s.IsDir() { - name := path.Base(arg[0]) + ".tar.gz" - m.Cmd(ice.CLI_SYSTEM, "tar", "zcf", name, arg[0]) - } else { - } - - if f, e := os.Open(arg[0]); m.Assert(e) { - defer f.Close() - - h := kit.Hashs(f) - p := path.Join(m.Conf("trash", "meta.path"), h[:2], h) - os.MkdirAll(path.Dir(p), 0777) - os.Rename(arg[0], p) - - m.Cmd(ice.WEB_FAVOR, "trash", "bin", arg[0], p) - } - } + TRASH: {Name: "trash file", Help: "删除", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { + _file_trash(m, arg[0]) }}, }, } diff --git a/base/nfs/search.go b/base/nfs/search.go new file mode 100644 index 00000000..f69e0aef --- /dev/null +++ b/base/nfs/search.go @@ -0,0 +1,125 @@ +package nfs + +import ( + ice "github.com/shylinux/icebergs" + kit "github.com/shylinux/toolkits" + + "io/ioutil" + "os" + "path" + "strings" +) + +func travel(m *ice.Message, root string, name string, cb func(name string)) { + if fs, e := ioutil.ReadDir(path.Join(root, name)); e != nil { + cb(name) + } else { + for _, f := range fs { + if f.Name() == "." || f.Name() == ".." { + continue + } + if strings.HasPrefix(f.Name(), ".") { + continue + } + + p := path.Join(root, name, f.Name()) + if f, e = os.Lstat(p); e != nil { + m.Log("info", "%s", e) + continue + } else if (f.Mode()&os.ModeSymlink) != 0 && f.IsDir() { + continue + } + if f.IsDir() { + travel(m, root, path.Join(name, f.Name()), cb) + cb(path.Join(name, f.Name())) + } else { + cb(path.Join(name, f.Name())) + } + } + } +} + +func init() { + Index.Register(&ice.Context{Name: "search", Help: "搜索", + Commands: map[string]*ice.Command{ + ice.ICE_INIT: {Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { + m.Cmd(ice.APP_SEARCH, "add", "dir", "base", m.AddCmd(&ice.Command{Name: "search word", Help: "搜索引擎", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { + switch arg[0] { + case "set": + m.Cmdy("nfs.dir", arg[5]) + return + } + + travel(m, "./", "", func(name string) { + if strings.Contains(name, arg[0]) { + s, e := os.Stat(name) + m.Assert(e) + m.Push("pod", m.Option(ice.MSG_USERPOD)) + m.Push("engine", "dir") + m.Push("favor", "file") + m.Push("id", kit.FmtSize(s.Size())) + m.Push("time", s.ModTime().Format(ice.ICE_TIME)) + m.Push("type", strings.TrimPrefix(path.Ext(name), ".")) + m.Push("name", path.Base(name)) + m.Push("text", name) + } + }) + }})) + m.Cmd(ice.APP_COMMEND, "add", "dir", "base", m.AddCmd(&ice.Command{Name: "commend word", Help: "推荐引擎", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { + switch arg[0] { + case "set": + m.Cmdy("nfs.dir", arg[5]) + return + } + + travel(m, "./", "", func(name string) { + score := 0 + m.Richs(ice.APP_COMMEND, "meta.user", m.Option(ice.MSG_USERNAME), func(key string, value map[string]interface{}) { + m.Grows(ice.APP_COMMEND, kit.Keys("meta.user", kit.MDB_HASH, key, "like"), "", "", func(index int, value map[string]interface{}) { + switch kit.Value(value, "extra.engine") { + case "dir": + if value["type"] == strings.TrimPrefix(path.Ext(name), ".") { + score += 1 + } + if value["name"] == path.Base(name) { + score += 2 + } + if value["text"] == name { + score += 3 + } + default: + } + }) + m.Grows(cmd, kit.Keys("meta.user", kit.MDB_HASH, key, "hate"), "", "", func(index int, value map[string]interface{}) { + switch kit.Value(value, "extra.engine") { + case "dir": + if value["type"] == strings.TrimPrefix(path.Ext(name), ".") { + score -= 1 + } + if value["name"] == path.Base(name) { + score -= 2 + } + if value["text"] == name { + score -= 3 + } + default: + } + }) + }) + + if s, e := os.Stat(name); e == nil { + m.Push("pod", m.Option(ice.MSG_USERPOD)) + m.Push("engine", "dir") + m.Push("favor", "file") + m.Push("id", kit.FmtSize(s.Size())) + m.Push("score", score) + m.Push("type", strings.TrimPrefix(path.Ext(name), ".")) + m.Push("name", path.Base(name)) + m.Push("text", name) + } + }) + }})) + }}, + }, + }, nil) +} diff --git a/core/code/bench.go b/core/code/bench.go index 519853b6..a67702f3 100644 --- a/core/code/bench.go +++ b/core/code/bench.go @@ -3,46 +3,86 @@ package code import ( "github.com/shylinux/icebergs" "github.com/shylinux/toolkits" + "github.com/shylinux/toolkits/util/bench" + + "io" + "net/http" + "os" + "strings" + "sync/atomic" ) -type Bench struct { - *ice.Message +const ( + BENCH = "bench" +) + +func _bench_list(m *ice.Message, zone string, id string, field ...interface{}) { + m.RichList(BENCH, zone, id, field) } -func (b *Bench) Get(key string) string { - return b.Conf("hi", key) +func _bench_show(m *ice.Message, nconn, nreq int64, list []*http.Request) { + nout, e := os.OpenFile("/dev/null", os.O_WRONLY, 0777) + m.Assert(e) + + var body int64 + s, e := bench.HTTP(nconn, nreq, list, func(req *http.Request, res *http.Response) { + n, _ := io.Copy(nout, res.Body) + atomic.AddInt64(&body, n) + }) + m.Assert(e) + + m.Echo(s.Show()) + m.Echo("body: %d\n", body) } -func (b *Bench) GetInt(key string) int64 { - return kit.Int64(b.Conf("hi", key)) +func _bench_create(m *ice.Message) { +} +func _bench_insert(m *ice.Message) { } -func (b *Bench) Log(key string, arg ...interface{}) { - b.Message.Info("%s: %v", key, kit.Simple(key)) -} -func (b *Bench) Logf(key string, str string, arg ...interface{}) { - b.Message.Info("%s: %v", key, kit.Format(str, arg...)) -} -func (b *Bench) Info(arg ...interface{}) { - b.Log("info", arg...) -} -func (b *Bench) Infof(str string, arg ...interface{}) { - b.Logf("info", str, arg...) -} -func (b *Bench) Warn(arg ...interface{}) { - b.Log("warn", arg...) -} -func (b *Bench) Warnf(str string, arg ...interface{}) { - b.Logf("warn", str, arg...) -} -func (b *Bench) Error(arg ...interface{}) { - b.Log("error", arg...) -} -func (b *Bench) Errorf(str string, arg ...interface{}) { - b.Logf("error", str, arg...) -} -func (b *Bench) Debug(arg ...interface{}) { - b.Log("debug", arg...) -} -func (b *Bench) Debugf(str string, arg ...interface{}) { - b.Logf("debug", str, arg...) +func init() { + Index.Merge(&ice.Context{ + Configs: map[string]*ice.Config{ + BENCH: {Name: "bench", Help: "性能压测", Value: kit.Data( + kit.MDB_SHORT, kit.MDB_ZONE, + )}, + }, + Commands: map[string]*ice.Command{ + BENCH: {Name: "bench list nconn nreq", Help: "性能压测", Action: map[string]*ice.Action{ + "show": {Name: "show", Help: "运行", Hand: func(m *ice.Message, arg ...string) { + list := []*http.Request{} + target := kit.Select("http://localhost:9020/", arg, 0) + for _, v := range strings.Split(target, ",") { + switch ls := kit.Split(v); ls[0] { + case http.MethodPost: + if f, e := os.Open(ls[2]); m.Assert(e) { + defer f.Close() + + req, err := http.NewRequest(http.MethodPost, ls[1], f) + m.Assert(err) + list = append(list, req) + } + default: + req, err := http.NewRequest(http.MethodGet, v, nil) + m.Assert(err) + list = append(list, req) + } + } + m.Echo("%s\n", target) + _bench_show(m, kit.Int64(kit.Select("3", arg, 1)), kit.Int64(kit.Select("100", arg, 2)), list) + }}, + "create": {Name: "create", Help: "创建", Hand: func(m *ice.Message, arg ...string) { + _bench_create(m) + + }}, + "insert": {Name: "insert", Help: "插入", Hand: func(m *ice.Message, arg ...string) { + _bench_insert(m) + }}, + }, Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { + _bench_list(m, BENCH, kit.Select("", arg, 0), kit.Select("", arg, 1)) + }}, + "/bench": {Name: "/bench cmd...", Help: "性能压测", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { + m.Cmdy(m.Optionv("cmd")) + }}, + }, + }, nil) } diff --git a/core/code/code.go b/core/code/code.go index 95fddb31..df649a3c 100644 --- a/core/code/code.go +++ b/core/code/code.go @@ -9,13 +9,9 @@ import ( "path" "runtime" "strings" - - "net/http" - _ "net/http/pprof" ) var Index = &ice.Context{Name: "code", Help: "编程中心", - Caches: map[string]*ice.Cache{}, Configs: map[string]*ice.Config{ "install": {Name: "install", Help: "安装", Value: kit.Data("path", "usr/install", "linux", "https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz", @@ -40,9 +36,6 @@ var Index = &ice.Context{Name: "code", Help: "编程中心", ))}, "login": {Name: "login", Help: "终端接入", Value: kit.Data()}, - "pprof": {Name: "pprof", Help: "性能分析", Value: kit.Data(kit.MDB_SHORT, kit.MDB_NAME, - "stop", "ps aux|grep pprof|grep -v grep|cut -d' ' -f2|xargs -n1 kill", - )}, }, Commands: map[string]*ice.Command{ ice.ICE_INIT: {Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { @@ -252,60 +245,6 @@ export %s }) } }}, - "pprof": {Name: "pprof run name time", Help: "性能分析", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { - if m.Show(cmd, arg...) { - return - } - - switch arg[0] { - case "run": - m.Richs(cmd, nil, arg[1], func(key string, value map[string]interface{}) { - m.Gos(m.Spawn(), func(msg *ice.Message) { - m.Sleep("1s").Grows(cmd, kit.Keys(kit.MDB_HASH, key), "", "", func(index int, value map[string]interface{}) { - // 压测命令 - m.Cmd(ice.WEB_FAVOR, "pprof", "shell", value[kit.MDB_TEXT], m.Cmdx(kit.Split(kit.Format(value[kit.MDB_TEXT])))) - }) - }) - - // 启动监控 - name := arg[1] + ".pd.gz" - value = value["meta"].(map[string]interface{}) - msg := m.Cmd(ice.WEB_SPIDE, "self", "cache", "GET", kit.Select("/code/pprof/profile", value["remote"]), "seconds", kit.Select("5", arg, 2)) - m.Cmd(ice.WEB_FAVOR, "pprof", "shell", "text", m.Cmdx(ice.CLI_SYSTEM, "go", "tool", "pprof", "-text", msg.Append("text"))) - m.Cmd(ice.WEB_FAVOR, "pprof", "pprof", name, msg.Append("data")) - - arg = kit.Simple("web", value[kit.MDB_TEXT], msg.Append("text")) - }) - - fallthrough - case "web": - // 展示结果 - p := kit.Format("%s:%s", m.Conf(ice.WEB_SHARE, "meta.host"), m.Cmdx("tcp.getport")) - m.Cmd(ice.CLI_DAEMON, "go", "tool", "pprof", "-http="+p, arg[1:]) - m.Cmd(ice.WEB_FAVOR, "pprof", "bin", arg[1], m.Cmd(ice.WEB_CACHE, "catch", "bin", arg[1]).Append("data")) - m.Cmd(ice.WEB_FAVOR, "pprof", "spide", arg[2], "http://"+p) - m.Echo(p) - - case "stop": - m.Cmd(ice.CLI_SYSTEM, "sh", "-c", m.Conf(cmd, "meta.stop")) - - case "add": - key := m.Rich(cmd, nil, kit.Data( - kit.MDB_NAME, arg[1], kit.MDB_TEXT, arg[2], "remote", arg[3], - )) - - for i := 4; i < len(arg)-1; i += 2 { - m.Grow(cmd, kit.Keys(kit.MDB_HASH, key), kit.Dict( - kit.MDB_NAME, arg[i], kit.MDB_TEXT, arg[i+1], - )) - } - } - }}, - "/pprof/": {Name: "/pprof/", Help: "性能分析", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { - m.R.URL.Path = strings.Replace("/code"+m.R.URL.Path, "code", "debug", 1) - http.DefaultServeMux.ServeHTTP(m.W, m.R) - m.Render(ice.RENDER_VOID) - }}, }, } diff --git a/core/code/pprof.go b/core/code/pprof.go new file mode 100644 index 00000000..65a1b6db --- /dev/null +++ b/core/code/pprof.go @@ -0,0 +1,80 @@ +package code + +import ( + "github.com/shylinux/icebergs" + "github.com/shylinux/toolkits" + + "net/http" + _ "net/http/pprof" + "strings" +) + +const ( + PPROF = "pprof" +) + +func init() { + Index.Merge(&ice.Context{ + Configs: map[string]*ice.Config{ + "pprof": {Name: "pprof", Help: "性能分析", Value: kit.Data(kit.MDB_SHORT, kit.MDB_NAME, + "stop", "ps aux|grep pprof|grep -v grep|cut -d' ' -f2|xargs -n1 kill", + )}, + }, + Commands: map[string]*ice.Command{ + "pprof": {Name: "pprof run name time", Help: "性能分析", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { + if m.Show(cmd, arg...) { + return + } + + switch arg[0] { + case "run": + m.Richs(cmd, nil, arg[1], func(key string, value map[string]interface{}) { + m.Gos(m.Spawn(), func(msg *ice.Message) { + m.Sleep("1s").Grows(cmd, kit.Keys(kit.MDB_HASH, key), "", "", func(index int, value map[string]interface{}) { + // 压测命令 + m.Cmd(ice.WEB_FAVOR, "pprof", "shell", value[kit.MDB_TEXT], m.Cmdx(kit.Split(kit.Format(value[kit.MDB_TEXT])))) + }) + }) + + // 启动监控 + name := arg[1] + ".pd.gz" + value = value["meta"].(map[string]interface{}) + msg := m.Cmd(ice.WEB_SPIDE, "self", "cache", "GET", kit.Select("/code/pprof/profile", value["remote"]), "seconds", kit.Select("5", arg, 2)) + m.Cmd(ice.WEB_FAVOR, "pprof", "shell", "text", m.Cmdx(ice.CLI_SYSTEM, "go", "tool", "pprof", "-text", msg.Append("text"))) + m.Cmd(ice.WEB_FAVOR, "pprof", "pprof", name, msg.Append("data")) + + arg = kit.Simple("web", value[kit.MDB_TEXT], msg.Append("text")) + }) + + fallthrough + case "web": + // 展示结果 + p := kit.Format("%s:%s", m.Conf(ice.WEB_SHARE, "meta.host"), m.Cmdx("tcp.getport")) + m.Cmd(ice.CLI_DAEMON, "go", "tool", "pprof", "-http="+p, arg[1:]) + m.Cmd(ice.WEB_FAVOR, "pprof", "bin", arg[1], m.Cmd(ice.WEB_CACHE, "catch", "bin", arg[1]).Append("data")) + m.Cmd(ice.WEB_FAVOR, "pprof", "spide", arg[2], "http://"+p) + m.Echo(p) + + case "stop": + m.Cmd(ice.CLI_SYSTEM, "sh", "-c", m.Conf(cmd, "meta.stop")) + + case "add": + key := m.Rich(cmd, nil, kit.Data( + kit.MDB_NAME, arg[1], kit.MDB_TEXT, arg[2], "remote", arg[3], + )) + + for i := 4; i < len(arg)-1; i += 2 { + m.Grow(cmd, kit.Keys(kit.MDB_HASH, key), kit.Dict( + kit.MDB_NAME, arg[i], kit.MDB_TEXT, arg[i+1], + )) + } + } + }}, + "/pprof/": {Name: "/pprof/", Help: "性能分析", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { + m.R.URL.Path = strings.Replace("/code"+m.R.URL.Path, "code", "debug", 1) + http.DefaultServeMux.ServeHTTP(m.W, m.R) + m.Render(ice.RENDER_VOID) + }}, + }, + }, nil) +} diff --git a/core/code/qrcode.go b/core/code/qrcode.go index bda8e830..bfde43c8 100644 --- a/core/code/qrcode.go +++ b/core/code/qrcode.go @@ -28,7 +28,7 @@ func init() { m.Echo("hello world") }}, "save": {Name: "save name text", Help: "保存", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { - if qr, e := qrs.New(arg[1], qrs.Medium); m.Assert(e) { + if qr, e := qrs.New(kit.Select(m.Option("content", arg, 1)), qrs.Medium); m.Assert(e) { if f, e := os.Create(arg[0]); m.Assert(e) { defer f.Close() m.Assert(qr.Write(kit.Int(kit.Select("256", arg, 2)), f)) diff --git a/core/wiki/draw.go b/core/wiki/draw.go index 81345212..c4aaea4b 100644 --- a/core/wiki/draw.go +++ b/core/wiki/draw.go @@ -36,7 +36,7 @@ func init() { "display", "/plugin/local/wiki/draw.js", ), Action: map[string]*ice.Action{ "save": {Name: "save path text", Help: "保存", Hand: func(m *ice.Message, arg ...string) { - nfs.Save(m, path.Join(m.Conf(DRAW, "meta.path"), kit.Select("hi.svg", arg, 0)), kit.Select(m.Option("content"), arg, 1)) + nfs.FileSave(m, path.Join(m.Conf(DRAW, "meta.path"), kit.Select("hi.svg", arg, 0)), kit.Select(m.Option("content"), arg, 1)) }}, "run": {Name: "show path text", Help: "运行", Hand: func(m *ice.Message, arg ...string) { _draw_show(m, arg[0], arg[1], arg[2], arg[3], arg[4:]...) diff --git a/data.go b/data.go index 85a9f200..659cb3f0 100644 --- a/data.go +++ b/data.go @@ -14,6 +14,10 @@ import ( "strings" ) +const ( + ZONE = "zone" +) + func (m *Message) Prefile(favor string, id string) map[string]string { res := map[string]string{} m.Option("render", "") @@ -460,3 +464,24 @@ func (m *Message) Show(cmd string, arg ...string) bool { } return false } + +func (m *Message) RichList(prefix string, zone string, id string, field ...interface{}) { + m.Richs(prefix, nil, kit.Select(kit.MDB_FOREACH, zone), func(key string, val map[string]interface{}) { + if zone = kit.Format(kit.Value(val, "meta.zone")); id == "" { + m.Grows(prefix, kit.Keys(kit.MDB_HASH, key), "", "", func(index int, value map[string]interface{}) { + m.Push(zone, value) + m.Push(kit.MDB_ZONE, zone) + }) + return + } + m.Grows(prefix, kit.Keys(kit.MDB_HASH, key), kit.MDB_ID, id, func(index int, value map[string]interface{}) { + m.Push("detail", value) + }) + }) +} +func (m *Message) RichCreate(prefix string, zone string, arg ...string) { + if m.Richs(prefix, nil, zone, nil) == nil { + m.Rich(prefix, nil, kit.Data(kit.MDB_ZONE, zone, arg)) + m.Log_CREATE(kit.MDB_ZONE, zone) + } +} diff --git a/info.go b/info.go index 47e6d2b0..5900f52f 100644 --- a/info.go +++ b/info.go @@ -2,6 +2,7 @@ package ice import ( kit "github.com/shylinux/toolkits" + "github.com/shylinux/toolkits/log" "fmt" "os" @@ -45,10 +46,11 @@ func (m *Message) log(level string, str string, arg ...interface{}) *Message { if os.Getenv("ctx_mod") != "" && m != nil { // 输出日志 - fmt.Fprintf(os.Stderr, "%s %02d %9s %s%s %s%s\n", + str := fmt.Sprintf("%s %02d %9s %s%s %s%s", m.time.Format(ICE_TIME), m.code, fmt.Sprintf("%4s->%-4s", m.source.Name, m.target.Name), prefix, level, str, suffix, ) + log.Info(str) } return m } diff --git a/misc/input/input.go b/misc/input/input.go index 7d2df4ae..653ad9e8 100644 --- a/misc/input/input.go +++ b/misc/input/input.go @@ -2,8 +2,11 @@ package input import ( "github.com/shylinux/icebergs" + "github.com/shylinux/icebergs/base/web" "github.com/shylinux/icebergs/core/code" "github.com/shylinux/toolkits" + "github.com/shylinux/toolkits/conf" + "github.com/shylinux/toolkits/task" "bufio" "bytes" @@ -14,147 +17,234 @@ import ( "strings" ) +const ( + INPUT = "input" +) +const ( + LINE = "line" + WORD = "word" +) +const ( + FILE = "file" + CODE = "code" + TEXT = "text" + WEIGHT = "weight" +) + +func _input_list(m *ice.Message, lib string) { + if lib == "" { + m.Richs(INPUT, "", kit.MDB_FOREACH, func(key string, value map[string]interface{}) { + m.Push(key, value[kit.MDB_META], []string{kit.MDB_ZONE, kit.MDB_COUNT, kit.MDB_STORE}) + }) + return + } + + m.Richs(INPUT, "", lib, func(key string, value map[string]interface{}) { + m.Grows(INPUT, kit.Keys(kit.MDB_HASH, key), "", "", func(index int, value map[string]interface{}) { + m.Push(key, value, []string{kit.MDB_ID, CODE, TEXT, WEIGHT}) + }) + }) +} +func _input_push(m *ice.Message, lib, text, code, weight string) { + m.Richs(INPUT, "", lib, func(key string, value map[string]interface{}) { + prefix := kit.Keys(kit.MDB_HASH, key) + m.Conf(INPUT, kit.Keys(prefix, "meta.limit"), 0) + m.Conf(INPUT, kit.Keys(prefix, "meta.least"), 0) + n := m.Grow(INPUT, prefix, kit.Dict(TEXT, text, CODE, code, WEIGHT, weight)) + m.Log_IMPORT(CODE, code, TEXT, text) + m.Echo("%s: %d", lib, n) + }) +} +func _input_find(m *ice.Message, method, word, limit string) { + // 搜索方法 + switch method { + case LINE: + case WORD: + word = "^" + word + "," + } + + // 搜索词汇 + res := m.Cmdx(ice.CLI_SYSTEM, "grep", "-rn", word, m.Conf(INPUT, "meta.store")) + bio := csv.NewReader(bytes.NewBufferString(strings.Replace(res, ":", ",", -1))) + + for i := 0; i < kit.Int(limit); i++ { + if line, e := bio.Read(); e != nil { + break + } else if len(line) < 3 { + + } else { + if method == WORD && i == 0 { + // 添加收藏 + web.FavorInsert(m, "input.word", "input", line[2], line[4], "id", line[3], WEIGHT, line[6]) + } + + // 输出词汇 + m.Push(FILE, path.Base(line[0])) + m.Push(kit.MDB_ID, line[3]) + m.Push(CODE, line[2]) + m.Push(TEXT, line[4]) + m.Push(WEIGHT, line[6]) + } + } + m.Sort(WEIGHT, "int_r") +} +func _input_find2(m *ice.Message, method, word, limit string) { + files := map[string]bool{} + list := []interface{}{} + m.Richs(INPUT, "", kit.MDB_FOREACH, func(key string, value map[string]interface{}) { + kit.Fetch(kit.Value(value, "meta.record"), func(index int, value map[string]interface{}) { + file := value["file"].(string) + if _, ok := files[file]; ok { + list = append(list, file) + } else { + files[file] = true + } + }) + }) + defer m.Cost("some") + + p.Sync(list, func(task *task.Task) error { + file := task.Arg.(string) + f, e := os.Open(file) + if e != nil { + return e + } + defer f.Close() + + r := csv.NewReader(f) + head, e := r.Read() + if e != nil { + return e + } + for { + line, e := r.Read() + if e != nil { + break + } + if head[0] == "code" { + if line[0] == word { + m.Push(FILE, file) + m.Push(kit.MDB_ID, line[1]) + m.Push(CODE, line[0]) + m.Push(TEXT, line[2]) + m.Push(kit.MDB_TIME, line[3]) + m.Push(WEIGHT, line[4]) + } + } + } + return nil + }) +} +func _input_save(m *ice.Message, file string, lib ...string) { + if f, p, e := kit.Create(file); m.Assert(e) { + defer f.Close() + n := 0 + m.Option("cache.limit", -2) + for _, lib := range lib { + m.Richs(INPUT, "", lib, func(key string, value map[string]interface{}) { + m.Grows(INPUT, kit.Keys(kit.MDB_HASH, key), "", "", func(index int, value map[string]interface{}) { + if value[CODE] != "z" { + fmt.Fprintf(f, "%s %s %s\n", value[TEXT], value[CODE], value[WEIGHT]) + n++ + } + }) + }) + } + m.Log_EXPORT(FILE, p, kit.MDB_COUNT, n) + m.Echo("%s: %d", p, n) + } +} +func _input_load(m *ice.Message, file string, libs ...string) { + if f, e := os.Open(file); m.Assert(e) { + defer f.Close() + + // 清空数据 + lib := kit.Select(path.Base(file), libs, 0) + m.Assert(os.RemoveAll(path.Join(m.Conf(INPUT, "meta.store"), lib))) + prefix := kit.Keys(kit.MDB_HASH, m.Rich(INPUT, "", kit.Data( + kit.MDB_STORE, path.Join(m.Conf(INPUT, "meta.store"), lib), + kit.MDB_FSIZE, m.Conf(INPUT, "meta.fsize"), + kit.MDB_LIMIT, m.Conf(INPUT, "meta.limit"), + kit.MDB_LEAST, m.Conf(INPUT, "meta.least"), + "zone", lib, + ))) + + // 缓存配置 + + // 加载词库 + for bio := bufio.NewScanner(f); bio.Scan(); { + if strings.HasPrefix(bio.Text(), "#") { + continue + } + line := kit.Split(bio.Text()) + if len(line) < 3 || line[2] == "0" { + continue + } + m.Grow(INPUT, prefix, kit.Dict(TEXT, line[0], CODE, line[1], WEIGHT, line[2])) + } + + // 保存词库 + m.Conf(INPUT, kit.Keys(prefix, "meta.limit"), 0) + m.Conf(INPUT, kit.Keys(prefix, "meta.least"), 0) + n := m.Grow(INPUT, prefix, kit.Dict(TEXT, "成功", CODE, "z", WEIGHT, "0")) + m.Log_IMPORT(INPUT, lib, kit.MDB_COUNT, n) + m.Echo("%s: %d", lib, n) + } +} + +var p = task.New(nil, 10) + var Index = &ice.Context{Name: "input", Help: "输入法", - Caches: map[string]*ice.Cache{}, Configs: map[string]*ice.Config{ - "input": {Name: "input", Help: "输入法", Value: kit.Data( - "store", "var/data/input", "fsize", "200000", "limit", "5000", "least", "1000", - "repos", "wubi-dict", "local", "some", + INPUT: {Name: "input", Help: "输入法", Value: kit.Data( + "repos", "wubi-dict", "local", "person", + kit.MDB_STORE, "var/data/input", kit.MDB_FSIZE, "200000", + kit.MDB_LIMIT, "5000", kit.MDB_LEAST, "1000", + kit.MDB_SHORT, "zone", )}, }, Commands: map[string]*ice.Command{ - ice.ICE_INIT: {Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { - m.Load() - }}, - ice.ICE_EXIT: {Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { - m.Save("input") - }}, + ice.ICE_INIT: {Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { m.Load() }}, + ice.ICE_EXIT: {Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { m.Save(INPUT) }}, - "load": {Name: "load file [name]", Help: "加载词库", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { + "list": {Name: "list [lib]", Help: "查看词库", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { + _input_list(m, kit.Select("", arg, 0)) + }}, + "push": {Name: "push lib text code [weight]", Help: "添加字码", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { + _input_push(m, arg[0], arg[1], arg[2], kit.Select("90919495", arg, 3)) + }}, + "find": {Name: "find key [word|line [limit]]", Help: "查找字码", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { if len(arg) == 0 { - // 下载词库 - if m.Cmd("web.code.git.repos", m.Conf("input", "meta.repos"), "usr/"+m.Conf("input", "meta.repos")); m.Confs("input", "wubi86") { - m.Echo("wubi86: %v", m.Conf("input", "wubi86.meta.count")) - return - } - arg = append(arg, path.Join("usr", m.Conf("input", "meta.repos"), "wubi86")) - } - - if f, e := os.Open(arg[0]); m.Assert(e) { - bio := bufio.NewScanner(f) - - // 清空数据 - lib := kit.Select(path.Base(arg[0]), arg, 1) - m.Assert(os.RemoveAll(path.Join(m.Conf("input", "meta.store"), lib))) - m.Conf("input", lib, "") - - // 缓存配置 - m.Conf("input", kit.Keys(lib, "meta.store"), path.Join(m.Conf("input", "meta.store"), lib)) - m.Conf("input", kit.Keys(lib, "meta.fsize"), m.Conf("input", "meta.fsize")) - m.Conf("input", kit.Keys(lib, "meta.limit"), m.Conf("input", "meta.limit")) - m.Conf("input", kit.Keys(lib, "meta.least"), m.Conf("input", "meta.least")) - - // 加载词库 - for bio.Scan() { - if strings.HasPrefix(bio.Text(), "#") { - continue - } - line := kit.Split(bio.Text()) - if line[2] == "0" { - continue - } - m.Grow("input", lib, kit.Dict("text", line[0], "code", line[1], "weight", line[2])) - } - // 保存词库 - m.Conf("input", kit.Keys(lib, "meta.limit"), 0) - m.Conf("input", kit.Keys(lib, "meta.least"), 0) - m.Echo("%s: %d", lib, m.Grow("input", lib, kit.Dict("text", "成功", "code", "z", "weight", "0"))) - } - }}, - "push": {Name: "push lib text code [weight]", Help: "添加词汇", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { - m.Conf("input", kit.Keys(arg[0], "meta.limit"), 0) - m.Conf("input", kit.Keys(arg[0], "meta.least"), 0) - m.Echo("%s: %d", arg[0], m.Grow("input", arg[0], kit.Dict( - "text", arg[1], "code", arg[2], "weight", kit.Select("90919495", arg, 3)))) - }}, - "list": {Name: "list [lib [offend [limit]]]", Help: "查看词汇", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { - if len(arg) == 0 { - kit.Fetch(m.Confv("input"), func(key string, value map[string]interface{}) { - if key != "meta" { - m.Push(key, value["meta"], []string{"key", "count"}) - } - }) + web.FavorList(m, "input.word", "") return } - - lib := kit.Select("person", arg, 0) - m.Option("cache.offend", kit.Select("0", arg, 1)) - m.Option("cache.limit", kit.Select("10", arg, 2)) - m.Grows("input", lib, "", "", func(index int, value map[string]interface{}) { - m.Push("", value, []string{"id", "code", "text", "weight"}) - }) + _input_find(m, kit.Select(WORD, arg, 1), arg[0], kit.Select("100", arg, 2)) }}, - "save": {Name: "save lib [filename]", Help: "导出词库", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { - lib := kit.Select("person", arg, 0) - if f, p, e := kit.Create(kit.Select(path.Join("usr", m.Conf("input", "meta.repos"), lib), arg, 1)); m.Assert(e) { - defer f.Close() - - n := 0 - m.Option("cache.offend", 0) - m.Option("cache.limit", -2) - m.Grows("input", lib, "", "", func(index int, value map[string]interface{}) { - if value["code"] != "z" { - n++ - fmt.Fprintf(f, "%s %s %s\n", value["text"], value["code"], value["weight"]) - } - }) - m.Log(ice.LOG_EXPORT, "%s: %d", p, n) - m.Echo("%s: %d", p, n) - } - }}, - - "find": {Name: "find key [word|line [limit]]", Help: "五笔字码", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { + "find2": {Name: "find2 key [word|line [limit]]", Help: "查找字码", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { if len(arg) == 0 { - // 收藏列表 - m.Cmdy(ice.WEB_FAVOR, "input.word") + web.FavorList(m, "input.word", "") return } + _input_find2(m, kit.Select(WORD, arg, 1), arg[0], kit.Select("100", arg, 2)) + }}, - // 搜索方法 - method := "word" - if len(arg) > 1 { - method = arg[1] - } - switch method { - case "line": - case "word": - arg[0] = "^" + arg[0] + "," - } + "save": {Name: "save file lib...", Help: "导出词库", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { + _input_save(m, arg[0], arg[1:]...) + }}, + "load": {Name: "load file lib", Help: "导入词库", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { + _input_load(m, arg[0]) + }}, - // 搜索词汇 - bio := csv.NewReader(bytes.NewBufferString(strings.Replace(m.Cmdx(ice.CLI_SYSTEM, "grep", "-rn", arg[0], m.Conf("input", "meta.store")), ":", ",", -1))) - for i := 0; i < kit.Int(kit.Select("100", arg, 2)); i++ { - if line, e := bio.Read(); e != nil { - break - } else if len(line) < 3 { - - } else { - if method == "word" && i == 0 { - // 添加收藏 - m.Cmd(ice.WEB_FAVOR, "input.word", "input", line[2], line[4], "id", line[3], "weight", line[6]) - } - - // 输出词汇 - m.Push("file", path.Base(line[0])) - m.Push("id", line[3]) - m.Push("code", line[2]) - m.Push("text", line[4]) - m.Push("weight", line[6]) - } - } - m.Sort("weight", "int_r") + "compare": {Name: "demo list nconn nreq", Help: "导入词库", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { + m.Cmdy("web.code.bench", "http://localhost:9020/code/bench?cmd=web.code.input.find&cmd=shwq") + m.Cmdy("web.code.bench", "http://localhost:9020/code/bench?cmd=web.code.input.find2&cmd=shwq") + }}, + "demo": {Name: "demo list nconn nreq", Help: "导入词库", Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) { + conf, e := conf.Open(kit.Select("hi.shy", arg, 0)) + m.Assert(e) + m.Echo(conf.Get("hi")) }}, }, } -func init() { code.Index.Register(Index, nil) } +func init() { code.Index.Register(Index, &web.Frame{}) }