mirror of
https://shylinux.com/x/icebergs
synced 2025-04-27 01:48:27 +08:00
255 lines
8.8 KiB
Go
255 lines
8.8 KiB
Go
package crx
|
|
|
|
import (
|
|
"github.com/shylinux/icebergs"
|
|
"github.com/shylinux/icebergs/base/mdb"
|
|
"github.com/shylinux/icebergs/base/nfs"
|
|
"github.com/shylinux/icebergs/base/web"
|
|
"github.com/shylinux/icebergs/core/code"
|
|
"github.com/shylinux/toolkits"
|
|
|
|
"encoding/csv"
|
|
"github.com/nareix/joy4/av"
|
|
"github.com/nareix/joy4/av/avutil"
|
|
"io/ioutil"
|
|
"os"
|
|
"path"
|
|
"sort"
|
|
)
|
|
|
|
const CHROME = "chrome"
|
|
const (
|
|
SPIDED = "spided"
|
|
CACHED = "cached"
|
|
)
|
|
|
|
var Index = &ice.Context{Name: "chrome", Help: "浏览器",
|
|
Configs: map[string]*ice.Config{
|
|
CHROME: {Name: "chrome", Help: "浏览器", Value: kit.Data(
|
|
kit.MDB_SHORT, "name", "history", "url.history",
|
|
)},
|
|
SPIDED: {Name: "spided", Help: "网页爬虫", Value: kit.Data(
|
|
kit.MDB_SHORT, kit.MDB_LINK, kit.MDB_PATH, "usr/spide",
|
|
)},
|
|
CACHED: {Name: "spided", Help: "网页爬虫", Value: kit.Data(
|
|
kit.MDB_SHORT, kit.MDB_LINK, kit.MDB_PATH, "usr/spide",
|
|
)},
|
|
},
|
|
Commands: map[string]*ice.Command{
|
|
ice.CTX_INIT: {Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) {
|
|
m.Load()
|
|
}},
|
|
ice.CTX_EXIT: {Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) {
|
|
m.Save(SPIDED, CACHED)
|
|
}},
|
|
CHROME: {Name: "chrome wid=auto url auto 编译:button 下载:button", Help: "浏览器", Action: map[string]*ice.Action{
|
|
"compile": {Name: "compile", Help: "编译", Hand: func(m *ice.Message, arg ...string) {
|
|
}},
|
|
}, Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) {
|
|
m.Cmdy(web.SPACE, CHROME, CHROME, arg)
|
|
}},
|
|
SPIDED: {Name: "spided wid=auto tid=auto cmd auto", Help: "网页爬虫", Action: map[string]*ice.Action{
|
|
"download": {Name: "download", Help: "下载", Hand: func(m *ice.Message, arg ...string) {
|
|
if m.Richs(CACHED, "", m.Option("link"), func(key string, value map[string]interface{}) {
|
|
if _, e := os.Stat(path.Join(m.Conf(CACHED, kit.META_PATH), m.Option("name"))); e == nil {
|
|
m.Push(key, value)
|
|
}
|
|
}) != nil && len(m.Appendv("name")) > 0 {
|
|
return
|
|
}
|
|
|
|
m.Cmd(mdb.INSERT, m.Prefix(CACHED), "", mdb.HASH,
|
|
kit.MDB_LINK, m.Option("link"),
|
|
kit.MDB_TYPE, m.Option("type"),
|
|
kit.MDB_NAME, m.Option("name"),
|
|
kit.MDB_TEXT, m.Option("text"),
|
|
)
|
|
|
|
// 进度
|
|
m.Richs(CACHED, "", m.Option("link"), func(key string, value map[string]interface{}) {
|
|
m.Optionv("progress", func(size int, total int) {
|
|
p := size * 100 / total
|
|
if p != value["progress"] {
|
|
m.Log_IMPORT(kit.MDB_FILE, m.Option("name"), "per", size*100/total, kit.MDB_SIZE, kit.FmtSize(int64(size)), "total", kit.FmtSize(int64(total)))
|
|
}
|
|
value["progress"], value["size"], value["total"] = p, size, total
|
|
})
|
|
})
|
|
|
|
// 下载
|
|
msg := m.Cmd(web.SPIDE, web.SPIDE_DEV, web.SPIDE_CACHE, web.SPIDE_GET, m.Option("link"))
|
|
p := path.Join(m.Conf(CACHED, kit.META_PATH), m.Option("name"))
|
|
m.Cmdy(nfs.LINK, p, msg.Append("file"))
|
|
|
|
if file, e := avutil.Open(p); m.Assert(e) {
|
|
defer file.Close()
|
|
|
|
if streams, e := file.Streams(); m.Assert(e) {
|
|
for _, stream := range streams {
|
|
if stream.Type().IsAudio() {
|
|
|
|
} else if stream.Type().IsVideo() {
|
|
vstream := stream.(av.VideoCodecData)
|
|
if vstream.Width() > vstream.Height() {
|
|
m.Cmdy(nfs.LINK, path.Join(m.Conf(CACHED, kit.META_PATH), "横屏", m.Option("name")), p)
|
|
} else {
|
|
m.Cmdy(nfs.LINK, path.Join(m.Conf(CACHED, kit.META_PATH), "竖屏", m.Option("name")), p)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}},
|
|
"compile": {Name: "compile", Help: "编译", Hand: func(m *ice.Message, arg ...string) {
|
|
}},
|
|
}, Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) {
|
|
msg := m.Cmd(web.SPACE, CHROME, CHROME, arg)
|
|
switch kit.Select("spide", arg, 2) {
|
|
case "cache":
|
|
m.Option("fields", "time,type,progress,size,total,name,text,link")
|
|
m.Cmdy(mdb.SELECT, m.Prefix(SPIDED), "", mdb.HASH)
|
|
case "spide":
|
|
if len(arg) > 1 {
|
|
msg.PushAction("下载")
|
|
msg.Table(func(index int, value map[string]string, head []string) {
|
|
m.Push("time", value["time"])
|
|
m.Push("type", value["type"])
|
|
m.Push("action", value["action"])
|
|
m.Push("name", value["name"])
|
|
switch value["type"] {
|
|
case "img":
|
|
m.Push("text", m.Cmdx(mdb.RENDER, web.RENDER.IMG, value["text"]))
|
|
case "video":
|
|
m.Push("text", m.Cmdx(mdb.RENDER, web.RENDER.Video, value["text"]))
|
|
default:
|
|
m.Push("text", value["text"])
|
|
}
|
|
m.Push("link", value["link"])
|
|
})
|
|
break
|
|
}
|
|
fallthrough
|
|
default:
|
|
m.Copy(msg)
|
|
}
|
|
}},
|
|
CACHED: {Name: "cached hash=auto auto 清理:button 导出:button", Help: "网页爬虫", Action: map[string]*ice.Action{
|
|
"download": {Name: "download", Help: "下载", Hand: func(m *ice.Message, arg ...string) {
|
|
m.Richs(CACHED, "", m.Option("link"), func(key string, value map[string]interface{}) {
|
|
m.Optionv("progress", func(size int, total int) {
|
|
value["progress"], value["size"], value["total"] = size*100/total, size, total
|
|
m.Log_IMPORT(kit.MDB_FILE, m.Option("name"), "per", size*100/total, kit.MDB_SIZE, kit.FmtSize(int64(size)), "total", kit.FmtSize(int64(total)))
|
|
})
|
|
})
|
|
|
|
msg := m.Cmd(web.SPIDE, web.SPIDE_DEV, web.SPIDE_CACHE, web.SPIDE_GET, m.Option("link"))
|
|
p := path.Join(m.Conf(CACHED, kit.META_PATH), m.Option("name"))
|
|
m.Cmdy(nfs.LINK, p, msg.Append("file"))
|
|
|
|
// 完成
|
|
if file, e := avutil.Open(p); m.Assert(e) {
|
|
defer file.Close()
|
|
if streams, e := file.Streams(); m.Assert(e) {
|
|
for _, stream := range streams {
|
|
if stream.Type().IsAudio() {
|
|
|
|
} else if stream.Type().IsVideo() {
|
|
vstream := stream.(av.VideoCodecData)
|
|
if vstream.Width() > vstream.Height() {
|
|
m.Cmdy(nfs.LINK, path.Join(m.Conf(CACHED, kit.META_PATH), "横屏", m.Option("name")), p)
|
|
} else {
|
|
m.Cmdy(nfs.LINK, path.Join(m.Conf(CACHED, kit.META_PATH), "竖屏", m.Option("name")), p)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}},
|
|
"prune": {Name: "prune", Help: "清理", Hand: func(m *ice.Message, arg ...string) {
|
|
m.Richs(CACHED, "", kit.MDB_FOREACH, func(key string, value map[string]interface{}) {
|
|
if kit.Int(value["progress"]) == 100 {
|
|
dir := path.Join("var/data", m.Prefix(CACHED), "")
|
|
name := path.Join(dir, kit.Keys(key, "json"))
|
|
if f, p, e := kit.Create(name); e == nil {
|
|
defer f.Close()
|
|
// 保存数据
|
|
if n, e := f.WriteString(kit.Format(value)); e == nil {
|
|
m.Log_EXPORT("file", p, kit.MDB_SIZE, n)
|
|
}
|
|
}
|
|
m.Conf(CACHED, kit.Keys(kit.MDB_HASH, key), "")
|
|
}
|
|
})
|
|
}},
|
|
"export": {Name: "export", Help: "导出", Hand: func(m *ice.Message, arg ...string) {
|
|
f, p, e := kit.Create(path.Join("usr/export", m.Prefix(CACHED), "list.csv"))
|
|
m.Assert(e)
|
|
defer f.Close()
|
|
|
|
w := csv.NewWriter(f)
|
|
defer w.Flush()
|
|
|
|
count := 0
|
|
head := []string{}
|
|
m.Cmd(nfs.DIR, path.Join("var/data", m.Prefix(CACHED))+"/").Table(func(index int, v map[string]string, h []string) {
|
|
|
|
f, e := os.Open(v["path"])
|
|
m.Assert(e)
|
|
defer f.Close()
|
|
|
|
b, e := ioutil.ReadAll(f)
|
|
m.Assert(e)
|
|
|
|
value, ok := kit.UnMarshal(string(b)).(map[string]interface{})
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
if index == 0 {
|
|
// 输出表头
|
|
for k := range value {
|
|
head = append(head, k)
|
|
}
|
|
sort.Strings(head)
|
|
w.Write(head)
|
|
}
|
|
|
|
// 输出数据
|
|
data := []string{}
|
|
for _, k := range head {
|
|
data = append(data, kit.Format(value[k]))
|
|
}
|
|
w.Write(data)
|
|
count++
|
|
})
|
|
m.Log_EXPORT(kit.MDB_FILE, p, kit.MDB_COUNT, count)
|
|
m.Echo(p)
|
|
}},
|
|
}, Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) {
|
|
m.Option("cache.limit", 100)
|
|
m.Option("fields", "time,hash,type,progress,size,total,name,text,link")
|
|
m.Cmdy(mdb.SELECT, m.Prefix(CACHED), "", mdb.HASH)
|
|
m.Sort("time", "time_r")
|
|
m.PushAction("下载")
|
|
m.Appendv(ice.MSG_APPEND, "time", "type", "name", "text",
|
|
"action", "progress", "size", "total", "hash", "link")
|
|
}},
|
|
|
|
"/crx": {Name: "/crx", Help: "插件", Action: map[string]*ice.Action{
|
|
web.HISTORY: {Name: "history", Help: "历史记录", Hand: func(m *ice.Message, arg ...string) {
|
|
m.Cmdy(web.SPIDE, web.SPIDE_DEV, "/code/chrome/favor", "cmds", mdb.INSERT, "sid", m.Option("sid"),
|
|
"tab", m.Conf(CHROME, "meta.history"), "name", arg[1], "note", arg[2])
|
|
}},
|
|
}, Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) {
|
|
}},
|
|
"/favor": {Name: "/favor", Help: "收藏", Action: map[string]*ice.Action{
|
|
mdb.INSERT: {Name: "insert", Help: "插入", Hand: func(m *ice.Message, arg ...string) {
|
|
m.Cmdy(web.FAVOR, mdb.INSERT, m.Option("tab"), web.SPIDE, m.Option("name"), m.Option("note"))
|
|
}},
|
|
}, Hand: func(m *ice.Message, c *ice.Context, cmd string, arg ...string) {
|
|
}},
|
|
},
|
|
}
|
|
|
|
func init() { code.Index.Register(Index, &web.Frame{}) }
|