liubin/ContextOS
1
0
Fork 0
forked from x/ContextOS
Code Pull Requests Activity

mac pro lex&yac 整理了一下词法解析与语法解析代码

Browse Source
This commit is contained in:
shaoying 2018-02-03 09:32:47 +08:00
parent a2395ddb2c
commit 55548aee0f
3 changed files with 109 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
README.md
View File

@ -11,46 +11,38 @@ context: 通过提供自由的模块，简洁的接口，动态的结构，让
* 编程: 接口与框架 * 编程: 接口与框架
* 测试: 语句与表达式 * 测试: 语句与表达式
## 接口设计CCC(command, config, cache) ## 接口设计
* 服务管理server * 功能树: Caches Configs Commands
* 模块管理context * 消息树: Request History Session
* 消息管理message
* 命令管理command
* 配置管理config
* 缓存管理cache
### Context ### Context功能树
* Cap() Conf() Cmd() * Cap() Conf() Cmd()
* Spawn() Begin() Start() Close() * Spawn() Begin() Start() Close()
### Message ### Message消息树
* Request[] History[] Session[] * Detail() Option() Result() Append()
* detail[] option[] result[] append[] * Req() His() Sess()
### Conbine
* Context Master Owner
* Search() Choice() Assert() Figure()
## 模块设计 ## 模块设计
* ctx cli aaa web * 应用层 ctx cli aaa web
* nfs tcp mdb ssh * 控制层 lex yac log gdb
* lex yac log gdb * 数据层 tcp nfs ssh mdb
### Core ### 应用层
* ctx: 模块中心 * ctx: 模块中心
* cli: 管理中心 * cli: 管理中心
* aaa: 认证中心 * aaa: 认证中心
* web: 应用中心 * web: 应用中心
### Base ### 控制层
* nfs: 存储中心 * lex: 词法中心
* tcp: 网络中心 * yac: 语法中心
* mdb: 数据中心
* ssh: 集群中心
### Draw
* lex: 词法解析
* yac: 语法解析
* log: 日志中心 * log: 日志中心
* gdb: 调试中心 * gdb: 调试中心
### 数据层
* tcp: 网络中心
* nfs: 存储中心
* ssh: 集群中心
* mdb: 数据中心

143
src/context/lex/lex.go
View File

@ -26,26 +26,24 @@ type LEX struct {
page map[string]int page map[string]int
hash map[string]int hash map[string]int
char map[byte][]byte
state map[State]*State
mat []map[byte]*State mat []map[byte]*State
state map[State]*State
char map[byte][]byte
*ctx.Message *ctx.Message
*ctx.Context *ctx.Context
} }
func (lex *LEX) index(hash string, h string) int { func (lex *LEX) index(hash string, h string) int {
if x, e := strconv.Atoi(h); e == nil { which := lex.page
return x if hash == "nhash" {
which = lex.hash
} }
which := lex.page if x, e := strconv.Atoi(h); e == nil {
switch hash { lex.Assert(hash != "npage" || x < lex.Capi("npage"))
case "npage": return x
which = lex.page
case "nhash":
which = lex.hash
} }
if x, ok := which[h]; ok { if x, ok := which[h]; ok {
@ -53,6 +51,7 @@ func (lex *LEX) index(hash string, h string) int {
} }
which[h] = lex.Capi(hash, 1) which[h] = lex.Capi(hash, 1)
lex.Assert(hash != "npage" || lex.Capi("npage") < lex.Capi("nlang"))
return which[h] return which[h]
} }
@ -65,10 +64,10 @@ func (lex *LEX) charset(c byte) []byte {
func (lex *LEX) train(page int, hash int, seed []byte) int { func (lex *LEX) train(page int, hash int, seed []byte) int {
ss := []int{page}
cn := make([]bool, lex.Capi("ncell")) cn := make([]bool, lex.Capi("ncell"))
c := make([]byte, 0, lex.Capi("ncell")) cc := make([]byte, 0, lex.Capi("ncell"))
sn := make([]bool, lex.Capi("nline")) sn := make([]bool, lex.Capi("nline"))
s := []int{page}
points := []*Point{} points := []*Point{}
@ -84,8 +83,8 @@ func (lex *LEX) train(page int, hash int, seed []byte) int {
for ; seed[p] != ']'; p++ { for ; seed[p] != ']'; p++ {
if seed[p] == '\\' { if seed[p] == '\\' {
p++ p++
for _, s := range lex.charset(seed[p]) { for _, c := range lex.charset(seed[p]) {
cn[s] = true cn[c] = true
} }
continue continue
} }
@ -95,8 +94,8 @@ func (lex *LEX) train(page int, hash int, seed []byte) int {
if begin > end { if begin > end {
begin, end = end, begin begin, end = end, begin
} }
for i := begin; i <= end; i++ { for c := begin; c <= end; c++ {
cn[i] = true cn[c] = true
} }
p += 2 p += 2
continue continue
@ -105,29 +104,29 @@ func (lex *LEX) train(page int, hash int, seed []byte) int {
cn[seed[p]] = true cn[seed[p]] = true
} }
for i := 0; i < len(cn); i++ { for c := 0; c < len(cn); c++ {
if (set && cn[i]) || (!set && !cn[i]) { if (set && cn[c]) || (!set && !cn[c]) {
c = append(c, byte(i)) cc = append(cc, byte(c))
} }
cn[i] = false cn[c] = false
} }
case '.': case '.':
for i := 0; i < len(cn); i++ { for c := 0; c < len(cn); c++ {
c = append(c, byte(i)) cc = append(cc, byte(c))
} }
case '\\': case '\\':
p++ p++
for _, s := range lex.charset(seed[p]) { for _, c := range lex.charset(seed[p]) {
c = append(c, s) cc = append(cc, c)
} }
default: default:
c = append(c, seed[p]) cc = append(cc, seed[p])
} }
lex.Log("debug", nil, "page: \033[31m%d %v\033[0m", len(s), s) lex.Log("debug", nil, "page: \033[31m%d %v\033[0m", len(ss), ss)
lex.Log("debug", nil, "cell: \033[32m%d %v\033[0m", len(c), c) lex.Log("debug", nil, "cell: \033[32m%d %v\033[0m", len(cc), cc)
flag := '\000' flag := '\000'
if p+1 < len(seed) { if p+1 < len(seed) {
@ -138,29 +137,30 @@ func (lex *LEX) train(page int, hash int, seed []byte) int {
} }
} }
for i := 0; i < len(s); i++ { for _, s := range ss {
for line, j := 0, byte(0); int(j) < len(c); j++ { line := 0
for _, c := range cc {
state := &State{} state := &State{}
if lex.mat[s[i]][c[j]] != nil { if lex.mat[s][c] != nil {
*state = *lex.mat[s[i]][c[j]] *state = *lex.mat[s][c]
} else { } else {
lex.Capi("nnode", 1) lex.Capi("nnode", 1)
} }
lex.Log("debug", nil, "GET(%d,%d): %v", s[i], c[j], state) lex.Log("debug", nil, "GET(%d,%d): %v", s, c, state)
switch flag { switch flag {
case '+': case '+':
state.star = true state.star = true
case '*': case '*':
state.star = true state.star = true
sn[s[i]] = true sn[s] = true
case '?': case '?':
sn[s[i]] = true sn[s] = true
} }
if state.next == 0 { if state.next == 0 {
if line == 0 || !lex.Caps("compact") { if line == 0 || !lex.Confs("compact") {
lex.mat = append(lex.mat, make(map[byte]*State)) lex.mat = append(lex.mat, make(map[byte]*State))
line = lex.Capi("nline", 1) - 1 line = lex.Capi("nline", 1) - 1
sn = append(sn, false) sn = append(sn, false)
@ -169,54 +169,50 @@ func (lex *LEX) train(page int, hash int, seed []byte) int {
} }
sn[state.next] = true sn[state.next] = true
lex.mat[s[i]][c[j]] = state lex.mat[s][c] = state
points = append(points, &Point{s[i], c[j]}) points = append(points, &Point{s, c})
lex.Log("debug", nil, "SET(%d,%d): %v(%s,%s)", s[i], c[j], state, lex.Cap("nnode"), lex.Cap("nreal")) lex.Log("debug", nil, "SET(%d,%d): %v(%s,%s)", s, c, state, lex.Cap("nnode"), lex.Cap("nreal"))
} }
} }
c, s = c[:0], s[:0] cc, ss = cc[:0], ss[:0]
for i := 0; i < len(sn); i++ { for s, b := range sn {
if sn[i] { if sn[s] = false; b {
s = append(s, i) ss = append(ss, s)
} }
sn[i] = false
} }
} }
for _, n := range s { for _, s := range ss {
if n < lex.Capi("nlang") || n >= len(lex.mat) { if s < lex.Capi("nlang") || s >= len(lex.mat) {
continue continue
} }
if len(lex.mat[n]) == 0 { if len(lex.mat[s]) == 0 {
lex.Log("debug", nil, "DEL: %d %d", lex.Capi("nline")-1, lex.Capi("nline", 0, n)) lex.Log("debug", nil, "DEL: %d-%d", lex.Capi("nline")-1, lex.Capi("nline", 0, s))
lex.mat = lex.mat[:n] lex.mat = lex.mat[:s]
} }
} }
for _, n := range s { for _, s := range ss {
for _, p := range points { for _, p := range points {
state := &State{} state := &State{}
*state = *lex.mat[p.s][p.c] *state = *lex.mat[p.s][p.c]
if state.next == n { if state.next == s {
lex.Log("debug", nil, "GET(%d, %d): %v", p.s, p.c, state) lex.Log("debug", nil, "GET(%d, %d): %v", p.s, p.c, state)
if state.next >= len(lex.mat) { if state.hash = hash; state.next >= len(lex.mat) {
state.next = 0 state.next = 0
} }
if hash > 0 { lex.mat[p.s][p.c] = state
state.hash = hash
}
lex.Log("debug", nil, "SET(%d, %d): %v", p.s, p.c, state) lex.Log("debug", nil, "SET(%d, %d): %v", p.s, p.c, state)
} }
if x, ok := lex.state[*state]; ok { if x, ok := lex.state[*state]; !ok {
lex.mat[p.s][p.c] = x lex.state[*state] = lex.mat[p.s][p.c]
} else {
lex.state[*state] = state
lex.mat[p.s][p.c] = state
lex.Capi("nreal", 1) lex.Capi("nreal", 1)
} else {
lex.mat[p.s][p.c] = x
} }
} }
} }
@ -224,7 +220,7 @@ func (lex *LEX) train(page int, hash int, seed []byte) int {
return hash return hash
} }
func (lex *LEX) parse(page int, line []byte) (hash int, word []byte, rest []byte) { func (lex *LEX) parse(page int, line []byte) (hash int, rest []byte, word []byte) {
pos := 0 pos := 0
for star, s := 0, page; s != 0 && pos < len(line); pos++ { for star, s := 0, page; s != 0 && pos < len(line); pos++ {
@ -237,23 +233,19 @@ func (lex *LEX) parse(page int, line []byte) (hash int, word []byte, rest []byte
state := lex.mat[s][c] state := lex.mat[s][c]
lex.Log("debug", nil, "(%d,%d): %v", s, c, state) lex.Log("debug", nil, "(%d,%d): %v", s, c, state)
if state == nil && star != 0 {
s, star = star, 0
state = lex.mat[s][c]
lex.Log("debug", nil, "(%d,%d): %v", s, c, state)
}
if state == nil { if state == nil {
break s, star, pos = star, 0, pos-1
} continue
if state, ok := lex.mat[star][c]; !ok || state == nil || !state.star {
star = 0
}
if state.star {
star = s
} }
word = append(word, c) word = append(word, c)
if state.star {
star = s
} else if _, ok := lex.mat[star][c]; !ok {
star = 0
}
if s, hash = state.next, state.hash; s == 0 { if s, hash = state.next, state.hash; s == 0 {
s, star = star, 0 s, star = star, 0
} }
@ -278,11 +270,10 @@ func (lex *LEX) Spawn(m *ctx.Message, c *ctx.Context, arg ...string) ctx.Server
} }
func (lex *LEX) Begin(m *ctx.Message, arg ...string) ctx.Server { func (lex *LEX) Begin(m *ctx.Message, arg ...string) ctx.Server {
if lex.Context == Index { if lex.Message = m; lex.Context == Index {
Pulse = m Pulse = m
} }
lex.Context.Master(nil) lex.Context.Master(nil)
lex.Message = m
lex.Caches["ncell"] = &ctx.Cache{Name: "字符上限", Value: "128", Help: "字符上限"} lex.Caches["ncell"] = &ctx.Cache{Name: "字符上限", Value: "128", Help: "字符上限"}
lex.Caches["nlang"] = &ctx.Cache{Name: "集合上限", Value: "32", Help: "集合上限"} lex.Caches["nlang"] = &ctx.Cache{Name: "集合上限", Value: "32", Help: "集合上限"}
@ -295,7 +286,7 @@ func (lex *LEX) Begin(m *ctx.Message, arg ...string) ctx.Server {
lex.Caches["nnode"] = &ctx.Cache{Name: "节点数量", Value: "0", Help: "节点数量"} lex.Caches["nnode"] = &ctx.Cache{Name: "节点数量", Value: "0", Help: "节点数量"}
lex.Caches["nreal"] = &ctx.Cache{Name: "实点数量", Value: "0", Help: "实点数量"} lex.Caches["nreal"] = &ctx.Cache{Name: "实点数量", Value: "0", Help: "实点数量"}
lex.Caches["compact"] = &ctx.Cache{Name: "紧凑模式", Value: "true", Help: "实点数量"} lex.Configs["compact"] = &ctx.Config{Name: "紧凑模式", Value: "true", Help: "实点数量"}
return lex return lex
} }

48
src/context/yac/yac.go
View File

@ -29,24 +29,24 @@ type YAC struct {
hash map[string]int hash map[string]int
hand map[int]string hand map[int]string
state map[State]*State
mat []map[byte]*State mat []map[byte]*State
state map[State]*State
*ctx.Message *ctx.Message
*ctx.Context *ctx.Context
} }
func (yac *YAC) name(page int) string { func (yac *YAC) name(page int) string {
name, ok := yac.word[page] if name, ok := yac.word[page]; ok {
if !ok {
name = fmt.Sprintf("yac%d", page)
}
return name return name
} }
return fmt.Sprintf("yac%d", page)
}
func (yac *YAC) train(m *ctx.Message, page, hash int, word []string) (int, []*Point, []*Point) { func (yac *YAC) train(m *ctx.Message, page, hash int, word []string) (int, []*Point, []*Point) {
sn := make([]bool, yac.Capi("nline"))
ss := []int{page} ss := []int{page}
sn := make([]bool, yac.Capi("nline"))
points := []*Point{} points := []*Point{}
ends := []*Point{} ends := []*Point{}
@ -91,18 +91,13 @@ func (yac *YAC) train(m *ctx.Message, page, hash int, word []string) (int, []*Po
fallthrough fallthrough
default: default:
x, ok := yac.page[word[i]] x, ok := yac.page[word[i]]
if !ok { if !ok {
lex := yac.Sess("lex").Cmd("parse", word[i], yac.name(s)) if x = yac.Sess("lex").Cmd("parse", word[i], yac.name(s)).Resulti(0); x == 0 {
if lex.Gets("result") { x = yac.Sess("lex").Cmd("train", word[i], len(yac.mat[s]), yac.name(s)).Resulti(0)
x = lex.Geti("result")
} else {
x = len(yac.mat[s])
lex.Cmd("train", word[i], x, yac.name(s))
} }
} }
c := byte(x) c := byte(x)
state := &State{} state := &State{}
if yac.mat[s][c] != nil { if yac.mat[s][c] != nil {
*state = *yac.mat[s][c] *state = *yac.mat[s][c]
@ -139,30 +134,31 @@ func (yac *YAC) train(m *ctx.Message, page, hash int, word []string) (int, []*Po
} }
} }
for _, n := range ss { for _, s := range ss {
if n < yac.Capi("nlang") || n >= len(yac.mat) { if s < yac.Capi("nlang") || s >= len(yac.mat) {
continue continue
} }
void := true void := true
for _, x := range yac.mat[n] { for _, x := range yac.mat[s] {
if x != nil { if x != nil {
void = false void = false
break break
} }
} }
if void { if void {
yac.Log("debug", nil, "DEL: %d %d", yac.Capi("nline")-1, yac.Capi("nline", 0, n)) yac.Log("debug", nil, "DEL: %d-%d", yac.Capi("nline")-1, yac.Capi("nline", 0, s))
yac.mat = yac.mat[:n] yac.mat = yac.mat[:s]
} }
} }
for _, n := range ss { for _, s := range ss {
for _, p := range points { for _, p := range points {
state := &State{} state := &State{}
*state = *yac.mat[p.s][p.c] *state = *yac.mat[p.s][p.c]
if state.next == n { if state.next == s {
yac.Log("debug", nil, "GET(%d, %d): %v", p.s, p.c, state) yac.Log("debug", nil, "GET(%d, %d): %v", p.s, p.c, state)
if state.next >= len(yac.mat) { if state.next >= len(yac.mat) {
state.next = 0 state.next = 0
@ -170,15 +166,15 @@ func (yac *YAC) train(m *ctx.Message, page, hash int, word []string) (int, []*Po
if hash > 0 { if hash > 0 {
state.hash = hash state.hash = hash
} }
yac.mat[p.s][p.c] = state
yac.Log("debug", nil, "SET(%d, %d): %v", p.s, p.c, state) yac.Log("debug", nil, "SET(%d, %d): %v", p.s, p.c, state)
} }
if x, ok := yac.state[*state]; ok { if x, ok := yac.state[*state]; !ok {
yac.mat[p.s][p.c] = x yac.state[*state] = yac.mat[p.s][p.c]
} else {
yac.state[*state] = state
yac.mat[p.s][p.c] = state
yac.Capi("nreal", 1) yac.Capi("nreal", 1)
} else {
yac.mat[p.s][p.c] = x
} }
} }
} }