Skip to content

Commit

Permalink
Merge pull request #2139 from xushiwei/q
Browse files Browse the repository at this point in the history
tpl/matcher
  • Loading branch information
xushiwei authored Feb 28, 2025
2 parents 8d46478 + ecd764a commit 73922e4
Show file tree
Hide file tree
Showing 5 changed files with 293 additions and 41 deletions.
36 changes: 15 additions & 21 deletions demo/gop-parser/_spec/expr.gop
Original file line number Diff line number Diff line change
Expand Up @@ -11,28 +11,29 @@ node (
BinaryExpr Expr
)

expr = termExpr | (X:expr binOp Y:expr as BinaryExpr {
expr = x:termExpr *(binOp:("+" | "-") Y:termExpr as BinaryExpr {
X: x
OpPos: binOp.Pos
Op: binOp.Tok
})

binOp = "+" | "-"

termExpr = unaryExpr | (X:termExpr termOp Y:termExpr as BinaryExpr {
OpPos: termOp.Pos
Op: termOp.Tok
termExpr = x:unaryExpr *(binOp:("*" | "/" | "%") Y:unaryExpr as BinaryExpr {
X: x
OpPos: binOp.Pos
Op: binOp.Tok
})

termOp = "*" | "/" | "%"

unaryExpr = primaryExpr | (unaryOp X:unaryExpr as UnaryExpr {
OpPos: unaryOp.Pos
Op: unaryOp.Tok
unaryExpr = primaryExpr | (uOp:("+" | "-" | "!" | "^") X:primaryExpr as UnaryExpr {
OpPos: uOp.Pos
Op: uOp.Tok
})

unaryOp = "+" | "-" | "!" | "^"

primaryExpr = operand | CallExpr
primaryExpr = operand *(lp:"(" ?(Args:(expr % ",") ?ell:"..." ?",") rp:")" as CallExpr {
Fun: operand
Lparen: lp.Pos
Ellipsis: ell != nil ? ell.Pos : 0
Rparen: rp.Pos
})

operand = BasicLit | Ident | ParenExpr

Expand All @@ -51,10 +52,3 @@ ParenExpr = lp:"(" X:expr rp:")" {
Lparen: lp.Pos
Rparen: rp.Pos
}

CallExpr = primaryExpr lp:"(" ?(Args:(expr % ",") ?ell:"..." ?",") rp:")" {
Fun: primaryExpr
Lparen: lp.Pos
Ellipsis: ell != nil ? ell.Pos : 0
Rparen: rp.Pos
}
233 changes: 233 additions & 0 deletions tpl/matcher/match.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,233 @@
/*
* Copyright (c) 2025 The GoPlus Authors (goplus.org). All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package matcher

import (
"github.com/goplus/gop/tpl/token"
"github.com/goplus/gop/tpl/types"
)

// -----------------------------------------------------------------------------
// Matcher

// Context represents the context of a matching process.
type Context struct {
}

// Matcher represents a matcher.
type Matcher interface {
Match(src []*types.Token, ctx *Context) (n int, result any, err error)
}

// -----------------------------------------------------------------------------

// Error represents a matching error.
type Error struct {
}

func (p *Error) Error() string {
panic("todo")
}

// -----------------------------------------------------------------------------

type gToken struct {
tok token.Token
}

func (p *gToken) Match(src []*types.Token, ctx *Context) (n int, result any, err error) {
if len(src) == 0 {
return 0, nil, &Error{} // TODO(xsw): err
}
t := src[0]
if t.Tok != p.tok {
return 0, nil, &Error{} // TODO(xsw): err
}
return 1, t, nil
}

// Token: ADD, SUB, IDENT, INT, FLOAT, CHAR, STRING, etc.
func Token(tok token.Token) Matcher {
return &gToken{tok}
}

// -----------------------------------------------------------------------------

type gLiteral struct {
tok token.Token
lit string
}

func (p *gLiteral) Match(src []*types.Token, ctx *Context) (n int, result any, err error) {
if len(src) == 0 {
return 0, nil, &Error{} // TODO(xsw): err
}
t := src[0]
if t.Tok != p.tok || t.Lit != p.lit {
return 0, nil, &Error{} // TODO(xsw): err
}
return 1, t, nil
}

// Literal: "abc", 'a', 123, 1.23, etc.
func Literal(tok token.Token, lit string) Matcher {
return &gLiteral{tok, lit}
}

// -----------------------------------------------------------------------------

type gChoice struct {
options []Matcher
}

func (p *gChoice) Match(src []*types.Token, ctx *Context) (n int, result any, err error) {
var nMax int
var errMax error
var multiErr = true

for _, g := range p.options {
if n, result, err = g.Match(src, ctx); err == nil {
return
}
if n >= nMax {
if n == nMax {
multiErr = true
} else {
nMax, errMax, multiErr = n, err, false
}
}
}
if multiErr {
errMax = &Error{} // TODO(xsw): err
}
return nMax, nil, errMax
}

// Choice: R1 | R2 | ... | Rn
func Choice(options ...Matcher) Matcher {
return &gChoice{options}
}

// -----------------------------------------------------------------------------

type gSequence struct {
items []Matcher
}

func (p *gSequence) Match(src []*types.Token, ctx *Context) (n int, result any, err error) {
rets := make([]any, len(p.items))
for i, g := range p.items {
n1, ret1, err1 := g.Match(src[n:], ctx)
if err1 != nil {
return n + n1, nil, err1
}
rets[i] = ret1
n += n1
}
result = rets
return
}

// Sequence: R1 R2 ... Rn
func Sequence(items ...Matcher) Matcher {
return &gSequence{items}
}

// -----------------------------------------------------------------------------

type gRepeat0 struct {
r Matcher
}

func (p *gRepeat0) Match(src []*types.Token, ctx *Context) (n int, result any, err error) {
g := p.r
rets := make([]any, 0, 2)
for {
n1, ret1, err1 := g.Match(src, ctx)
if err1 != nil {
result = rets
return
}
rets = append(rets, ret1)
n += n1
src = src[n1:]
}
}

// Repeat0: *R
func Repeat0(r Matcher) Matcher {
return &gRepeat0{r}
}

// -----------------------------------------------------------------------------

type gRepeat1 struct {
r Matcher
}

func (p *gRepeat1) Match(src []*types.Token, ctx *Context) (n int, result any, err error) {
g := p.r
n, ret0, err := g.Match(src, ctx)
if err != nil {
return
}

rets := make([]any, 1, 2)
rets[0] = ret0
for {
n1, ret1, err1 := g.Match(src[n:], ctx)
if err1 != nil {
result = rets
return
}
rets = append(rets, ret1)
n += n1
}
}

// Repeat1: +R
func Repeat1(r Matcher) Matcher {
return &gRepeat1{r}
}

// -----------------------------------------------------------------------------

type gRepeat01 struct {
r Matcher
}

func (p *gRepeat01) Match(src []*types.Token, ctx *Context) (n int, result any, err error) {
n, result, err = p.r.Match(src, ctx)
if err != nil {
return 0, nil, nil
}
return
}

// Repeat01: ?R
func Repeat01(r Matcher) Matcher {
return &gRepeat01{r}
}

// -----------------------------------------------------------------------------

// List: R1 % R2 is equivalent to R1 *(R2 R1)
func List(a, b Matcher) Matcher {
return Sequence(a, Repeat0(Sequence(b, a)))
}

// -----------------------------------------------------------------------------
19 changes: 2 additions & 17 deletions tpl/scanner/scanner.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,24 +25,9 @@ import (
"unicode/utf8"

"github.com/goplus/gop/tpl/token"
"github.com/goplus/gop/tpl/types"
)

// A Token is a lexical unit returned by Scan.
type Token struct {
Tok token.Token
Pos token.Pos
Lit string
}

// End returns end position of this token.
func (p *Token) End() token.Pos {
n := len(p.Lit)
if n == 0 {
n = p.Tok.Len()
}
return p.Pos + token.Pos(n)
}

// An ScanErrorHandler may be provided to Scanner.Init. If a syntax error is
// encountered and a handler was installed, the handler is called with a
// position and an error message. The position points to the beginning of
Expand Down Expand Up @@ -630,7 +615,7 @@ func (s *Scanner) switch4(tok0, tok1 token.Token, ch2 rune, tok2, tok3 token.Tok
// Scan adds line information to the file added to the file
// set with Init. Token positions are relative to that file
// and thus relative to the file set.
func (s *Scanner) Scan() (t Token) {
func (s *Scanner) Scan() (t types.Token) {
scanAgain:
s.skipWhitespace()

Expand Down
9 changes: 6 additions & 3 deletions tpl/scanner/scanner_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,11 @@ import (
"testing"

"github.com/goplus/gop/tpl/token"
"github.com/goplus/gop/tpl/types"
)

type Token = types.Token

type tokenTest struct {
Pos token.Pos
Kind token.Token
Expand Down Expand Up @@ -132,7 +135,7 @@ world !=
if c.Tok == token.EOF {
break
}
expect := Token{expected[i].Kind, expected[i].Pos, expected[i].Literal}
expect := Token{Tok: expected[i].Kind, Pos: expected[i].Pos, Lit: expected[i].Literal}
if c != expect {
t.Fatal("Scan failed:", c, expect)
}
Expand All @@ -155,7 +158,7 @@ world !=
if expected[i].Kind == token.COMMENT {
i++
}
expect := Token{expected[i].Kind, expected[i].Pos, expected[i].Literal}
expect := Token{Tok: expected[i].Kind, Pos: expected[i].Pos, Lit: expected[i].Literal}
if c != expect {
t.Fatal("Scan failed:", c.Pos, c.Lit, expect)
}
Expand Down Expand Up @@ -183,7 +186,7 @@ world !=
if expected[i].Kind == token.COMMENT {
i++
}
expect := Token{expected[i].Kind, expected[i].Pos, expected[i].Literal}
expect := Token{Tok: expected[i].Kind, Pos: expected[i].Pos, Lit: expected[i].Literal}
if c != expect {
t.Fatal("Scan failed:", c.Pos, c.Lit, expect)
}
Expand Down
Loading

0 comments on commit 73922e4

Please sign in to comment.