Skip to content
GitLab
Menu
Why GitLab
Pricing
Contact Sales
Explore
Why GitLab
Pricing
Contact Sales
Explore
Sign in
Get free trial
Primary navigation
Search or go to…
Project
L
lexer
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Container registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Service Desk
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Privacy statement
Keyboard shortcuts
?
What's new
6
Snippets
Groups
Projects
Show more breadcrumbs
cznic
lexer
Commits
ae04e842
Commit
ae04e842
authored
13 years ago
by
Jan Mercl
Browse files
Options
Downloads
Patches
Plain Diff
Adjust for weekly.2011-11-09
parent
5d882cb9
No related branches found
No related tags found
No related merge requests found
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
all_test.go
+1
-1
1 addition, 1 deletion
all_test.go
lexer.go
+3
-3
3 additions, 3 deletions
lexer.go
regex.go
+4
-4
4 additions, 4 deletions
regex.go
source.go
+4
-5
4 additions, 5 deletions
source.go
with
12 additions
and
13 deletions
all_test.go
+
1
−
1
View file @
ae04e842
...
...
@@ -226,7 +226,7 @@ func BenchmarkNFA(b *testing.B) {
var
v
visitor
for
i
:=
0
;
i
<
b
.
N
;
i
++
{
v
=
visitor
{
s
:
lex
.
Scanner
(
"test-go-scanner"
,
nil
)}
filepath
.
Walk
(
runtime
.
GOROOT
()
+
"/src"
,
func
(
pth
string
,
info
*
os
.
FileInfo
,
err
os
.
E
rror
)
os
.
E
rror
{
filepath
.
Walk
(
runtime
.
GOROOT
()
+
"/src"
,
func
(
pth
string
,
info
*
os
.
FileInfo
,
err
e
rror
)
e
rror
{
if
err
!=
nil
{
panic
(
err
)
}
...
...
This diff is collapsed.
Click to expand it.
lexer.go
+
3
−
3
View file @
ae04e842
...
...
@@ -213,13 +213,13 @@ type Lexer struct {
type
StartSetID
int
//TODO:full docs
func
CompileLexer
(
starts
[][]
int
,
tokdefs
map
[
string
]
int
,
grammar
,
start
string
)
(
lexer
*
Lexer
,
err
os
.
E
rror
)
{
func
CompileLexer
(
starts
[][]
int
,
tokdefs
map
[
string
]
int
,
grammar
,
start
string
)
(
lexer
*
Lexer
,
err
e
rror
)
{
lexer
=
&
Lexer
{}
defer
func
()
{
if
e
:=
recover
();
e
!=
nil
{
lexer
=
nil
err
=
e
.
(
os
.
E
rror
)
err
=
e
.
(
e
rror
)
}
}()
...
...
@@ -303,7 +303,7 @@ func CompileLexer(starts [][]int, tokdefs map[string]int, grammar, start string)
// MustCompileLexer is like CompileLexer but panics if the definitions cannot be compiled.
// It simplifies safe initialization of global variables holding compiled Lexers.
func
MustCompileLexer
(
starts
[][]
int
,
tokdefs
map
[
string
]
int
,
grammar
,
start
string
)
(
lexer
*
Lexer
)
{
var
err
os
.
E
rror
var
err
e
rror
if
lexer
,
err
=
CompileLexer
(
starts
,
tokdefs
,
grammar
,
start
);
err
!=
nil
{
if
list
,
ok
:=
err
.
(
scanner
.
ErrorList
);
ok
{
scanner
.
PrintError
(
os
.
Stderr
,
list
)
...
...
This diff is collapsed.
Click to expand it.
regex.go
+
4
−
4
View file @
ae04e842
...
...
@@ -10,8 +10,8 @@
package
lexer
import
(
"errors"
"fmt"
"os"
"strings"
"unicode"
)
...
...
@@ -23,14 +23,14 @@ func qs(s string) string {
// ParseRE compiles a regular expression re into Nfa, returns the re component starting
// and accepting states or an Error if any.
func
(
n
*
Nfa
)
ParseRE
(
name
,
re
string
)
(
in
,
out
*
NfaState
,
err
os
.
E
rror
)
{
func
(
n
*
Nfa
)
ParseRE
(
name
,
re
string
)
(
in
,
out
*
NfaState
,
err
e
rror
)
{
s
:=
NewScannerSource
(
name
,
strings
.
NewReader
(
re
))
defer
func
()
{
if
e
:=
recover
();
e
!=
nil
{
in
,
out
=
nil
,
nil
pos
:=
s
.
CurrentRune
()
.
Position
err
=
fmt
.
Errorf
(
`%s - "%s^%s" - %s`
,
pos
,
qs
(
re
[
:
pos
.
Offset
]),
qs
(
re
[
pos
.
Offset
:
]),
e
.
(
os
.
E
rror
))
err
=
fmt
.
Errorf
(
`%s - "%s^%s" - %s`
,
pos
,
qs
(
re
[
:
pos
.
Offset
]),
qs
(
re
[
pos
.
Offset
:
]),
e
.
(
e
rror
))
}
}()
...
...
@@ -245,7 +245,7 @@ func (s *ScannerSource) hex() (v int) {
return
v
-
'A'
+
10
}
panic
(
os
.
NewError
(
"expected hex digit"
))
panic
(
errors
.
New
(
"expected hex digit"
))
}
func
(
s
*
ScannerSource
)
expect
(
rune
int
)
{
...
...
This diff is collapsed.
Click to expand it.
source.go
+
4
−
5
View file @
ae04e842
...
...
@@ -12,14 +12,13 @@ package lexer
import
(
"go/token"
"io"
"os"
)
// EOFReader implements a RuneReader allways returning 0 (EOF)
type
EOFReader
int
func
(
r
EOFReader
)
ReadRune
()
(
rune
int
,
size
int
,
err
os
.
E
rror
)
{
return
0
,
0
,
o
s
.
EOF
func
(
r
EOFReader
)
ReadRune
()
(
rune
int
,
size
int
,
err
e
rror
)
{
return
0
,
0
,
i
o
.
EOF
}
type
source
struct
{
...
...
@@ -69,7 +68,7 @@ func (s *Source) Read() (r ScannerRune) {
for
{
r
.
Position
=
s
.
Position
()
r
.
Rune
,
r
.
Size
,
r
.
Err
=
s
.
tos
.
reader
.
ReadRune
()
if
r
.
Err
==
nil
||
r
.
Err
!=
o
s
.
EOF
{
if
r
.
Err
==
nil
||
r
.
Err
!=
i
o
.
EOF
{
p
:=
&
s
.
tos
.
position
p
.
Offset
+=
r
.
Size
if
r
.
Rune
!=
'\n'
{
...
...
@@ -98,7 +97,7 @@ type ScannerRune struct {
Position
token
.
Position
// Starting position of Rune
Rune
int
// Rune value
Size
int
// Rune size
Err
os
.
E
rror
// os.EOF or nil. Any other value invalidates all other fields of a ScannerRune.
Err
e
rror
// os.EOF or nil. Any other value invalidates all other fields of a ScannerRune.
}
// ScannerSource is a Source with one ScannerRune look behind and an on demand one ScannerRune lookahead.
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment