Commit bffcbc02 authored by Jamie A. Jennings's avatar Jamie A. Jennings

Enhanced compile.make_parser() so that we can include a post-processor, which...

Enhanced compile.make_parser() so that we can include a post-processor, which in this case is infix.to_prefix().  Most tests are passing, but not all.
parent 688220d3
Pipeline #63349426 passed with stage
in 1 minute and 19 seconds
......@@ -25,9 +25,12 @@ RUN make LUADEBUG=1
RUN make test
RUN make install
# ********************************* TEMP TEMP TEMP *********************************
COPY subtitles2016-sample /opt/data
# Here is one simple invocation of valgrind, just to make sure it
# works. Get an interactive shell to this container to run more tests.
RUN valgrind --leak-check=full rosie -f /opt/data/syslog-2018-09-05.rpl match -o byte syslog /opt/data/syslog100k >/dev/null
#RUN valgrind --leak-check=full rosie -f /opt/data/syslog-2018-09-05.rpl match -o byte syslog /opt/data/syslog100k >/dev/null
......
......@@ -143,7 +143,7 @@ grammar
bracket = { openbracket complement? exp atmos closebracket }
cooked = { open exp atmos close }
raw = { openraw exp atmos closeraw }
predicate = { predicate_symbol ws? exp }
predicate = { predicate_symbol ws? term }
-- binding forms
binding = { atmos grammar_block / let_block / simple / empty }
......
......@@ -295,8 +295,13 @@ function convert_bracket(pt, sref)
table.remove(exps, 1)
end
assert(exps[1] and (not exps[2]) and (exps[1].type=="form.exp"))
-- !@#
if exps[1].subs and exps[1].subs[2] then print("*** " .. util.table_to_pretty_string(exps)); end
exps = exps[1].subs
assert(exps[1] and (not exps[2]))
assert(exps[1])
-- for i=2, #exps do assert(not common.not_atmosphere(exps[i]), util.table_to_pretty_string(exps[i])) end
assert(not exps[2], "*** " .. util.table_to_pretty_string(exps))
local cexp
if exps[1].type=="form.sequence" then
local explist = filter(not_atmosphere, flatten_exp(exps[1], "form.sequence"))
......
......@@ -94,7 +94,7 @@ end
-- list, and then fix up that list into a tree afterwards. We would have to
-- move grammar/let expressions into the 'term' definition.
local function make_parser_from(parse_something, expected_pt_node)
local function make_parser_from(parse_something, expected_pt_node, optional_post_processor)
return function(source_record, messages)
assert(common.source.is(source_record))
assert(type(messages)=="table", "missing messages arg?")
......@@ -143,19 +143,29 @@ local function make_parser_from(parse_something, expected_pt_node)
table.insert(messages, err)
return false
end
if optional_post_processor then
local pt2 = optional_post_processor(pt)
if not pt2 then
local msg = "parsing post processor failed"
local err = violation.syntax.new{who='parser', message=msg, sourceref=source_record}
table.insert(messages, err)
return false
end
pt = pt2
end -- if optional_post_processor
local a = ast.from_parse_tree(pt, source_record, messages)
return a
end
end
function c2.make_parse_block(rplx_preparse, rplx_statements, supported_version)
function c2.make_parse_block(rplx_preparse, rplx_statements, supported_version, optional_post_processor)
local parse_block = parse.make_parse_block(rplx_preparse, rplx_statements, supported_version)
return make_parser_from(parse_block, "rpl_statements")
return make_parser_from(parse_block, "rpl_statements", optional_post_processor)
end
function c2.make_parse_expression(rplx_expression)
function c2.make_parse_expression(rplx_expression, optional_post_processor)
local parse_expression = parse.make_parse_expression(rplx_expression)
return make_parser_from(parse_expression, "rpl_expression")
return make_parser_from(parse_expression, "rpl_expression", optional_post_processor)
end
c2.dependencies_of = ast.dependencies_of
......@@ -706,7 +716,7 @@ end
function initialize_bindings(stmts, pkgenv, prefix, messages)
for _, b in ipairs(stmts) do
-- assert(ast.binding.is(b)) -- ensured by expand.block()
assert(ast.binding.is(b)) -- ensured by expand.block()
local ref = b.ref
if ref.packagename then
table.insert(messages,
......
......@@ -35,8 +35,13 @@ local infix = {}
infix.DEBUG=false -- temporary
assert(rosie, "'rosie' not defined. Use this file in rosie lua, not plain lua.")
import = rosie.import
local util = require("util")
local common = require("common")
local list = require("list")
map = list.map; filter = list.filter
local map = list.map
local filter = list.filter
local function new_stack()
local data = {}
......@@ -65,12 +70,12 @@ local function new_stack()
end
local function is_atmosphere(pt)
return (pt.type=="comment") or (pt.type=="newline")
return not common.not_atmosphere(pt)
end
-- Find each exp that has a quantifier succeeding it, and attach the quantifier
-- to the exp.
function attach_quantifiers(exps)
local function attach_quantifiers(exps)
if exps==nil then return nil end
local i = 1
local result = {}
......@@ -96,7 +101,7 @@ end
-- Find two exps in a row with no operator between, and insert an explicit
-- sequence operator. Assumes quantifiers have already been attached.
function insert_seq_operators(exps)
local function insert_seq_operators(exps)
if exps==nil then return nil end
local result = {}
for i, item in ipairs(exps) do
......@@ -122,16 +127,22 @@ local function shunting_yard_exp(exp, attribute_table)
local function map_shunting_yard_exp(ls)
return map(function(e) return shunting_yard_exp(e, attribute_table) end, ls)
end
if exp.type == "form.exp" then
if exp.type == "form.exp" or
exp.type == "form.bracket" then
return {type=exp.type,
s=exp.s,
e=exp.e,
data=exp.data,
subs=shunting_yard_explist(exp.subs)}
elseif exp.type == "form.quantified_exp" then
return {type=exp.type,
s=exp.s,
e=exp.e,
data=exp.data,
subs={shunting_yard_exp(exp.subs[1]),
exp.subs[2]}}
elseif exp.type == "form.cooked" or
exp.type == "form.raw" or
exp.type == "form.bracket" or
exp.type == "form.predicate" then
exp.type == "form.raw" then
return {type=exp.type,
s=exp.s,
e=exp.e,
......@@ -156,6 +167,17 @@ local function shunting_yard_exp(exp, attribute_table)
e=exp.e,
data=exp.data,
subs=newsubs}
elseif exp.type == "form.predicate" then
local n = #exp.subs
local exp_form = exp.subs[n]
local newsubs = {}
for i=1,n-1 do newsubs[i] = exp.subs[i] end
newsubs[n] = shunting_yard_exp(exp_form)
return {type=exp.type,
s=exp.s,
e=exp.e,
data=exp.data,
subs=newsubs}
else
if infix.DEBUG then print("*** skipping: ", exp.type, " ", exp.data) end
return exp
......@@ -234,6 +256,9 @@ local function make_prefix_tree(op, exps)
local attr = OPERATOR_ATTRIBUTES[op.data]
assert(attr)
op.type = attr.nodetype
if infix.DEBUG then
print("result: "); table.print(op)
end
return op
end
......@@ -259,7 +284,7 @@ function shunting_yard_explist(subs, attribute_table)
subs = insert_seq_operators(subs)
if infix.DEBUG then
io.stdout:write(" after inserting seq operators: ")
for _,v in ipairs(subs) do io.stdout:write(v.type, " ", v.data, " ") end
for _,v in ipairs(subs) do io.stdout:write(v.type, " ") end
print()
end
local exps = new_stack()
......@@ -393,17 +418,42 @@ end
function test1(filename)
io.stdout:write("Testing " .. filename .. " ... ")
io.stdout:flush()
-- set globals for ease of debugging
-- NOTE: using globals here, for ease of debugging
src = slurp(filename)
p = stmt:match(src)
ok, x = pcall(infix.to_prefix, p)
ok, pkgname, err = e:load(src)
if ok then
print("ok")
print("OK")
else
print("ERROR:")
error(x, 1)
error("load failed. inspect 'err' for details.")
end
print("(not testing match() yet).")
end
-- -- Parse
-- local messages = {}
-- source = rosie.env.common.source.new{text=src}
-- a = parse_block(source, messages) -- stmt:match()
-- if a then
-- io.stdout:write("parsed with parse_block, ")
-- else
-- error("parse/prefix failed:\n" .. util.table_to_pretty_string(messages))
-- end
-- messages = {}
-- ok = rosie.env.loadpkg.validate_block(a, messages)
-- if ok then
-- io.stdout:write("validated/reorganized, ")
-- else
-- error("validate_block failed: " .. util.table_to_pretty_string(messages))
-- end
-- ok = e.compiler.expand_block(a, e.env, messages)
-- if ok then io.stdout:write("expanded, ") else error("Unknown error!", 1); end
-- if #messages ~= 0 then table.print(messages, 60) end
-- ok, err = pcall(e.compiler.compile_block, a, e.env, nil, messages)
-- if ok then io.stdout:write("loaded, ") else error("Unknown error!", 1); end
-- if #messages ~= 0 then table.print(messages, 60) end
-- -- rplx = e:compile('x')
-- -- if ok then io.stdout:write("compiled, ") else print("ERROR:"); error(err, 1); end
-- print("(not testing match() yet).")
-- end
function test()
test1("/Users/jennings/Projects/rosie-pattern-language/dev/rpl/all.rpl")
......@@ -437,12 +487,17 @@ function test()
end
function setup()
e = rosie.engine.new();
e:import('rosie/rpl_1_3', '.');
exp = e:compile('rpl_expression');
stmt = e:compile('rpl_statements');
import = rosie.import;
infix = dofile("/Users/jennings/Projects/rosie-pattern-language/dev/src/lua/infix.lua")
oe = rosie.engine.new()
assert((oe:import('rosie/rpl_1_3', '.')))
exp = assert((oe:compile('rpl_expression')))
stmt = assert((oe:compile('rpl_statements')))
preparse = assert((oe:compile('preparse')))
e = rosie.engine.new()
parse_block = rosie.env.compile.make_parse_block(preparse, stmt, e.compiler.version, infix.to_prefix)
parse_expression = rosie.env.compile.make_parse_expression(exp, infix.to_prefix)
e.compiler.parse_block = parse_block
e.compiler.parse_expression = parse_expression
end
return infix
......@@ -129,17 +129,10 @@ function p2.make_parse_block(rplx_preparse, rplx_statements, supported_version)
local maj, min, start, err = preparser(src)
if not maj then return nil, {err}, 0; end
-- Input is compatible with what is supported, so we continue parsing
--local t0=os.clock()
local pt, leftover, abend, ttotal, tmatch = rplx_statements:match(src, start)
--print("*** rplx_statements clock time = ", (os.clock()-t0)*1000)
--print("*** total time returned by match: ", ttotal/1000)
--t0=os.clock()
local syntax_errors, n = find_syntax_errors(pt, src)
--print("*** find_syntax_errors clock time = ", (os.clock()-t0)*1000)
--print("*** find_syntax_errors reports #nodes = ", n)
-- FUTURE: If successful, we could do a 'lint' pass to produce warnings, and return
-- them in place of the empty error list in the return values.
-- FUTURE: We could do a 'lint' pass to produce warnings, and return
-- them in (place of the empty?) error list in the return values.
return pt, syntax_errors, leftover
end -- parse_block
end -- make_parse_block
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment