Commit e2acb109 authored by gerd's avatar gerd

read config parts from files


git-svn-id: https://gps.dynxs.de/private/svn/app-plasma/trunk@227 55289a75-7b90-4627-9e07-ffb4263930b2
parent 2d3e1244
......@@ -25,28 +25,7 @@ let extract_config cf =
| [nn] ->
nn in
let nn_nodes =
cf#resolve_section nn "node" in
if nn_nodes = [] then
failwith "missing 'namenodes.node' config";
let nn_addrs =
List.map
(fun n ->
try
let p = cf#resolve_parameter n "addr" in
cf#string_param p
with
| Not_found ->
failwith "missing 'addr' parameter in 'namenodes.node'"
)
nn_nodes in
List.iter
(fun addr ->
try ignore(Plasma_util.parse_host_port addr)
with _ ->
failwith ("bad 'addr' parameter in 'namenodes.node': " ^ addr)
)
nn_addrs;
let nn_addrs = Plasma_util.node_list cf nn in
let clustername =
try
......@@ -68,45 +47,39 @@ let extract_config cf =
| [mr] ->
mr in
let mr_task_nodes =
cf#resolve_section mr "task_node" in
if mr_task_nodes = [] then
failwith "missing 'mapred.task_node' config";
let mr_addrs = Plasma_util.node_list cf mr in
let mr_hosts =
List.map
(fun n ->
try
let p = cf#resolve_parameter n "host" in
cf#string_param p
with
| Not_found ->
failwith "missing 'host' parameter in 'mapred.task_node'"
(fun s ->
fst (Plasma_util.parse_host_port s)
)
mr_task_nodes in
mr_addrs in
if mr_hosts = [] then
failwith "missing 'mapred.node' config";
let mr_task_port =
try
let p = cf#resolve_parameter mr "task_port" in
let p = cf#resolve_parameter mr "port" in
cf#int_param p
with
| Not_found ->
failwith "missing 'mapred.task_port' parameter" in
failwith "missing 'mapred.port' parameter" in
let mr_task_tmpdir =
try
let p = cf#resolve_parameter mr "task_tmpdir" in
let p = cf#resolve_parameter mr "tmpdir" in
cf#string_param p
with
| Not_found ->
failwith "missing 'mapred.task_tmpdir' parameter" in
failwith "missing 'mapred.tmpdir' parameter" in
let mr_task_load_limit =
try
let p = cf#resolve_parameter mr "task_load_limit" in
let p = cf#resolve_parameter mr "load_limit" in
cf#float_param p
with
| Not_found ->
failwith "missing 'mapred.task_load_limit' parameter" in
failwith "missing 'mapred.load_limit' parameter" in
let mr_shm_low =
try
......
......@@ -36,12 +36,12 @@ val extract_config : Netplex_types.config_file -> mapred_config
...
}
mapred {
task_node { host = "hostname" }
task_node { host = "hostname" }
node { addr = "hostname" }
node { addr = "hostname" }
...
task_port = 1234;
task_tmpdir = "/somewhere";
task_load_limit = 8.1;
port = 1234;
tmpdir = "/somewhere";
load_limit = 8.1;
}
}
]}
......
......@@ -30,28 +30,7 @@ let extract_node_config (cf:Netplex_types.config_file) =
| _ :: _ :: _ ->
failwith "more than one 'namenodes' config"
| [nn] ->
let nodes =
cf#resolve_section nn "node" in
if nodes = [] then
failwith "missing 'namenodes.node' config";
let addrs =
List.map
(fun n ->
try
let p = cf#resolve_parameter n "addr" in
cf#string_param p
with
| Not_found ->
failwith "missing 'addr' parameter in 'namenodes.node'"
)
nodes in
List.iter
(fun addr ->
try ignore(Plasma_util.parse_host_port addr)
with _ ->
failwith ("bad 'addr' parameter in 'namenodes.node': " ^ addr)
)
addrs;
let addrs = Plasma_util.node_list cf nn in
let timeout =
try
let p = cf#resolve_parameter nn "timeout" in
......@@ -150,26 +129,7 @@ let extract_node_config (cf:Netplex_types.config_file) =
| _ :: _ :: _ ->
failwith "more than one 'datanodes' config"
| [dn] ->
let nodes =
cf#resolve_section dn "node" in
let addrs =
List.map
(fun n ->
try
let p = cf#resolve_parameter n "addr" in
cf#string_param p
with
| Not_found ->
failwith "missing 'addr' parameter in 'datanodes.node'"
)
nodes in
List.iter
(fun addr ->
try ignore(Plasma_util.parse_host_port addr)
with _ ->
failwith ("bad 'addr' parameter in 'datanodes.node': " ^ addr)
)
addrs;
let addrs = Plasma_util.node_list cf dn in
let timeout =
try
let p = cf#resolve_parameter dn "timeout" in
......
......@@ -70,6 +70,13 @@ val extract_node_config : Netplex_types.config_file -> nn_node_config
}
]}
Instead of describing the nodes directly in the config file, one
can also point to external file:
{[ node_list = "filename";
port = <default_port>;
]}
inodecache: must be running on all namenodes on the given port.
The client gets only the inodecache running on the coordinator.
*)
......
......@@ -81,6 +81,85 @@ let fallocate =
(* TODO: support this even if the OS/file system does not have it *)
(* read config file support : *)
let comment_re = Pcre.regexp "[^ \t]*#.*$"
let strip_comments s =
Pcre.qreplace ~rex:comment_re ~templ:"" s
let node_list (cf:Netplex_types.config_file) addr =
let port_opt =
try
let p = cf#resolve_parameter addr "port" in
Some(cf#int_param p)
with Not_found -> None in
let parse_host_port s =
try
parse_host_port s
with _ ->
( match port_opt with
| None ->
failwith("Not in <host>:<port> syntax (no default port given): "
^ s)
| Some p ->
(s,p)
) in
let nodes =
cf#resolve_section addr "node" in
let direct_addrs =
List.map
(fun n ->
try
let p = cf#resolve_parameter n "addr" in
let a = cf#string_param p in
let (h,p) = parse_host_port a in
sprintf "%s:%d" h p
with
| Not_found ->
failwith ("missing 'addr' parameter in '" ^ cf#print n ^ "'")
)
nodes in
let node_list_opt =
try
let p = cf#resolve_parameter addr "node_list" in
Some(cf#string_param p)
with Not_found -> None in
match node_list_opt with
| None ->
if nodes = [] then
failwith ("missing 'node' or 'node_list' in '" ^ cf#print addr ^ "'");
direct_addrs
| Some node_list_file ->
let fn =
if Filename.is_implicit node_list_file then
Filename.dirname cf#filename ^ "/" ^ node_list_file
else
node_list_file in
let f = open_in fn in
let addrs = ref [] in
( try
while true do
let line = input_line f in
let line' = strip_comments line in
if line' <> "" then (
let (h,p) = parse_host_port line' in
addrs := (sprintf "%s:%d" h p) :: !addrs
)
done
with
| End_of_file -> ()
);
close_in f;
direct_addrs @ List.rev !addrs
(**********************************************************************)
(* shared memory *)
(**********************************************************************)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment