Commit b3c6f0ae authored by Arne Köhn's avatar Arne Köhn

training implemented

parent 30396e6a
......@@ -10,6 +10,7 @@ use strict;
use Getopt::Long;
use Curses::UI;
use threads;
use threads::shared;
use Thread::Queue;
# This imports POSTaggerEvaluator::NegraProcess
require 'negra-sentence-selector.pl';
......@@ -94,34 +95,16 @@ my $tsetbox = $win1->add(
-vscrollbar => 'right',
);
## BEGIN JOBMANAGEMENT
my $running_jobs = Thread::Queue->new();
my $pending_jobs = Thread::Queue->new();
my $job_results = Thread::Queue->new();
my $free_hosts = Thread::Queue->new();
sub manage_jobs {
my $num_pending = $pending_jobs->pending();
while ( $num_pending and $free_hosts->pending()) {
my $host = $free_hosts->dequeue();
my %job = $pending_jobs->dequeue();
threads->create(\&train_tagger, $job{'tagger'}, $job{'tset'}, $host);
$running_jobs->enqueue(%job);
$num_pending = $pending_jobs->pending();
}
my $num_running = $running_jobs->pending();
if ( $num_pending or $num_running) {
$statuslabel->text("$num_running Jobs running, $num_pending Jobs pending,");
} else {
$statuslabel->text("No jobs running...");
}
while ( $job_results->pending()) {
my %job = $job_results->dequeue();
$free_hosts->enqueue($job{'host'});
}
}
$cui->add_callback ('manage_jobs',\&manage_jobs);
## END JOBMANAGEMENT
my $buttonbox = $win1->add(
'buttonbox', 'Buttonbox',
-buttons => [
{
-label => 'train',
-onpress => \&schedule_train,
}
],
-y => 22,
);
......@@ -146,16 +129,60 @@ $dbh->do('CREATE TABLE IF NOT EXISTS tsets
numtrain INTEGER,
created_on DEFAULT CURRENT_TIMESTAMP,
infoblob DEFAULT NULL);');
$dbh->do('CREATE TABLE IF NOT EXISTS taggers
(id INTEGER PRIMARY KEY ASC, name);');
# We don't need this table for now - maybe add it later again.
#$dbh->do('CREATE TABLE IF NOT EXISTS taggers
# (id INTEGER PRIMARY KEY ASC, name);');
$dbh->do('CREATE TABLE IF NOT EXISTS train
(id INTEGER PRIMARY KEY ASC,
tagger INTEGER,
tagger TEXT,
tset INTEGER);');
my @taggers;
@taggers=split(/,/,$cfg{'taggers'});
## BEGIN JOBMANAGEMENT
my $num_running :shared = 0;
my $pending_jobs = Thread::Queue->new();
my $job_results = Thread::Queue->new();
my $free_hosts = Thread::Queue->new();
# We keep track of all threads to join them later.
my @threadlist;
sub manage_jobs {
my $num_pending = $pending_jobs->pending();
while ( $num_pending and $free_hosts->pending()) {
my $host = $free_hosts->dequeue();
my $job = $pending_jobs->dequeue();
push @threadlist,
threads->create(\&train_tagger, $job->{'tagger'}, $job->{'tset'}, $host);
$num_running++;
$num_pending = $pending_jobs->pending();
}
if ( $num_pending or $num_running) {
$statuslabel->text("$num_running Jobs running, $num_pending Jobs pending,");
} else {
$statuslabel->text("No jobs running...");
while (my $t = pop @threadlist) {
$t->join();
}
}
while ( $job_results->pending()) {
my $job = $job_results->dequeue();
$free_hosts->enqueue($job->{'host'});
$num_running--;
my $tagger = $job->{'tagger'};
my $tset = $job->{'tset'};
$dbh->do("INSERT INTO train (tagger, tset) VALUES ('$tagger', $tset);");
die "negative running jobs?!" if ( $num_running<0);
}
}
$cui->set_timer('manage_jobs',\&manage_jobs,1);
## END JOBMANAGEMENT
$taggerbox->values(@taggers);
$free_hosts->enqueue(split(/,/,$cfg{'hosts'}));
......@@ -175,16 +202,27 @@ sub update_tsetbox {
update_tsetbox();
sub schedule_train {
foreach my $tagger ($taggerbox->get() ) {
foreach my $tset ($tsetbox->get()) {
$pending_jobs->enqueue({'tagger'=>$tagger, 'tset'=>$tset});
}
}
manage_jobs();
}
sub train_tagger {
my $tagger = shift;
my $tset = shift;
my $host = shift;
if ( $host eq 'local') {
# TODO: measure time of execution
`taggers/$tagger/train tfiles/$tset/train`;
$job_results->enqueue(($tagger, $tset, $host));
my $command = "sh taggers/$tagger/train \$(pwd)/tfiles/$tset/train $tset";
print STDERR "running ",$command;
`$command`;
$job_results->enqueue({'tagger'=>$tagger,'tset'=> $tset,'host'=> $host});
} else {
die "train not yet implemented";
die "remote train not yet implemented";
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment