#!/usr/bin/perl -w

use strict;

# Check a TAC 2014 KBP Track Slot Filler Validation task submission for various common errors:
#      * missing SFV query ID (exactly one judgment required per SFV query ID)
#      * invalid SFV query ID
#      * invalid judgment (must be -1 or 1)

# Usage is:
#    check_kbp2014_slot-filler-validation.pl results_file
#  where results_file is the name of the results file to be checked

# Results input file is in the form 
#   <SFV_query_id> <judgment>

# Messages regarding submission are printed to an error log

# Change this variable to the directory where the error log should be put
my $errlog_dir = ".";

my $MAX_ERRORS = 25;

# Array containing number of judgments returned for each SFV query id.
# These query id values are specific to the TAC 2014 KBP track.
# SFV query id is of the form "SFV2014_x_y"; where x and y are positive integers
my %SFV_queries;

# number of candidate fillers in each SF run
my %SF_runs = ("SFV2014_01_2" => 696, 
    "SFV2014_01_3" => 652, 
    "SFV2014_01_4" => 904, 
    "SFV2014_01_5" => 833, 
    "SFV2014_02_1" => 513, 
    "SFV2014_02_2" => 489, 
    "SFV2014_02_3" => 499, 
    "SFV2014_03_1" => 509, 
    "SFV2014_03_2" => 456, 
    "SFV2014_03_3" => 795, 
    "SFV2014_03_4" => 748, 
    "SFV2014_03_5" => 902, 
    "SFV2014_04_1" => 412, 
    "SFV2014_04_2" => 412, 
    "SFV2014_04_3" => 402, 
    "SFV2014_04_5" => 372, 
    "SFV2014_05_1" => 287, 
    "SFV2014_05_2" => 824, 
    "SFV2014_05_3" => 1355, 
    "SFV2014_05_4" => 648, 
    "SFV2014_05_5" => 1660, 
    "SFV2014_06_1" => 1132, 
    "SFV2014_06_2" => 1127, 
    "SFV2014_06_3" => 860, 
    "SFV2014_06_4" => 543, 
    "SFV2014_06_5" => 1650, 
    "SFV2014_07_1" => 1241, 
    "SFV2014_08_1" => 658, 
    "SFV2014_08_2" => 602, 
    "SFV2014_08_3" => 633, 
    "SFV2014_08_4" => 615, 
    "SFV2014_08_5" => 655, 
    "SFV2014_09_1" => 192, 
    "SFV2014_09_2" => 146, 
    "SFV2014_09_3" => 146, 
    "SFV2014_09_4" => 49, 
    "SFV2014_09_5" => 246, 
    "SFV2014_10_1" => 26, 
    "SFV2014_11_1" => 811, 
    "SFV2014_11_2" => 768, 
    "SFV2014_11_3" => 757, 
    "SFV2014_11_4" => 547, 
    "SFV2014_12_1" => 727, 
    "SFV2014_12_2" => 863, 
    "SFV2014_12_3" => 568, 
    "SFV2014_12_4" => 600, 
    "SFV2014_13_1" => 95, 
    "SFV2014_13_2" => 823, 
    "SFV2014_13_3" => 76, 
    "SFV2014_13_4" => 199, 
    "SFV2014_13_5" => 282, 
    "SFV2014_14_1" => 523, 
    "SFV2014_15_1" => 592, 
    "SFV2014_15_2" => 568, 
    "SFV2014_15_3" => 396, 
    "SFV2014_15_4" => 374, 
    "SFV2014_15_5" => 518, 
    "SFV2014_16_1" => 808, 
    "SFV2014_17_1" => 663, 
    "SFV2014_17_2" => 1003, 
    "SFV2014_17_3" => 2035, 
    "SFV2014_17_4" => 1955, 
    "SFV2014_18_1" => 152, 
    "SFV2014_18_2" => 83, 
    "SFV2014_18_3" => 84);

my $results_file;		# input file name
my ($errlog,$num_errors,$line_num);
my ($id, $run_id, $judgment);
my ($i, $last_i, $line);


if ($#ARGV != 0) {
    print STDERR "Usage: $0 resultsfile \n";
    die "\n";
}

$results_file = $ARGV[0];
$num_errors = 0;

# set up output files 
$last_i = -1;
while ( ($i=index($results_file,"/",$last_i+1)) > -1) {
    $last_i = $i;
}
$errlog = $errlog_dir . "/" . substr($results_file,$last_i+1) . ".errlog";
open ERRLOG, ">$errlog" ||
	die "Cannot open error log for writing\n";

foreach $run_id (keys %SF_runs) {
    foreach $i (1 .. $SF_runs{$run_id}) {
	$id = $run_id . "_$i";
	$SFV_queries{$id} = 0;
    }
}

# The submission file for the KBP slot-filler validation task has at exactly one response for
# each SFV query ID.
# A response line is of the form
#    <SFV_query_id> <judgment>

open RESULTS, "<$results_file" ||
    die "Unable to open results file $results_file: $!";
$line_num = 0;
while ($line = <RESULTS>) {
    chomp $line;
    $line_num++;

    next if ($line =~ /^\s*$/);

    $line =~ s/^\s*(.*\S)\s*$/$1/g;
    undef $id,
    undef $judgment;
    ($id, $judgment) = split " ", $line, 2;
    
    if (!defined $judgment || length($judgment) == 0) {
	&error("Wrong number of fields");
	next;
    }

    # check that SFV query ID is valid
    if (!defined $SFV_queries{$id}) {
	&error("unrecognizable SFV query ID `$id'");
	next;
    }

    # check that judgment is valid
    if ($judgment != 1 and $judgment != -1) {
	&error("Unrecognizable judgment `$judgment'");
	next;
    }

    $SFV_queries{$id} += 1;	

}


# Do global check:
#   error if we don't have exactly one judgment for each SFV query
foreach $id (keys %SFV_queries) {
    if ($SFV_queries{$id} == 0) {
	&error("Missing response for SFV query $id");
    } elsif ($SFV_queries{$id} > 1) {
	&error("Too many responses ($SFV_queries{$id}) for query $id");
    }
}
print ERRLOG "Finished processing $results_file\n";

close ERRLOG || die "Close failed for error log $errlog: $!\n";
if ($num_errors) { exit 255; }
exit 0;


 # print error message, keeping track of total number of errors
sub error {
   my $msg_string = pop(@_);

    print ERRLOG 
    "$0 $results_file: Error on line $line_num --- $msg_string\n";

    $num_errors++;
    if ($num_errors > $MAX_ERRORS) {
        print ERRLOG "$0 of $results_file: Quit. Too many errors!\n";
        close ERRLOG ||
		die "Close failed for error log $errlog: $!\n";
	exit 255;
    }
}
