#!/usr/bin/perl -w

use strict;

# Check a TAC 2013 KBP Track Slot Filler Validation task submission for various common errors:
#      * missing SFV query ID (exactly one judgment required per SFV query ID)
#      * invalid SFV query ID
#      * invalid judgment (must be -1 or 1)

# Usage is:
#    check_kbp2013_slot-filler-validation.pl results_file
#  where results_file is the name of the results file to be checked

# Results input file is in the form 
#   <SFV_query_id> <judgment>

# Messages regarding submission are printed to an error log

# Change this variable to the directory where the error log should be put
my $errlog_dir = ".";

my $MAX_ERRORS = 25;

# Array containing number of judgments returned for each SFV query id.
# These query id values are specific to the TAC 2012 KBP track.
# SFV query id is of the form "SFV_x_y"; where x and y are positive integers
my %SFV_queries;

# number of candidate fillers in each SF run
my %SF_runs = ("SFV2013_01_1" => 238,
    "SFV2013_01_2" => 195,
    "SFV2013_01_3" => 177,
    "SFV2013_02_1" => 520,
    "SFV2013_02_2" => 614,
    "SFV2013_02_3" => 666,
    "SFV2013_02_4" => 1424,
    "SFV2013_02_5" => 1635,
    "SFV2013_03_1" => 468,
    "SFV2013_03_2" => 486,
    "SFV2013_03_3" => 176,
    "SFV2013_04_1" => 485,
    "SFV2013_04_2" => 456,
    "SFV2013_04_3" => 456,
    "SFV2013_05_1" => 1145,
    "SFV2013_05_2" => 746,
    "SFV2013_05_3" => 1459,
    "SFV2013_05_4" => 1579,
    "SFV2013_05_5" => 1379,
    "SFV2013_06_1" => 1872,
    "SFV2013_06_2" => 639,
    "SFV2013_06_3" => 103,
    "SFV2013_07_1" => 457,
    "SFV2013_08_1" => 1841,
    "SFV2013_08_2" => 2097,
    "SFV2013_08_3" => 1042,
    "SFV2013_08_4" => 1770,
    "SFV2013_08_5" => 1727,
    "SFV2013_09_1" => 1147,
    "SFV2013_09_2" => 1163,
    "SFV2013_09_3" => 1118,
    "SFV2013_09_4" => 1065,
    "SFV2013_09_5" => 1025,
    "SFV2013_10_1" => 1355,
    "SFV2013_10_2" => 1091,
    "SFV2013_10_3" => 544,
    "SFV2013_11_1" => 779,
    "SFV2013_11_2" => 779,
    "SFV2013_12_1" => 2731,
    "SFV2013_13_1" => 2490,
    "SFV2013_14_1" => 1404,
    "SFV2013_14_2" => 939,
    "SFV2013_14_3" => 465,
    "SFV2013_15_1" => 1083,
    "SFV2013_15_2" => 1011,
    "SFV2013_15_3" => 1046,
    "SFV2013_15_4" => 871,
    "SFV2013_15_5" => 887,
    "SFV2013_16_1" => 1920,
    "SFV2013_17_1" => 603,
    "SFV2013_17_2" => 473,
    "SFV2013_18_1" => 800);

my $results_file;		# input file name
my ($errlog,$num_errors,$line_num);
my ($id, $run_id, $judgment);
my ($i, $last_i, $line);


if ($#ARGV != 0) {
    print STDERR "Usage: $0 resultsfile \n";
    die "\n";
}

$results_file = $ARGV[0];
$num_errors = 0;

# set up output files 
$last_i = -1;
while ( ($i=index($results_file,"/",$last_i+1)) > -1) {
    $last_i = $i;
}
$errlog = $errlog_dir . "/" . substr($results_file,$last_i+1) . ".errlog";
open ERRLOG, ">$errlog" ||
	die "Cannot open error log for writing\n";

foreach $run_id (keys %SF_runs) {
    foreach $i (1 .. $SF_runs{$run_id}) {
	$id = $run_id . "_$i";
	$SFV_queries{$id} = 0;
    }
}

# The submission file for the KBP slot-filler validation task has at exactly one response for
# each SFV query ID.
# A response line is of the form
#    <SFV_query_id> <judgment>

open RESULTS, "<$results_file" ||
    die "Unable to open results file $results_file: $!";
$line_num = 0;
while ($line = <RESULTS>) {
    chomp $line;
    $line_num++;

    next if ($line =~ /^\s*$/);

    $line =~ s/^\s*(.*\S)\s*$/$1/g;
    undef $id,
    undef $judgment;
    ($id, $judgment) = split " ", $line, 2;
    
    if (!defined $judgment || length($judgment) == 0) {
	&error("Wrong number of fields");
	next;
    }

    # check that SFV query ID is valid
    if (!defined $SFV_queries{$id}) {
	&error("unrecognizable SFV query ID `$id'");
	next;
    }

    # check that judgment is valid
    if ($judgment != 1 and $judgment != -1) {
	&error("Unrecognizable judgment `$judgment'");
	next;
    }

    $SFV_queries{$id} += 1;	

}


# Do global check:
#   error if we don't have exactly one judgment for each SFV query
foreach $id (keys %SFV_queries) {
    if ($SFV_queries{$id} == 0) {
	&error("Missing response for SFV query $id");
    } elsif ($SFV_queries{$id} > 1) {
	&error("Too many responses ($SFV_queries{$id}) for query $id");
    }
}
print ERRLOG "Finished processing $results_file\n";

close ERRLOG || die "Close failed for error log $errlog: $!\n";
if ($num_errors) { exit 255; }
exit 0;


 # print error message, keeping track of total number of errors
sub error {
   my $msg_string = pop(@_);

    print ERRLOG 
    "$0 $results_file: Error on line $line_num --- $msg_string\n";

    $num_errors++;
    if ($num_errors > $MAX_ERRORS) {
        print ERRLOG "$0 of $results_file: Quit. Too many errors!\n";
        close ERRLOG ||
		die "Close failed for error log $errlog: $!\n";
	exit 255;
    }
}
