#!/usr/bin/perl -w

use strict;

# Author: Hoa Dang (hoa.dang@nist.gov)

# Date: November 4, 2015
# Version: 1.1
#  
# Check a TAC 2015 KBP Track Slot Filler Validation task submission for various common errors:
#      * invalid SFV query ID
#      * invalid judgment (must be -1 or 1)
#      * too many judgments for the same SFV query ID
#
# Usage is:
#    check_kbp2015_slot-filler-validation.pl task in_dir out_dir results_file
#
# task is either "filtering" or "ensemble"
#     filtering indicates the results file should be treated as a SFV Filtering task submission
#     ensemble indicates the results file should be treated as a SFV Ensemble task submission
#
# in_dir contains the base SF runs that are input to SFV
#
# out_dir is the directory to write to, containing the single ensemble
# SF run (if task is ensemble) or the filtered SF runs (if task is
# filtering) produced by applying the SFV submission to the base SF
# runs in in_dir; out_dir must already exist before running this
# script
#
# results_file is the name of the results file to be checked, and is in the form 
#   <SFV_query_id> <judgment>
# where judgment is either 1 or -1
#
# Messages regarding submission are printed to an error log


# Change this variable to the directory where the error log should be put
my $errlog_dir = ".";

my $MAX_ERRORS = 25;

# Array containing judgment returned for each SFV query id (0 is unjudged).
# These query id values are specific to the TAC 2015 KBP track.
# SFV query id is of the form "SFV2015_(SF|KB)_x_y"; where x and y are positive integers
my %SFV_queries;

# number of candidate fillers in each SF run
my %SF_runs = ("SFV2015_KB_03_1" => 21841,
    "SFV2015_KB_03_2" => 94645,
    "SFV2015_KB_05_1" => 31281,
    "SFV2015_KB_05_2" => 41962,
    "SFV2015_KB_05_3" => 35650,
    "SFV2015_KB_05_4" => 42178,
    "SFV2015_KB_05_5" => 39427,
    "SFV2015_KB_09_1" => 10602,
    "SFV2015_KB_10_1" => 98624,
    "SFV2015_KB_10_2" => 130419,
    "SFV2015_KB_10_3" => 58562,
    "SFV2015_KB_10_4" => 56263,
    "SFV2015_KB_11_1" => 680,
    "SFV2015_KB_11_2" => 34,
    "SFV2015_KB_11_3" => 823,
    "SFV2015_KB_11_4" => 41,
    "SFV2015_KB_12_1" => 118078,
    "SFV2015_KB_12_2" => 46297,
    "SFV2015_KB_12_3" => 117571,
    "SFV2015_KB_12_4" => 184170,
    "SFV2015_KB_12_5" => 105768,
    "SFV2015_KB_16_1" => 39923,
    "SFV2015_KB_16_2" => 26622,
    "SFV2015_KB_16_3" => 23214,
    "SFV2015_KB_16_4" => 40839,
    "SFV2015_KB_16_5" => 27649,
    "SFV2015_SF_01_1" => 19619,
    "SFV2015_SF_01_2" => 13536,
    "SFV2015_SF_02_1" => 32552,
    "SFV2015_SF_02_2" => 3046,
    "SFV2015_SF_02_3" => 19299,
    "SFV2015_SF_02_4" => 8363,
    "SFV2015_SF_03_1" => 37033,
    "SFV2015_SF_03_2" => 23061,
    "SFV2015_SF_03_3" => 62118,
    "SFV2015_SF_03_4" => 21084,
    "SFV2015_SF_04_1" => 19507,
    "SFV2015_SF_04_2" => 16885,
    "SFV2015_SF_04_4" => 4004,
    "SFV2015_SF_04_5" => 2938,
    "SFV2015_SF_06_1" => 8508,
    "SFV2015_SF_07_1" => 10433,
    "SFV2015_SF_07_2" => 7443,
    "SFV2015_SF_07_3" => 10172,
    "SFV2015_SF_07_4" => 7472,
    "SFV2015_SF_07_5" => 5345,
    "SFV2015_SF_08_1" => 7601,
    "SFV2015_SF_08_2" => 38421,
    "SFV2015_SF_08_3" => 42361,
    "SFV2015_SF_08_4" => 38127,
    "SFV2015_SF_08_5" => 58419,
    "SFV2015_SF_10_1" => 45270,
    "SFV2015_SF_10_2" => 127740,
    "SFV2015_SF_10_3" => 62332,
    "SFV2015_SF_10_4" => 37149,
    "SFV2015_SF_10_5" => 44721,
    "SFV2015_SF_13_1" => 38149,
    "SFV2015_SF_13_2" => 15553,
    "SFV2015_SF_13_3" => 18288,
    "SFV2015_SF_13_4" => 26781,
    "SFV2015_SF_13_5" => 26781,
    "SFV2015_SF_14_1" => 2341,
    "SFV2015_SF_17_1" => 617,
    "SFV2015_SF_17_2" => 619,
    "SFV2015_SF_17_3" => 232,
    "SFV2015_SF_18_1" => 32838,
    "SFV2015_SF_18_2" => 33369,
    "SFV2015_SF_18_3" => 36863,
    "SFV2015_SF_18_4" => 48076);

my $results_file;		# input file name
my ($errlog,$num_errors,$line_num);
my ($id, $run_id, $judgment);
my ($i, $last_i, $filter_id, $line, $in_dir, $out_dir);
my ($missing_judgments, $task, $default);
my ($cssf_qid, $slot, $rest);

if ($#ARGV != 3) {
    print STDERR "Usage: $0 task CSRuns_directory output_directory resultsfile \n";
    die "\n";
}

$task = $ARGV[0];
if ($task eq "filtering") {
    $default = 1;
} elsif ($task eq "ensemble") {
    $default = -1;
} else {
    print STDERR "Usage: $0 task CSRuns_directory output_directory resultsfile \n";
    print STDERR "task must be either 'filtering' or 'ensemble'\n";
    die "\n";
}

# Read in CSRuns from CSRuns_directory
$in_dir = $ARGV[1];


$out_dir = $ARGV[2];

$results_file = $ARGV[3];
$num_errors = 0;

# set up output files 
$last_i = -1;
while ( ($i=index($results_file,"/",$last_i+1)) > -1) {
    $last_i = $i;
}
$filter_id = substr($results_file,$last_i+1);  # name of the SFV filter file
$errlog = $errlog_dir . "/" . substr($results_file,$last_i+1) . ".errlog";
open ERRLOG, ">$errlog" ||
	die "Cannot open error log for writing\n";


foreach $run_id (keys %SF_runs) {
    foreach $i (1 .. $SF_runs{$run_id}) {
	$id = $run_id . "_$i";
	$SFV_queries{$id} = 0;
    }
}

# The submission file for the KBP slot-filler validation task has zero or one response for
# each SFV query ID.
# A response line is of the form
#    <SFV_query_id> <judgment>
# where judgment is either 1 or -1

open RESULTS, "<$results_file" ||
    die "Unable to open results file $results_file: $!";
$line_num = 0;
while ($line = <RESULTS>) {
    chomp $line;
    $line_num++;

    next if ($line =~ /^\s*$/);

    $line =~ s/^\s*(.*\S)\s*$/$1/g;
    undef $id,
    undef $judgment;
    ($id, $judgment) = split " ", $line, 2;
    
    if (!defined $judgment || length($judgment) == 0) {
	&error("Wrong number of fields");
	next;
    }

    # check that SFV query ID is valid
    if (!defined $SFV_queries{$id}) {
	&error("unrecognizable SFV query ID `$id'");
	next;
    }

    # check that judgment is valid
    if ($judgment != 1 and $judgment != -1) {
	&error("Unrecognizable judgment `$judgment'");
	next;
    }

    if($SFV_queries{$id} != 0) {
	&error("More than one judgment returned for query $id");
	next;
    }
    $SFV_queries{$id} = $judgment;	

}
close RESULTS || die "Close failed for results file $results_file: $!\n";

# Do global check:
#   warn if we have no judgment for an SFV query
foreach $id (keys %SFV_queries) {
    if ($SFV_queries{$id} == 0) {
	$missing_judgments +=1;
	$SFV_queries{$id} = $default;
    }
}
if ($missing_judgments) {
    print ERRLOG "WARNING: Missing judgments for $missing_judgments SFV query IDs, defaulting to judgment of $default for SFV $task task\n";
}


print ERRLOG "Finished processing $results_file\n";

close ERRLOG || die "Close failed for error log $errlog: $!\n";


if ($num_errors) { exit 255; }

# No errors, so write filtered runs or ensemble run 
if ($task eq "filtering") {
    foreach $run_id (keys %SF_runs) {
	open FILTERED, ">$out_dir/$filter_id.$run_id.filtered" ||
	    die "Cannot open filtered file $out_dir/$filter_id.$run_id.filtered for writing\n";
	open SFRUN, "<$in_dir/$run_id" ||
	    die "Unable to open SF run file $in_dir/$run_id for reading: $!";
	while ($line = <SFRUN>) {
	    chomp $line;
	    next if ($line =~ /^\s*$/);
	    ($cssf_qid, $slot, $id, $rest) = split " ", $line, 4;
	    if (!defined $SFV_queries{$id}) {
		die "undefined SFV query id $id in $in_dir/$run_id\n";
	    }
	    if ($SFV_queries{$id} == 1) {
		$id =~ s/^(.+_\d+_\d+)_\d+$/$1/g;
		print FILTERED "$cssf_qid\t$slot\t$id\t$rest\n";
	    }
	}
        close SFRUN ||
		die "Close failed for SF run file $in_dir/$run_id: $!\n";
        close FILTERED ||
		die "Close failed for filtered file $out_dir/$filter_id.$run_id.filtered: $!\n";

    }
} elsif ($task eq "ensemble") {
    open ENSEMBLE, ">$out_dir/$filter_id.ensemble" ||
	die "Cannot open filtered file $out_dir/$filter_id.ensemble for writing\n";
    foreach $run_id (keys %SF_runs) {
	open SFRUN, "<$in_dir/$run_id" ||
	    die "Unable to open SF run file $in_dir/$run_id for reading: $!";
	while ($line = <SFRUN>) {
	    chomp $line;
	    next if ($line =~ /^\s*$/);
	    ($cssf_qid, $slot, $id, $rest) = split " ", $line, 4;
	    if (!defined $SFV_queries{$id}) {
		die "undefined SFV query id $id in $in_dir/$run_id\n";
	    }
	    if ($SFV_queries{$id} == 1) {
		$id = "ensemble";
		print ENSEMBLE "$cssf_qid\t$slot\t$id\t$rest\n";
	    }
	}
        close SFRUN ||
	    die "Close failed for SF run file $in_dir/$run_id: $!\n";
    }
    close ENSEMBLE ||
	die "Close failed for ensemble file $out_dir/$filter_id.ensemble: $!\n";
}

exit 0;

 # print error message, keeping track of total number of errors
sub error {
   my $msg_string = pop(@_);

    print ERRLOG 
    "$0 $results_file: Error on line $line_num --- $msg_string\n";

    $num_errors++;
    if ($num_errors > $MAX_ERRORS) {
        print ERRLOG "$0 of $results_file: Quit. Too many errors!\n";
        close ERRLOG ||
		die "Close failed for error log $errlog: $!\n";
	exit 255;
    }
}
