# This zymake script runs all experiments for (Breck, Choi & Cardie '07)

# data inputs not provided:
# .db database files
# document lists
# baseline "tff" files

# basic features, not including dictionaries
basefeatures=#indse w-4 w-3 w-2 w-1 w0 w1 w2 w3 w4 pos prev1cat next1cat node0cat

# "comment" features, used for evaluation and debugging
commentfeatures=#ID #origid #fold #endsent #indse #inese #indse_ese

# use slightly modified version of MALLET, hacked to use at least 200 iterations
# always
mallet=/home/ebreck/misc/unixpkg/mallet-0.3.2a

# Use the SimpleTagger interface to MALLET
simpletagger=java -Xmx1000m -cp $(mallet)/class:$(mallet)/lib/mallet-deps.jar
edu.umass.cs.mallet.base.fst.SimpleTagger

# feature infrastructure
inf=/home/ebreck/misc/code/inf

# this directory contains mpqa data used in these experiments
mpqa=/home/ebreck/misc/rsrch/mpqa/fa04

## begin build rules

# create the master-attribute file for the features
$(inf)/mkattr.py -p doclists/469
$(mpqa)/newmkdb/alldbs 'select ind as inst from ord_token'
doclists/TrainDocs.[0-9] doclists/69 > $().attr

# FIXME: including 69 above is a bug from the results in the published
# version of the paper; it should only have the effect of slightly lowering
# the precision of the baseline results.

# create a feature column for feature
$(inf)/mkfeat.py $().attr 'queries/$(fname).query' '$(fname)' $(>).f

# create feature vectors
$(inf)/mkvecs.py $().attr $(>).v $(fname = * features).f

# Remove comments and fold K
grep -v '#fold K' $().v/svm | grep -v '^#' > $().nz-svm

# Create test data
grep "fold [$(fold)]" $().nz-svm | perl -ane '$eos = /#endsent/; s/#.*//; 
@F=split; print "@F[1..$#F]\n"; print "\n" if $eos' >$().mallet-test

# Make training file
grep -v "fold $(fold)" $().nz-svm | perl -ane '$eos = /#endsent/; 
$class = m{#in'$(class)' }?"pos":"neg"; s/#.*//; @F=split; 
print "@F[1..$#F] $class\n"; print "\n" if $eos' >$(train="2way").mallet-train

# Create training file, both classes at once
grep -v "fold $(fold)" $().nz-svm | perl -ane '$eos = /#endsent/; 
$class=m{#indse }?"dse":m{#inese }?"ese":"neg"; s/#.*//; @F=split; 
print "@F[1..$#F] $class\n"; print "\n" if $eos' >$(train="3way").mallet-train

# Train CRF
$(simpletagger) --train true --gaussian-variance 0.25 --orders $(order) 
--training-proportion 1.0 --default-label neg --model-file $(>).mallet-model 
$().mallet-train 2>$().mallet-log >$().mallet-train-stdout

# Predict with CRF
$(simpletagger) --model-file $().mallet-model $().mallet-test > $().mallet-out

# Convert MALLET output to what looks like SVM-light output
grep . $().mallet-out | perl -ne 'print m{pos}?1:-1,"\n"' 
> $(train="2way").svm-out

# Convert MALLET output to what looks like SVM-light output
grep . $().mallet-out | perl -ne 'print m{^dse}?1:-1,"\n"' 
> $(train="3way" class="dse").svm-out

# Convert MALLET output to what looks like SVM-light output
grep . $().mallet-out | perl -ne 'print m{^ese}?1:-1,"\n"' 
>$(train="3way" class="ese").svm-out

# Convert MALLET output to what looks like SVM-light output
grep . $().mallet-out | perl -ne 'print m{^[de]se}?1:-1,"\n"' 
>$(train="3way" class="dse_ese").svm-out

#### baselines

# this TFF list comes from an e-mail from Jan, Mar 29, 2006, it's from
# experiments reported in Wiebe & Riloff Cicling 2005
echo 'basilisk-2000nouns-strongsubj basilisk-2000nouns-weaksubj
metaboot-2000nouns-strongsubj metaboot-2000nouns-weaksubj lev-fn-psych-plus2
cl02pses PolMman PolPman ballmer-se2JWman weakjandiss strongjandiss trainJWman'
| tr ' ' '\n' | xargs -i cat ~nlp/projects/mpqa/lib/{}.tff 
| grep -v 'type=\(dsadj\|dsverb\|fixed2gram\|fixed3gram\)' > $(list="wiebe").tff

# this TFF list comes from an e-mail from Theresa, Mar 29, 2006, from
# HLT-EMNLP-2005
cp /home/ebreck/misc/unixpkg/hltemnlp05-twilson-clues/subjclueslen1-HLTEMNLP05.tff $(>list="twilson").tff

# this TFF list comes from an e-mail from Jan to Richard Calvi & Claire,
# which I got forwarded from Claire 2/27/2006
cp /home/nlp/projects/mpqa/lib/wiebe_wordlist_2005-08-02.tff
$(>list="wiebe1").tff

# this is the baseline used in earlier DSESE experiments
grep 'type=\(bl_str_se_verb\|fn_communication_statement_se_v\)' ~nlp/projects/mpqa/lib/lev-fn-se1.tff >$(list="oldbase").tff

# these features are used to run the tff files
baselinefeatures=#inese w0 w1 w2 w3 w4 w5 p0 p1 p2 p3 p4 p5 s0 s1 s2 s3 s4 s5 $(*commentfeatures)

# had to fix typo in ~nlp/projects/mpqa/lib/trainJWman.tff - two lines word=
# had to fix typo in theresa's.tff two lines had 'stemmed1=n m'

classtff.py -f "#in$(class) " $().tff $().v >$().tff-pruned

runtff.py $().tff-pruned $().v >$().tff-out 2>$().tfflog

# baseline models predict on all folds; create 1-fold evaluation file
abut $().tff-out $().nz-svm | grep "fold $(fold)" > $(learned=0).eval-in

# learned models predict only on one fold; create a 1-fold evaluation file
grep "fold $(fold)" $().nz-svm | abut $().svm-out - > $(learned=1).eval-in

# Evaluate
eval.py -f 'in'$(class) < $().eval-in > $().eval

# average together results for all folds
aveeval.py 'overlap recall' 'overlap precision' 'overlap F'
'exact recall' 'exact precision' 'exact F' -
$(fold = * (range "A" "J")).eval > $().rpf

rpftable.py method $(class)
'Wiebe baseline' $(learned=0 list="wiebe" features=baselinefeatures).rpf
'Wilson baseline' $(learned=0 list="twilson" features=baselinefeatures).rpf
'crf-1-$(classnicename)' $(learned=1 order="0,1" train="2way").rpf
'crf-1-DSE\&ESE' $(learned=1 order="0,1" train="3way").rpf
'crf-0-$(classnicename)' $(learned=1 order="0" train="2way").rpf
'crf-0-DSE\&ESE' $(learned=1 order="0" train="3way").rpf
> $().method

# shorthand to make the following more clear
cp $(learned=1 features=(flatten basefeatures f commentfeatures)).rpf $(>).arpf

rpftable.py 'feature set' ablation
'base' $(f='()).arpf
'base + Levin/FN' $(f='("lev_fn_se")).arpf
'base + Wilson' $(f='("twilson")).arpf
'base + Wilson + Levin/FN' $(f='("twilson" "lev_fn_se")).arpf
'base + WordNet' $(f='("wordnet")).arpf
'base + Wilson + WordNet' $(f='("twilson" "wordnet")).arpf
'base + Levin/FN + WordNet' $(f='("lev_fn_se" "wordnet")).arpf
'base + Levin/FN + WordNet + Wilson' $(f='("lev_fn_se" "wordnet" "twilson")).arpf
> $().ablation

allfeatures=$(*basefeatures) lev_fn_se wordnet twilson $(*commentfeatures)

cat 
$(class="dse" classnicename="DSE" features=allfeatures).method
$(class="ese" classnicename="ESE" features=allfeatures).method
$(class="dse_ese" classnicename="DSE+ESE" features=allfeatures).method
$(class="dse" train="3way" order="0").ablation
> $().tables

# calculate maximum overlap alignment quality; fake dependency on .tables
# so that it runs on all .eval files produced.
calc_max_oaq.py o/*.eval > $().maxoaq # $().tables

cat $().tables $().maxoaq > $().numbers

# main target
: $().numbers