#!/usr/bin/env python

"""
Simple illustration of how to build MaxEnt classifier models using
pdtb_classifier to predict the semantics of Implicit connectives.
"""

__author__ = "Christopher Potts"
__copyright__ = "Copyright 2011, Christopher Potts"
__credits__ = []
__license__ = "Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License: http://creativecommons.org/licenses/by-nc-sa/3.0/"
__version__ = "1.0"
__maintainer__ = "Christopher Potts"
__email__ = "See the author's website"

######################################################################

import re
import csv
import pickle
from collections import defaultdict
from itertools import product
import numpy
from pdtb import CorpusReader
from pdtb_classifier import PdtbClassifier
import wordnet_functions
import iqap_experiments
import review_functions

######################################################################

def word_pair_features(datum, tags=None):
    """
    Create a dictionary feats mapping string-pairs to True, where the
    strings are lemmatized words from the cross-product of the words
    in Arg1 and the words in Arg2. If tags is specified, then we
    filter to just words whose tags are in that set.
    """
    feats = {}
    # Lists of (word, tag) pairs, lemmatized:
    lems1 = datum.arg1_pos(lemmatize=True)
    lems2 = datum.arg2_pos(lemmatize=True)
    # If the user supplied a set of tags, restrict attention to them:
    if tags:
        # Downcase to ensure consistency with what the lemmatizing does:
        tags = map(str.lower, tags)
        # Filter the lists:
        lems1 = filter((lambda x : x[1] in tags), lems1)
        lems2 = filter((lambda x : x[1] in tags), lems2)
    # Remove the tags:
    words1 = [lem[0] for lem in lems1]
    words2 = [lem[0] for lem in lems2]
    # Create the cross-product features:
    for w1, w2 in product(words1, words2):
        feats[(w1, w2)] = True
    return feats

def pdtb_predict_implicit_experiment(feature_function):
    """
    Generic experiment code for training, testing, and assessing a classifier.

    Argument:
    feature_function: should be a function taking datum objects as the sole argument
    and returning a dictionary mapping feature names to values.
    """
    # Instantiate the corpus:
    corpus = CorpusReader('pdtb2.csv')
    # Create the classifier instance:
    model = PdtbClassifier(corpus, feature_function, train_percentage=0.3, test_percentage=0.1)
    # Sequence of commands for running the experiment and viewing the output:
    model.train()
    model.test()
    model.print_cm()
    # Some basic work with the classifier attribute:
    print 'Accuracy:', numpy.sum(numpy.diag(model.cm)) / numpy.sum(model.cm)
    # It would be good to continue with precision and recall estimates for each category:
    # ...
    # Look at the most informative features, restricting just to the positive-value estimates:
    model.classifier.show_most_informative_features(n=20, show='pos')

def verb_pairs_experiment():
    """Test a set of features (w1, w2), where w1 and w2 are restricted
    to being (stemmed) verbs or modals."""
    def feature_function(datum):
        return word_pair_features(datum, tags=['md', 'v'])
    pdtb_predict_implicit_experiment(feature_function)

######################################################################
    
def words2inquirer_classes(lems, inquirer):
    """
    Returns the set of semantic classes for the lemmas in lems,
    according to inquirer, the dictionary in harvard_inquirer.pickle.
    """
    classes = []
    for lem in lems:
        classes += inquirer[lem]
    return set(classes)

def inquirer_features(datum, inquirer):
    """
    Creates features (cls1, cls2) where cls1 and cls2 are drawn from
    the cross-product of semantic classes represented in Arg1 and
    Arg2.
    """
    feats = {}
    # Lemmatize Arg1 and get its associated Inquirer classes:
    lems1 = datum.arg1_pos(lemmatize=True)
    classes1 = words2inquirer_classes(lems1, inquirer)
    # Lemmatize Arg2 and get its associated Inquirer classes:
    lems2 = datum.arg2_pos(lemmatize=True)
    classes2 = words2inquirer_classes(lems2, inquirer)
    # Inquirer class-pair features:
    for cls1, cls2 in product(classes1, classes2):
        feats[(cls1, cls2)] = True
    return feats

def inquirer_pairs_experiment():
    """Test a set of features (cls1, cls2) where cls1 and cls2 are
    drawn from the cross-product of semantic classes represented in
    Arg1 and Arg2."""
    inq = pickle.load(file('harvard_inquirer.pickle'))
    def feature_function(datum):
        return inquirer_features(datum, inq)
    pdtb_predict_implicit_experiment(feature_function)

######################################################################

#SCORER = review_functions.get_all_imdb_scores('imdb-words-assess.csv')

def polarity_counts(lems):
    """
    Auxilary function for polarity_features(). This function sums the
    positive and negative words in a set of lemmas and returns
    (positive, negative), both floats, which are then added to the
    feature dictionary by polarity_features().
    """
    positive = 0
    negative = 0
    for lem in lems:
        lem = wordnet_functions.wordnet_sanitize(lem)
        score = iqap_experiments.coef_score(lem)
        if score and score > 0:
            positive += 1
        elif score and score < 0:
            negative += 1
    return (positive, negative)

def polarity_features(datum):
    """Polarity features using the IMDB review data."""
    feats = {
        'Arg1_Positive': 0, 'Arg1_Negative': 0,
        'Arg2_Positive': 0, 'Arg2_Negative': 0
        }
    def polarity_arg_features(lems, index, feats):
        pos, neg = polarity_counts(lems)
        feats['Arg%s_Positive' % index] = pos
        feats['Arg%s_Negative' % index] = neg
        return feats    
    lems1 = datum.arg1_pos(lemmatize=True)
    feats = polarity_arg_features(lems1, 1, feats)
    lems2 = datum.arg2_pos(lemmatize=True)
    feats = polarity_arg_features(lems2, 2, feats)
    return feats