import spacy
from spacy_wordnet.wordnet_annotator import WordnetAnnotator
from spacy.lang.en.stop_words import STOP_WORDS
import numpy as np
import math
from typing import Dict
import gensim.downloader as api
from gensim import corpora, models, similarities
from gensim.test.utils import get_tmpfile, datapath
from gensim.models import KeyedVectors
from rich import print
import requests
import json
from database.repository_base import RepositoryBase
from nltk.sentiment.vader import SentimentIntensityAnalyzer
import textstat
import nltk
nltk.download('vader_lexicon')
from textblob import TextBlob



nlp = spacy.load("en_core_web_lg")

#Get gensim semantic similarity word vector model
word_vectors = api.load("glove-wiki-gigaword-100")
fname = get_tmpfile("vectors.kv")
word_vectors.save(fname)
word_vectors = KeyedVectors.load(fname, mmap='r')
wv_from_text = KeyedVectors.load_word2vec_format(datapath('word2vec_pre_kv_c'), binary=False)  # C text format
wv_from_bin = KeyedVectors.load_word2vec_format(datapath("euclidean_vectors.bin"), binary=True)  # C bin format

# Get database instance
db = RepositoryBase()



# arg string text
# lemmatizes the stext
# returns string new_string
def lemmatize(text):
    doc = nlp(text)
    new_list = [token.lemma_ for token in doc]
    new_string = " ".join(new_list)

    return new_string


# arg string text
# removes stopwords
# returns string new_string
def remove_stopwords(text):
    new_list = []
    doc = nlp(text)

    tokens = [token.text for token in doc]

    for token in tokens:
        lexeme = nlp.vocab[token]
        if lexeme.is_stop == False:
            new_list.append(token)

    new_string = " ".join(new_list)

    return new_string


# arg string text
# removes punctuation
# returns string new_string
def remove_punctuation(text):
    new_list = []
    doc = nlp(text)

    tokens = [token.text for token in doc]

    for token in tokens:
        lexeme = nlp.vocab[token]
        if lexeme.is_punct == False:
            new_list.append(token)

    new_string = " ".join(new_list)

    return new_string


# arg string word, string topic
# computes term frequency of a word in a topic
# returns float term_frequency
def tf(word, topic):
    word_count = 0
    total_words = 0
    doc = nlp(topic)

    for token in doc:
        if token.text == word:
            word_count += 1
        total_words += 1

    term_frequency = word_count / total_words

    return term_frequency


# arg string word, string clean_rulebook
# computes inverse document frequency of a word in a rulebook
# returns float inverse document frequency
def idf(word, clean_rulebook):
    key_count = 0  # number of keys in the rulebook
    word_count = 0  # number keys containing the word

    for key in clean_rulebook:
        key_count += 1
        if word in clean_rulebook[key]:
            word_count += 1

    # checks for division by 0
    if word_count <= 0:
        print("Error: Incorrect rulebook or word detected!")
        return -1

    inverse_document_frequency = math.log10(key_count / word_count)

    return inverse_document_frequency


# arg string text, list of tuples (string keyword, float tfidf)
# adds each keyword tfidf found in a topic
# returns float count
def keyword_count(text, tfidf_keywords):
    count = 0
    doc = nlp(text)

    for token in doc:
        for val in tfidf_keywords:
            if token.text == val[0]:
                count += val[1]

    return count


# arg string id
# creates a rulebook defined by a dictionary topic:answer
# returns dict rulebook string key: string value
def create_rulebook(question_id):
    rulebook = {}

    results = db.find("trainingdata", {"question_id": question_id})

    for data in results:
        topic = data['topic']
        answer = data['answer']

        if topic != 'passage' and topic != 'question':
            rulebook.update({topic: answer})

    return rulebook


# arg dictionary rulebook string key: string value
# Removes stopwords, punctuation. Applies lemmatization and lower case to words
# returns dictionary new_rulebook string key: string value
def clean_rulebook(rulebook):
    clean_rulebook = {}

    for key in rulebook:
        new_string = remove_punctuation(rulebook[key])
        new_string = remove_stopwords(new_string)
        new_string = lemmatize(new_string)
        new_string = new_string.lower()
        clean_rulebook.update({key: new_string})

    return clean_rulebook


# arg dictionary rulebook string key: string value
# creates a rulebook with keywords
# returns dict rulebook string key: list of string values
def create_keywords_rulebook(clean_rulebook):
    keywords_rulebook = {}

    for key in clean_rulebook:
        keyword_list = []
        doc = nlp(clean_rulebook[key])

        for token in doc:
            keyword_list.append(token.text)

        keywords_rulebook.update({key: keyword_list})

    return keywords_rulebook


# args dict clean_rulebook string key: string value, dict keywords_rulebook string key: list of string values
# creates a rulebook with list of tuple values (keyword, tfidf)
# returns dict tfidf_rulebook string key: list of tuples (string keywords, float tfidf)
def create_tfidf_rulebook(clean_rulebook, keywords_rulebook):
    tfidf_rulebook = {}

    for topic in keywords_rulebook:
        tfidf_list = []

        for word in keywords_rulebook[topic]:
            tfidf = tf(word, clean_rulebook[topic]) * idf(word, clean_rulebook)
            tfidf_list.append((word, tfidf))

        tfidf_rulebook.update({topic: tfidf_list})

    return tfidf_rulebook


# arg string word
# returns list of string synonyms
# TODO Fix function so that the list doesn't contain duplicate words
def synonyms(word, pos):
    nlp.add_pipe("spacy_wordnet", after='tagger', config={'lang': nlp.lang})
    token = nlp(word)[0]
    # returns POS tag + lemma
    lemmas = token._.wordnet.lemmas()
    print(lemmas)
    # cleaning output:
    synlist = []
    for w in range(len(lemmas)):

        lemma = repr(lemmas[w]).split(".")  # Lemma('issue.n.01.issue')
        for ch in lemma[1]:
            if (ch.isalnum() and ch == pos):
                s = ''.join(ch for ch in lemma[3] if ch.isalnum())  # Synonym
                if s not in synlist:
                    synlist.append(s)

    # ms = nlp.vocab.vectors.most_similar(
    # np.asarray([nlp.vocab.vectors[nlp.vocab.strings[word]]]))
    # words = [nlp.vocab.strings[w] for w in ms[0][0]]
    for set in synlist:
        print(set)
    # return set(synlist + list(([w.lower() for w in words])))
    return (synlist)


# arg string stu_answer, string pro_answer
# unfinished function
# TODO implement
def compare_answers(std_answer, pro_answer):
    new_stu_answer = remove_stopwords(std_answer)
    new_pro_answer = remove_stopwords(pro_answer)
    new_stu_answer = remove_punctuation(std_answer)
    new_pro_answer = remove_punctuation(pro_answer)
    new_stu_answer = lemmatize(std_answer)
    new_pro_answer = lemmatize(pro_answer)

    doc1 = nlp(new_stu_answer)
    doc2 = nlp(new_pro_answer)

    return doc1.similarity(doc2)


# arg dict rulebook string key: string value
# returns string professor_answer
def create_prof_answer(rulebook):
    professor_answer = ""

    for key in rulebook:
        professor_answer += rulebook[key]

    return professor_answer


# arg string std_answer, dict topic: list of string keywords
# returns a dictionary with each sentence of the student answer labeled
def identify_topics_tfidf(std_answer, tfidf_rulebook):
    doc = nlp(std_answer)
    std_topics = {}
    std_count = {}

    for sentence in doc.sents:
        for topic in tfidf_rulebook:
            std_count.update({topic: keyword_count(sentence.text, tfidf_rulebook[topic])})

        a_topic = max(std_count, key=std_count.get)

        if a_topic in std_topics:
            std_topics[a_topic].append(sentence.text)
        else:
            std_topics.update({a_topic: [sentence.text]})

    return std_topics


# converts a string into NLP doc object
# returns an array of sentences as strings
def getSentences(text):
    doc = nlp(text)
    sent_list = []
    for sent in doc.sents:
        sent_list.append(sent.text + " ")
    return sent_list


#returns a list of grammatical errors, their location, and corrections
def calc_grammar(text):
    api_url = 'https://api.languagetoolplus.com/v2/check'
    payload = dict(text=text, language='en-US', motherTongue='en-US', enabledOnly='false', level='picky')
    response = requests.post(api_url, payload)
    json_response = json.loads(response.text)
    matches = json_response["matches"]
    return matches


def format_data(question_id):
    formatted_data: Dict = {}

    formatted_data = {
        'question_id': question_id,
        'passage': '',  # full answer
        'topics': {}  # topics: {'issue': [<<Array of sentences>>]}
    }

    results = db.find("trainingdata", {"question_id": question_id})

    for data in results:
        topic = data['topic']
        answer = data['answer']

        if topic == 'passage':
            formatted_data['passage'] += answer
        elif topic != 'passage' and topic != 'question':
            print('got here')
            sentences = getSentences(answer)
            formatted_data['topics'].update({topic: []})
            for sent in sentences:
                formatted_data["topics"][topic].append(sent)

    return formatted_data


#Semantic similarity using Gensim word2vec model
#Calculates distance between word vectors
#Lower score = more similar
def similarity(text1, text2):
    text1 = text1.lower().split()
    text2 = text2.lower().split()
    return word_vectors.wmdistance(text1, text2)


#Uses semantic similarity to identify topics by tagging sentences with the highest 
#similarity to one's in the rulebook.
def identify_topics(std_answer, prof_answer_data):

    print('printing rulebook from aeg.identify_topics()')
    print(prof_answer_data['topics'])

    # Topics identified:
    std_topics = []
    identified = []
    std_sentences = getSentences(std_answer)
    # Topics to identify:
    topics = list(prof_answer_data['topics'].keys())

    for sent in std_sentences:
        min = 0.6
        label = ''
        sim = 0
        for key in prof_answer_data['topics']:
            for topic_sent in prof_answer_data['topics'][key]["sentence"]:
                wv_distance = similarity(sent, topic_sent)
                if (wv_distance < min):
                    label = key
                    min = wv_distance
                    sim = min
                    print(prof_answer_data['topics'][key]["sentence"])

                print(wv_distance)
                print(sent)
                print()

        if label != '':
            std_topics.append([sent, label, sim])
            identified.append(label)
        else:
            std_topics.append([sent, '', 0])

    # Calculates which topics are not identified.
    topics = list(set(topics) - set(identified))

    return {'topics': std_topics, 'missing': topics}



#Determines whether a text has a neutral, positive, or negative tone.
def sentiment_analyzer(student_answer):  
    vader = SentimentIntensityAnalyzer()  
    results = vader.polarity_scores(student_answer)      
    scores_list = [results['neg'],results['pos'],results['neu']]    
    print(results)
    if max(scores_list) == results['neg']:
        return {'neg' :results['neg']}

    elif max(scores_list) == results['pos']:
        return {'pos' :results['pos']}
    
    elif max(scores_list) == results['neu']:
        return {'neu' :results['neu']}



# calculate student readability using flesch reading ease and store result and suggestions as json object
def readability(text):
    readability = textstat.flesch_reading_ease(text)
    if readability > 70:
        if readability > 100:
            readability = 100
        suggestion = "Your answer is easy to read, however it is too simple for your intended audience. Please try to expand your answer with more technical language."
    elif int(readability) in range (30,70):
        suggestion = "Your answer is readable and appropriate for your intended audience."
    else:
        suggestion = "Your answer is appropriate for your target audience, but difficult to read. Consider simplifying and improving the clarity of your answer."
    scoring = {
        'score': int(readability),
        'suggestion': suggestion
    }
    return scoring

# uses language tool to check for errors relating to conciseness, then returns the number of relevant errors
def conciseness(text):
    api_url = 'https://api.languagetoolplus.com/v2/check'
    # enabledOnly not set to true as it fails to return any issues
    payload = dict(text = text, language = 'en-US', motherTongue = 'en-US', enabledRules = ['BE_A_X_ONE','DUE_TO_THE_FACT','EN_PLAIN_ENGLISH_REPLACE',], enabledCategories = ['REDUNDANCY'], enabledOnly = 'false', level = 'picky')
    response = requests.post(api_url , payload)
    json_response = json.loads(response.text)
    matches = json_response["matches"]
    i = 0
    for error in matches:
        if error['rule']['id'] in ("EN_PLAIN_ENGLISH_REPLACE", "IN_X_ORDER", "DUE_TO_THE_FACT", "TOO_LONG_SENTENCE") or error['rule']['category']['id'] == 'REDUNDANCY':
            i = i+1
    return i
    
#calculate the academic tone of the student answer
# need to install  
# python -m textblob.download_corpora
# pip install -U textblob
def academic_tone (std_ans, rulebook):
    
    # measure if the ans contains topics (issue, rule,policy, conclusion..), that makes an academic tone when present
    topic_sel = identify_topics(std_ans, rulebook)
    std_topics_result = int(100 * (1-(len(topic_sel['missing'])/len(rulebook['topics']))))

    #measure the subjectivty, from 0 to 1. If closer to 0 that means more objective, we aim for that. 
    subj = TextBlob(std_ans)
    subject_result = subj.subjectivity
    #calculate by multiplying 100 - (subjectivity * 100), which will give you how objective is the ans
    objectivity = 100 - (subject_result * 100)

    #Do the average of the two measure, giving an overall 
    academic_tone_result = (objectivity + std_topics_result)/2
     
    return {'std_topics_result: ': std_topics_result,'objectivity:': objectivity, 'academic_tone_result:': academic_tone_result} 



# updates dictionary with student results
def results(std_answer, prof_answer, prof_answer_data, question_id):
    results = {}
    results.update({'id': question_id})
    results.update({'std_answer': std_answer})
    results.update({'prof_answer': prof_answer})

    # order of the function calls to get text classification solution
    # rulebook = format_data(question_id)
    std_results = identify_topics(std_answer, prof_answer_data)
    results.update({'topics': std_results['topics'], 'missing': std_results['missing']})
    results.update({'prof_topics': prof_answer_data['topics']})
    prof_sentences = []
    for sent in getSentences(prof_answer):
        found = False
        for topic in prof_answer_data['topics'].keys():
            if sent in prof_answer_data['topics'][topic]['sentence']: 
                prof_sentences.append([sent, topic])
                found = True
        if (found == False):
            prof_sentences.append([sent, ''])

    results.update({'prof_sentences': prof_sentences})

    results.update({'std_sentences': getSentences(std_answer)})

    results.update({

        'grammarScore':  100-(len(calc_grammar(std_answer))*5), 
        'similarityScore': 100 - int(similarity(std_answer, prof_answer) * 100),
        'structureScore': int(100 * (1 - (len(std_results['missing']) / len(prof_answer_data['topics'])))),
        'readabilityScore': readability(std_answer),
        'concisenessScore': 100-(conciseness(std_answer)*5),
        'sentiment': sentiment_analyzer(std_answer)
    })
    results.update({
        'overallScore': round(0.4*results['structureScore'] + 0.2*results['concisenessScore'] + 0.2*results['grammarScore'] + 0.2*results['readabilityScore']['score'])
    })

    grammar = calc_grammar(std_answer)
    grammar_results = {}

    index = 0
    for error in grammar:
        name = "error" + str(index)
        grammar_results.update({name: {}})
        grammar_results[name].update({'startLocation': error['offset']})
        grammar_results[name].update({'endLocation': (error['offset'] + error['length'])})
        grammar_results[name].update({'suggestion': error['replacements']})
        index = index + 1

    results.update({'grammar_results': grammar_results})

    print('printing results from aeg.results')
    print(results)
    return results

