import sys
import numpy as np
from normalization import tokenize
from helpers import ahash

class KerasVectorizer():
    '''
        Convert list of documents to numpy array for input into Keras model
    '''
    def __init__(self, n_features=100000, maxlen=None, maxper=100, hash_function=ahash):
        self.maxlen = maxlen
        self.maxper = maxper
        self.n_features = n_features
        self.hash_function = hash_function
    
    def _exact_hash(self, word, n_features):
        return self.token_lookup.get(word, 0)
    
    def fit_transform(self, raw_documents, y=None, suffix='', verbose=True):
        if verbose:
            print >> sys.stderr, 'splitting raw documents'
        
        # Some way to print progress?
        tokens = map(self._split_function, raw_documents)
        
        if self.maxlen:
            maxlen = self.maxlen
        else:
            maxlen = int(np.percentile(map(len, tokens), self.maxper))
            self.maxlen = maxlen
        
        X = np.zeros((len(tokens), maxlen))
        for i,t in enumerate(tokens):
            if verbose:
                if not i % 10000:
                    print >> sys.stderr, 'processed %d tokens' % i
            
            if len(t) > 0:
                X[i,-len(t):] = map(lambda x: self.hash_function(x + suffix, self.n_features), t[:maxlen])
        
        return X


class KerasCharacterVectorizer(KerasVectorizer):
    '''
        Split a string into characters
    '''
    def _split_function(self, doc):
        return list(doc)


class KerasTokenVectorizer(KerasVectorizer):
    '''
        Split a string into words, 
    '''
    def _split_function(self, doc):
        return tokenize(doc, keep_punctuation=True)


class 
(KerasVectorizer):
    def _split_function(self, doc):
        return doc


'''
from keras_vectorizer import KerasTokenVectorizer, KerasCharacterVectorizer

ktv = KerasTokenVectorizer()
ktv.fit_transform(['this is a test'])
ktv.fit_transform(['this is a test', 'this is a another test'])

ktv = KerasTokenVectorizer(maxlen=2)
ktv.fit_transform(['this is a test', 'this is a another test'])

kcv = KerasCharacterVectorizer()
kcv.fit_transform(['something', 'else'])
'''

KerasPretokenizedVectorizer