gnes.preprocessor.base module

class gnes.preprocessor.base.BaseAudioPreprocessor(uniform_doc_weight=True, *args, **kwargs)

Bases: gnes.preprocessor.base.BasePreprocessor

doc_type = 4

train(*args, **kwargs)

Train the model, need to be overrided

class gnes.preprocessor.base.BaseImagePreprocessor(uniform_doc_weight=True, *args, **kwargs)

Bases: gnes.preprocessor.base.BasePreprocessor

doc_type = 2

train(*args, **kwargs)

Train the model, need to be overrided

class gnes.preprocessor.base.BasePreprocessor(uniform_doc_weight=True, *args, **kwargs)

Bases: gnes.base.TrainableBase

apply(doc)

Return type:None

doc_type = 0

train(*args, **kwargs)

Train the model, need to be overrided

class gnes.preprocessor.base.BaseTextPreprocessor(uniform_doc_weight=True, *args, **kwargs)

Bases: gnes.preprocessor.base.BasePreprocessor

doc_type = 1

train(*args, **kwargs)

Train the model, need to be overrided

class gnes.preprocessor.base.BaseVideoPreprocessor(uniform_doc_weight=True, *args, **kwargs)

Bases: gnes.preprocessor.base.BasePreprocessor

doc_type = 3

train(*args, **kwargs)

Train the model, need to be overrided

class gnes.preprocessor.base.PipelinePreprocessor(*args, **kwargs)

Bases: gnes.base.CompositionalTrainableBase

apply(doc)

Return type:None

train(data, *args, **kwargs)

Train the model, need to be overrided

class gnes.preprocessor.base.RawChunkPreprocessor(uniform_doc_weight=True, *args, **kwargs)

Bases: gnes.preprocessor.base.BasePreprocessor

apply(doc)

Return type:None

train(*args, **kwargs)

Train the model, need to be overrided

class gnes.preprocessor.base.UnaryPreprocessor(doc_type, *args, **kwargs)

Bases: gnes.preprocessor.base.BasePreprocessor

apply(doc)

is_trained = True

raw_to_chunk(chunk, raw_bytes)

train(*args, **kwargs)

Train the model, need to be overrided