tok2vec loss doesnt change from 0 while training #13776
Unanswered
ravi-abv
asked this question in
Help: Model Advice
Replies: 0 comments
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
-
I am trying to predict categories of text of reviews using a pipeline of ['tok2vec', 'ner', 'textcat_multilabel']
The NER generates entities from the text which are then used in a multilabel text classifier to predict the categories
The tok2vec loss is always 0. How to ensure that tok2vec component gets trained
============================= Training pipeline =============================
ℹ Pipeline: ['tok2vec', 'ner', 'textcat_multilabel']
ℹ Initial learn rate: 0.001
E # LOSS TOK2VEC LOSS NER LOSS TEXTC... ENTS_F ENTS_P ENTS_R CATS_SCORE SCORE
0 0 0.00 168.67 0.28 0.05 0.10 0.03 19.34 0.10
0 200 0.00 6749.35 2.01 85.21 92.53 78.96 17.51 0.51
0 400 0.00 1756.33 0.76 88.96 91.53 86.53 20.18 0.55
0 600 0.00 1456.59 0.62 91.07 93.74 88.54 22.27 0.57
My config is as follows
[paths]
train = "corpus/train_data.spacy"
dev = "corpus/dev_data.spacy"
output = "training"
vectors = null
init_tok2vec = null
[system]
gpu_allocator = "pytorch"
seed = 42
[nlp]
lang = "en"
pipeline = ["tok2vec", "ner", "textcat_multilabel"]
disabled = []
batch_size = 512
tokenizer = {"@Tokenizers": "spacy.Tokenizer.v1"}
before_creation = null
after_creation = null
after_pipeline_creation = null
[components]
[components.ner]
factory = "ner"
incorrect_spans_key = null
moves = null
scorer = {"@scorers": "spacy.ner_scorer.v1"}
update_with_oracle_cut_size = 100
[components.ner.model]
@architectures = "spacy.TransitionBasedParser.v2"
state_type = "ner"
extra_state_tokens = false
hidden_width = 64
maxout_pieces = 2
use_upper = true
nO = null
[components.textcat_multilabel]
factory = "textcat_multilabel"
scorer = {"@scorers": "spacy.textcat_multilabel_scorer.v2"}
threshold = 0.4
[components.textcat_multilabel.model]
@architectures = "spacy.TextCatEnsemble.v2"
nO = null
[components.textcat_multilabel.model.tok2vec]
@architectures = "spacy.Tok2Vec.v2"
[components.textcat_multilabel.model.tok2vec.embed]
@architectures = "spacy.MultiHashEmbed.v2"
width = 64
rows = [2000, 2000, 500, 1000, 500]
attrs = ["NORM", "LOWER", "PREFIX", "SUFFIX", "SHAPE"]
include_static_vectors = false
[components.textcat_multilabel.model.tok2vec.encode]
@architectures = "spacy.MaxoutWindowEncoder.v2"
width = 64
window_size = 1
maxout_pieces = 3
depth = 2
[components.textcat_multilabel.model.linear_model]
@architectures = "spacy.TextCatBOW.v3"
exclusive_classes = false
ngram_size = 1
no_output_layer = false
nO = null
[components.tok2vec]
factory = "tok2vec"
[components.tok2vec.model]
@architectures = "spacy.Tok2Vec.v2"
[components.tok2vec.model.embed]
@architectures = "spacy.MultiHashEmbed.v2"
width = 300
attrs = ["NORM", "PREFIX", "SUFFIX", "SHAPE"]
rows = [5000, 2500, 2500, 2500]
include_static_vectors = false
[components.tok2vec.model.encode]
@architectures = "spacy.MaxoutWindowEncoder.v2"
width = 300
depth = 4
window_size = 1
maxout_pieces = 3
[corpora]
[corpora.dev]
@readers = "spacy.Corpus.v1"
path = ${paths.dev}
max_length = 0
gold_preproc = false
limit = 0
augmenter = null
[corpora.train]
@readers = "spacy.Corpus.v1"
path = ${paths.train}
max_length = 0
gold_preproc = false
limit = 0
augmenter = null
[training]
max_epochs = 10
dropout = 0.2
train_corpus = "corpora.train"
dev_corpus = "corpora.dev"
seed = 42
gpu_allocator = "pytorch"
patience = 1600
max_steps = 20000
eval_frequency = 200
accumulate_gradient = 1
frozen_components = []
annotating_components = []
before_to_disk = null
before_update = null
[training.batcher]
@batchers = "spacy.batch_by_words.v1"
size = 512
tolerance = 0.2
discard_oversize = false
get_length = null
[training.logger]
@Loggers = "spacy.ConsoleLogger.v1"
progress_bar = false
[training.optimizer]
@optimizers = "Adam.v1"
beta1 = 0.9
beta2 = 0.999
L2_is_weight_decay = true
L2 = 0.01
grad_clip = 1.0
use_averages = false
eps = 0.00000001
learn_rate = 0.001
[training.score_weights]
ents_f = 0.5
ents_p = 0.0
ents_r = 0.0
ents_per_type = null
cats_score = 0.5
cats_score_desc = null
cats_micro_p = null
cats_micro_r = null
cats_micro_f = null
cats_macro_p = null
cats_macro_r = null
cats_macro_f = null
cats_macro_auc = null
cats_f_per_type = null
[initialize]
vectors = null
init_tok2vec = null
vocab_data = null
lookups = null
before_init = null
after_init = null
[initialize.components]
[initialize.tokenizer]
Beta Was this translation helpful? Give feedback.
All reactions