forked from kaldi-asr/kaldi
-
Notifications
You must be signed in to change notification settings - Fork 3
/
Copy pathnnet-simple-component.h
2047 lines (1750 loc) · 88.5 KB
/
nnet-simple-component.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// nnet3/nnet-simple-component.h
// Copyright 2011-2013 Karel Vesely
// 2012-2015 Johns Hopkins University (author: Daniel Povey)
// 2013 Xiaohui Zhang
// 2014-2015 Vijayaditya Peddinti
// 2014-2015 Guoguo Chen
// 2015 Daniel Galvez
// 2015 Tom Ko
// See ../../COPYING for clarification regarding multiple authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
// See the Apache 2 License for the specific language governing permissions and
// limitations under the License.
#ifndef KALDI_NNET3_NNET_SIMPLE_COMPONENT_H_
#define KALDI_NNET3_NNET_SIMPLE_COMPONENT_H_
#include "nnet3/nnet-common.h"
#include "nnet3/nnet-component-itf.h"
#include "nnet3/natural-gradient-online.h"
#include <iostream>
namespace kaldi {
namespace nnet3 {
/// @file nnet-simple-component.h
/// This file contains declarations of components that are "simple", meaning
/// they don't care about the indexes they are operating on, produce one
/// output for one input, and return the kSimpleComponent flag in their
/// Properties(): for example, tanh and affine components. In
/// nnet-general-component.h there are components that don't fit this pattern.
// This "nnet3" version of the p-norm component only supports the 2-norm.
class PnormComponent: public Component {
public:
void Init(int32 input_dim, int32 output_dim);
explicit PnormComponent(int32 input_dim, int32 output_dim) {
Init(input_dim, output_dim);
}
virtual int32 Properties() const {
return kSimpleComponent|kLinearInInput|kBackpropNeedsInput|kBackpropNeedsOutput;
}
PnormComponent(): input_dim_(0), output_dim_(0) { }
virtual std::string Type() const { return "PnormComponent"; }
virtual void InitFromConfig(ConfigLine *cfl);
virtual int32 InputDim() const { return input_dim_; }
virtual int32 OutputDim() const { return output_dim_; }
virtual void Propagate(const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in,
CuMatrixBase<BaseFloat> *out) const;
virtual void Backprop(const std::string &debug_info,
const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in_value,
const CuMatrixBase<BaseFloat> &out_value,
const CuMatrixBase<BaseFloat> &out_deriv,
Component *to_update,
CuMatrixBase<BaseFloat> *in_deriv) const;
virtual Component* Copy() const { return new PnormComponent(input_dim_,
output_dim_); }
virtual void Read(std::istream &is, bool binary); // This Read function
// requires that the Component has the correct type.
/// Write component to stream
virtual void Write(std::ostream &os, bool binary) const;
protected:
int32 input_dim_;
int32 output_dim_;
};
// This component randomly zeros dropout_proportion of the input
// and the derivatives are backpropagated through the nonzero inputs.
// Typically this component used during training but not in test time.
// The idea is described under the name Dropout, in the paper
// "Dropout: A Simple Way to Prevent Neural Networks from Overfitting".
class DropoutComponent : public RandomComponent {
public:
void Init(int32 dim, BaseFloat dropout_proportion = 0.0, bool dropout_per_frame = false);
DropoutComponent(int32 dim, BaseFloat dropout = 0.0, bool dropout_per_frame = false) { Init(dim, dropout, dropout_per_frame); }
DropoutComponent(): dim_(0), dropout_proportion_(0.0), dropout_per_frame_(false) { }
virtual int32 Properties() const {
return kLinearInInput|kBackpropInPlace|kSimpleComponent|kBackpropNeedsInput|kBackpropNeedsOutput;
}
virtual std::string Type() const { return "DropoutComponent"; }
virtual void InitFromConfig(ConfigLine *cfl);
virtual int32 InputDim() const { return dim_; }
virtual int32 OutputDim() const { return dim_; }
virtual void Read(std::istream &is, bool binary);
// Write component to stream
virtual void Write(std::ostream &os, bool binary) const;
virtual void Propagate(const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in,
CuMatrixBase<BaseFloat> *out) const;
virtual void Backprop(const std::string &debug_info,
const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in_value,
const CuMatrixBase<BaseFloat> &out_value,
const CuMatrixBase<BaseFloat> &out_deriv,
Component *to_update,
CuMatrixBase<BaseFloat> *in_deriv) const;
virtual Component* Copy() const { return new DropoutComponent(dim_,
dropout_proportion_,
dropout_per_frame_); }
virtual std::string Info() const;
void SetDropoutProportion(BaseFloat dropout_proportion, bool dropout_per_frame) {
dropout_proportion_ = dropout_proportion;
dropout_per_frame_ = dropout_per_frame;
}
private:
int32 dim_;
/// dropout-proportion is the proportion that is dropped out,
/// e.g. if 0.1, we set 10% to zero value.
BaseFloat dropout_proportion_;
bool dropout_per_frame_;
};
class ElementwiseProductComponent: public Component {
public:
void Init(int32 input_dim, int32 output_dim);
explicit ElementwiseProductComponent(int32 input_dim, int32 output_dim) {
Init(input_dim, output_dim);
}
virtual int32 Properties() const {
return kSimpleComponent|kBackpropNeedsInput;
}
ElementwiseProductComponent(): input_dim_(0), output_dim_(0) { }
virtual std::string Type() const { return "ElementwiseProductComponent"; }
virtual void InitFromConfig(ConfigLine *cfl);
virtual int32 InputDim() const { return input_dim_; }
virtual int32 OutputDim() const { return output_dim_; }
virtual void Propagate(const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in,
CuMatrixBase<BaseFloat> *out) const;
virtual void Backprop(const std::string &debug_info,
const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in_value,
const CuMatrixBase<BaseFloat> &out_value,
const CuMatrixBase<BaseFloat> &out_deriv,
Component *to_update,
CuMatrixBase<BaseFloat> *in_deriv) const;
virtual Component* Copy() const { return new ElementwiseProductComponent(input_dim_,
output_dim_); }
virtual void Read(std::istream &is, bool binary); // This Read function
// requires that the Component has the correct type.
/// Write component to stream
virtual void Write(std::ostream &os, bool binary) const;
protected:
int32 input_dim_;
int32 output_dim_;
};
class NormalizeComponent: public Component {
public:
void Init(int32 input_dim, BaseFloat target_rms, bool add_log_stddev);
explicit NormalizeComponent(int32 input_dim,
BaseFloat target_rms = 1.0,
bool add_log_stddev = false) {
Init(input_dim, target_rms, add_log_stddev);
}
explicit NormalizeComponent(const NormalizeComponent &other);
// note: there is some special code in NonlinerComponent::Info() that
// specifically caters to this class.
virtual int32 Properties() const {
return (add_log_stddev_ ?
kSimpleComponent|kBackpropNeedsInput|kBackpropAdds :
kSimpleComponent|kBackpropNeedsInput|kPropagateInPlace|
kBackpropAdds|kBackpropInPlace);
}
NormalizeComponent(): target_rms_(1.0), add_log_stddev_(false) { }
virtual std::string Type() const { return "NormalizeComponent"; }
virtual void InitFromConfig(ConfigLine *cfl);
virtual Component* Copy() const { return new NormalizeComponent(*this); }
virtual void Propagate(const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in,
CuMatrixBase<BaseFloat> *out) const;
virtual void Backprop(const std::string &debug_info,
const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in_value,
const CuMatrixBase<BaseFloat> &, // out_value
const CuMatrixBase<BaseFloat> &out_deriv,
Component *to_update,
CuMatrixBase<BaseFloat> *in_deriv) const;
virtual void Read(std::istream &is, bool binary);
virtual void Write(std::ostream &os, bool binary) const;
virtual int32 InputDim() const { return input_dim_; }
virtual int32 OutputDim() const {
return (input_dim_ + (add_log_stddev_ ? 1 : 0));
}
virtual std::string Info() const;
private:
NormalizeComponent &operator = (const NormalizeComponent &other); // Disallow.
enum { kExpSquaredNormFloor = -66 };
static const BaseFloat kSquaredNormFloor;
int32 input_dim_;
BaseFloat target_rms_; // The target rms for outputs.
// about 0.7e-20. We need a value that's exactly representable in
// float and whose inverse square root is also exactly representable
// in float (hence, an even power of two).
bool add_log_stddev_; // If true, log(max(epsi, sqrt(row_in^T row_in / D)))
// is an extra dimension of the output.
};
class SigmoidComponent: public NonlinearComponent {
public:
explicit SigmoidComponent(const SigmoidComponent &other): NonlinearComponent(other) { }
SigmoidComponent() { }
virtual std::string Type() const { return "SigmoidComponent"; }
virtual int32 Properties() const {
return kSimpleComponent|kBackpropNeedsOutput|kPropagateInPlace|kStoresStats;
}
virtual Component* Copy() const { return new SigmoidComponent(*this); }
virtual void Propagate(const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in,
CuMatrixBase<BaseFloat> *out) const;
virtual void Backprop(const std::string &debug_info,
const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &, //in_value
const CuMatrixBase<BaseFloat> &out_value,
const CuMatrixBase<BaseFloat> &out_deriv,
Component *to_update,
CuMatrixBase<BaseFloat> *in_deriv) const;
virtual void StoreStats(const CuMatrixBase<BaseFloat> &out_value);
private:
// this function is called from Backprop code and only does something if the
// self-repair-scale config value is set.
void RepairGradients(const CuMatrixBase<BaseFloat> &out_value,
CuMatrixBase<BaseFloat> *in_deriv,
SigmoidComponent *to_update) const;
SigmoidComponent &operator = (const SigmoidComponent &other); // Disallow.
};
class TanhComponent: public NonlinearComponent {
public:
explicit TanhComponent(const TanhComponent &other): NonlinearComponent(other) { }
TanhComponent() { }
virtual std::string Type() const { return "TanhComponent"; }
virtual Component* Copy() const { return new TanhComponent(*this); }
virtual int32 Properties() const {
return kSimpleComponent|kBackpropNeedsOutput|kPropagateInPlace|kStoresStats;
}
virtual void Propagate(const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in,
CuMatrixBase<BaseFloat> *out) const;
virtual void Backprop(const std::string &debug_info,
const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &, //in_value
const CuMatrixBase<BaseFloat> &out_value,
const CuMatrixBase<BaseFloat> &out_deriv,
Component *to_update,
CuMatrixBase<BaseFloat> *in_deriv) const;
virtual void StoreStats(const CuMatrixBase<BaseFloat> &out_value);
private:
// this function is called from Backprop code and only does something if the
// self-repair-scale config value is set.
void RepairGradients(const CuMatrixBase<BaseFloat> &out_value,
CuMatrixBase<BaseFloat> *in_deriv,
TanhComponent *to_update) const;
TanhComponent &operator = (const TanhComponent &other); // Disallow.
};
class RectifiedLinearComponent: public NonlinearComponent {
public:
explicit RectifiedLinearComponent(const RectifiedLinearComponent &other):
NonlinearComponent(other) { }
RectifiedLinearComponent() { }
virtual std::string Type() const { return "RectifiedLinearComponent"; }
virtual Component* Copy() const { return new RectifiedLinearComponent(*this); }
virtual int32 Properties() const {
return kSimpleComponent|kLinearInInput|kBackpropNeedsOutput|kPropagateInPlace|
kStoresStats;
}
virtual void Propagate(const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in,
CuMatrixBase<BaseFloat> *out) const;
virtual void Backprop(const std::string &debug_info,
const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &, //in_value
const CuMatrixBase<BaseFloat> &out_value,
const CuMatrixBase<BaseFloat> &out_deriv,
Component *to_update,
CuMatrixBase<BaseFloat> *in_deriv) const;
virtual void StoreStats(const CuMatrixBase<BaseFloat> &out_value);
private:
// this function is called from Backprop code and only does something if the
// self-repair-scale config value is set.
void RepairGradients(CuMatrixBase<BaseFloat> *in_deriv,
RectifiedLinearComponent *to_update) const;
RectifiedLinearComponent &operator = (const RectifiedLinearComponent &other); // Disallow.
};
/**
This component is a fixed (non-trainable) nonlinearity that sums its inputs
to produce outputs. Currently the only supported configuration is that its
input-dim is interpreted as consisting of n blocks, and the output is just a
summation over the n blocks, where n = input-dim / output-dim, so for instance
output[n] = input[n] + input[block-size + n] + .... .
Later if needed we can add a configuration variable that allows you to sum
over 'interleaved' input.
*/
class SumReduceComponent: public Component {
public:
void Init(int32 input_dim, int32 output_dim);
explicit SumReduceComponent(int32 input_dim, int32 output_dim) {
Init(input_dim, output_dim);
}
virtual int32 Properties() const {
return kSimpleComponent|kLinearInInput;
}
SumReduceComponent(): input_dim_(0), output_dim_(0) { }
virtual std::string Type() const { return "SumReduceComponent"; }
virtual void InitFromConfig(ConfigLine *cfl);
virtual int32 InputDim() const { return input_dim_; }
virtual int32 OutputDim() const { return output_dim_; }
virtual void Propagate(const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in,
CuMatrixBase<BaseFloat> *out) const;
virtual void Backprop(const std::string &debug_info,
const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &, // in_value
const CuMatrixBase<BaseFloat> &, // out_value,
const CuMatrixBase<BaseFloat> &out_deriv,
Component *, // to_update
CuMatrixBase<BaseFloat> *in_deriv) const;
virtual Component* Copy() const { return new SumReduceComponent(input_dim_,
output_dim_); }
virtual void Read(std::istream &is, bool binary); // This Read function
// requires that the Component has the correct type.
/// Write component to stream
virtual void Write(std::ostream &os, bool binary) const;
protected:
int32 input_dim_;
int32 output_dim_;
};
class FixedAffineComponent;
class FixedScaleComponent;
class PerElementScaleComponent;
class PerElementOffsetComponent;
// Affine means a linear function plus an offset.
// Note: although this class can be instantiated, it also
// functions as a base-class for more specialized versions of
// AffineComponent.
class AffineComponent: public UpdatableComponent {
friend class SoftmaxComponent; // Friend declaration relates to mixing up.
public:
virtual int32 InputDim() const { return linear_params_.NumCols(); }
virtual int32 OutputDim() const { return linear_params_.NumRows(); }
virtual std::string Info() const;
virtual void InitFromConfig(ConfigLine *cfl);
AffineComponent() { } // use Init to really initialize.
virtual std::string Type() const { return "AffineComponent"; }
virtual int32 Properties() const {
return kSimpleComponent|kUpdatableComponent|kLinearInParameters|
kBackpropNeedsInput|kBackpropAdds;
}
virtual void Propagate(const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in,
CuMatrixBase<BaseFloat> *out) const;
virtual void Backprop(const std::string &debug_info,
const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in_value,
const CuMatrixBase<BaseFloat> &, // out_value
const CuMatrixBase<BaseFloat> &out_deriv,
Component *to_update,
CuMatrixBase<BaseFloat> *in_deriv) const;
virtual void Read(std::istream &is, bool binary);
virtual void Write(std::ostream &os, bool binary) const;
virtual Component* Copy() const;
// Some functions from base-class UpdatableComponent.
virtual void Scale(BaseFloat scale);
virtual void Add(BaseFloat alpha, const Component &other);
virtual void SetZero(bool treat_as_gradient);
virtual void PerturbParams(BaseFloat stddev);
virtual BaseFloat DotProduct(const UpdatableComponent &other) const;
virtual int32 NumParameters() const;
virtual void Vectorize(VectorBase<BaseFloat> *params) const;
virtual void UnVectorize(const VectorBase<BaseFloat> ¶ms);
// Some functions that are specific to this class.
// This new function is used when mixing up:
virtual void SetParams(const VectorBase<BaseFloat> &bias,
const MatrixBase<BaseFloat> &linear);
const CuVector<BaseFloat> &BiasParams() const { return bias_params_; }
const CuMatrix<BaseFloat> &LinearParams() const { return linear_params_; }
explicit AffineComponent(const AffineComponent &other);
// The next constructor is used in converting from nnet1.
AffineComponent(const CuMatrixBase<BaseFloat> &linear_params,
const CuVectorBase<BaseFloat> &bias_params,
BaseFloat learning_rate);
void Init(int32 input_dim, int32 output_dim,
BaseFloat param_stddev, BaseFloat bias_stddev);
void Init(std::string matrix_filename);
// This function resizes the dimensions of the component, setting the
// parameters to zero, while leaving any other configuration values the same.
virtual void Resize(int32 input_dim, int32 output_dim);
// The following functions are used for collapsing multiple layers
// together. They return a pointer to a new Component equivalent to
// the sequence of two components. We haven't implemented this for
// FixedLinearComponent yet.
Component *CollapseWithNext(const AffineComponent &next) const ;
Component *CollapseWithNext(const FixedAffineComponent &next) const;
Component *CollapseWithNext(const FixedScaleComponent &next) const;
Component *CollapseWithPrevious(const FixedAffineComponent &prev) const;
protected:
friend class NaturalGradientAffineComponent;
// This function Update() is for extensibility; child classes may override
// this, e.g. for natural gradient update.
virtual void Update(
const std::string &debug_info,
const CuMatrixBase<BaseFloat> &in_value,
const CuMatrixBase<BaseFloat> &out_deriv) {
UpdateSimple(in_value, out_deriv);
}
// UpdateSimple is used when *this is a gradient. Child classes may override
// this if needed, but typically won't need to.
virtual void UpdateSimple(
const CuMatrixBase<BaseFloat> &in_value,
const CuMatrixBase<BaseFloat> &out_deriv);
const AffineComponent &operator = (const AffineComponent &other); // Disallow.
CuMatrix<BaseFloat> linear_params_;
CuVector<BaseFloat> bias_params_;
};
class RepeatedAffineComponent;
/// This class implements an affine transform using a block diagonal matrix
/// e.g., one whose weight matrix is all zeros except for blocks on the
/// diagonal. All these blocks have the same dimensions.
/// input-dim: num cols of block diagonal matrix.
/// output-dim: num rows of block diagonal matrix.
/// num-blocks: number of blocks in diagonal of the matrix.
/// num-blocks must divide both input-dim and output-dim
class BlockAffineComponent : public UpdatableComponent {
public:
virtual int32 InputDim() const { return linear_params_.NumCols() * num_blocks_; }
virtual int32 OutputDim() const { return linear_params_.NumRows(); }
virtual std::string Info() const;
virtual void InitFromConfig(ConfigLine *cfl);
BlockAffineComponent() { }
virtual std::string Type() const { return "BlockAffineComponent"; }
virtual int32 Properties() const {
return kSimpleComponent|kUpdatableComponent|kLinearInParameters|
kBackpropNeedsInput|kBackpropAdds;
}
virtual void Propagate(const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in,
CuMatrixBase<BaseFloat> *out) const;
virtual void Backprop(const std::string &debug_info,
const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in_value,
const CuMatrixBase<BaseFloat> &, // out_value
const CuMatrixBase<BaseFloat> &out_deriv,
Component *to_update,
CuMatrixBase<BaseFloat> *in_deriv) const;
virtual void Read(std::istream &is, bool binary);
virtual void Write(std::ostream &os, bool binary) const;
virtual Component* Copy() const;
// Functions from base-class UpdatableComponent.
virtual void Scale(BaseFloat scale);
virtual void Add(BaseFloat alpha, const Component &other);
virtual void SetZero(bool treat_as_gradient);
virtual void PerturbParams(BaseFloat stddev);
virtual BaseFloat DotProduct(const UpdatableComponent &other) const;
virtual int32 NumParameters() const;
virtual void Vectorize(VectorBase<BaseFloat> *params) const;
virtual void UnVectorize(const VectorBase<BaseFloat> ¶ms);
// BlockAffine-specific functions.
void Init(int32 input_dim, int32 output_dim, int32 num_blocks,
BaseFloat param_stddev, BaseFloat bias_mean,
BaseFloat bias_stddev);
explicit BlockAffineComponent(const BlockAffineComponent &other);
explicit BlockAffineComponent(const RepeatedAffineComponent &rac);
protected:
// The matrix linear_params_ has a block structure, with num_blocks_ blocks of
// equal size. The blocks are stored in linear_params_ as
// [ M
// N
// O ] but we actually treat it as the matrix:
// [ M 0 0
// 0 N 0
// 0 0 O ]
CuMatrix<BaseFloat> linear_params_;
CuVector<BaseFloat> bias_params_;
int32 num_blocks_;
private:
const BlockAffineComponent &operator = (const BlockAffineComponent &other); // Disallow.
};
class RepeatedAffineComponent: public UpdatableComponent {
public:
virtual int32 InputDim() const { return linear_params_.NumCols() * num_repeats_; }
virtual int32 OutputDim() const { return linear_params_.NumRows() * num_repeats_; }
virtual std::string Info() const;
virtual void InitFromConfig(ConfigLine *cfl);
RepeatedAffineComponent() { } // use Init to really initialize.
virtual std::string Type() const { return "RepeatedAffineComponent"; }
virtual int32 Properties() const {
return kSimpleComponent|kUpdatableComponent|kLinearInParameters|
kBackpropNeedsInput|kBackpropAdds|kInputContiguous|kOutputContiguous;
}
virtual void Propagate(const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in,
CuMatrixBase<BaseFloat> *out) const;
virtual void Backprop(const std::string &debug_info,
const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in_value,
const CuMatrixBase<BaseFloat> &, // out_value
const CuMatrixBase<BaseFloat> &out_deriv,
Component *to_update,
CuMatrixBase<BaseFloat> *in_deriv) const;
virtual void Read(std::istream &is, bool binary);
virtual void Write(std::ostream &os, bool binary) const;
virtual Component* Copy() const;
// Some functions from base-class UpdatableComponent.
virtual void Scale(BaseFloat scale);
virtual void Add(BaseFloat alpha, const Component &other);
virtual void SetZero(bool treat_as_gradient);
virtual void PerturbParams(BaseFloat stddev);
virtual BaseFloat DotProduct(const UpdatableComponent &other) const;
virtual int32 NumParameters() const;
virtual void Vectorize(VectorBase<BaseFloat> *params) const;
virtual void UnVectorize(const VectorBase<BaseFloat> ¶ms);
// Some functions that are specific to this class.
const CuVector<BaseFloat> &BiasParams() const { return bias_params_; }
const CuMatrix<BaseFloat> &LinearParams() const { return linear_params_; }
explicit RepeatedAffineComponent(const RepeatedAffineComponent &other);
void Init(int32 input_dim, int32 output_dim, int32 num_repeats,
BaseFloat param_stddev, BaseFloat bias_mean,
BaseFloat bias_stddev);
friend BlockAffineComponent::BlockAffineComponent(const RepeatedAffineComponent &rac);
protected:
// This function Update(), called from backprop, is broken out for
// extensibility to natural gradient update.
virtual void Update(
const CuMatrixBase<BaseFloat> &in_value,
const CuMatrixBase<BaseFloat> &out_deriv);
// This function does nothing here but is redefined in child-class
// NaturalGradientRepeatedAffineComponent. This help avoid repeated code.
virtual void SetNaturalGradientConfigs() { }
const RepeatedAffineComponent &operator = (const RepeatedAffineComponent &other); // Disallow.
CuMatrix<BaseFloat> linear_params_;
CuVector<BaseFloat> bias_params_;
int32 num_repeats_;
};
class NaturalGradientRepeatedAffineComponent: public RepeatedAffineComponent {
public:
// Use Init() to really initialize.
NaturalGradientRepeatedAffineComponent() { }
// Most of the public functions are inherited from RepeatedAffineComponent.
virtual std::string Type() const {
return "NaturalGradientRepeatedAffineComponent";
}
virtual Component* Copy() const;
// Copy constructor
explicit NaturalGradientRepeatedAffineComponent(
const NaturalGradientRepeatedAffineComponent &other);
private:
virtual void Update(
const CuMatrixBase<BaseFloat> &in_value,
const CuMatrixBase<BaseFloat> &out_deriv);
const NaturalGradientRepeatedAffineComponent &operator=(
const NaturalGradientRepeatedAffineComponent &other); // Disallow.
// Applies the default configuration to preconditioner_in_.
virtual void SetNaturalGradientConfigs();
// For efficiency reasons we only apply the natural gradient to the input
// side, i.e. not to the space of output derivatives-- we believe the input
// side is the more important side. We don't make the natural-gradient
// configurable; we just give it a reasonable configuration.
// Instead of using the individual data-points, for efficiency reasons we use
// the distribution of per-minibatch summed derivatives over each dimension of
// the output space, as the source for the Fisher matrix.
OnlineNaturalGradient preconditioner_in_;
};
class SoftmaxComponent: public NonlinearComponent {
public:
explicit SoftmaxComponent(const SoftmaxComponent &other):
NonlinearComponent(other) { }
SoftmaxComponent() { }
virtual std::string Type() const { return "SoftmaxComponent"; }
virtual int32 Properties() const {
return kSimpleComponent|kBackpropNeedsOutput|kStoresStats;
}
virtual void Propagate(const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in,
CuMatrixBase<BaseFloat> *out) const;
virtual void Backprop(const std::string &debug_info,
const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in_value,
const CuMatrixBase<BaseFloat> &out_value,
const CuMatrixBase<BaseFloat> &out_deriv,
Component *to_update,
CuMatrixBase<BaseFloat> *in_deriv) const;
virtual void StoreStats(const CuMatrixBase<BaseFloat> &out_value);
virtual Component* Copy() const { return new SoftmaxComponent(*this); }
private:
SoftmaxComponent &operator = (const SoftmaxComponent &other); // Disallow.
};
class LogSoftmaxComponent: public NonlinearComponent {
public:
explicit LogSoftmaxComponent(const LogSoftmaxComponent &other):
NonlinearComponent(other) { }
LogSoftmaxComponent() { }
virtual std::string Type() const { return "LogSoftmaxComponent"; }
virtual int32 Properties() const {
return kSimpleComponent|kBackpropNeedsOutput|kStoresStats;
}
virtual void Propagate(const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in,
CuMatrixBase<BaseFloat> *out) const;
virtual void Backprop(const std::string &debug_info,
const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in_value,
const CuMatrixBase<BaseFloat> &out_value,
const CuMatrixBase<BaseFloat> &out_deriv,
Component *to_update,
CuMatrixBase<BaseFloat> *in_deriv) const;
virtual Component* Copy() const { return new LogSoftmaxComponent(*this); }
private:
LogSoftmaxComponent &operator = (const LogSoftmaxComponent &other); // Disallow.
};
/// Keywords: natural gradient descent, NG-SGD, naturalgradient. For
/// the top-level of the natural gradient code look here, and also in
/// nnet-precondition-online.h.
/// NaturalGradientAffineComponent is
/// a version of AffineComponent that has a non-(multiple of unit) learning-rate
/// matrix. See nnet-precondition-online.h for a description of the technique.
/// It is described, under the name Online NG-SGD, in the paper "Parallel
/// training of DNNs with Natural Gradient and Parameter Averaging" (ICLR
/// workshop, 2015) by Daniel Povey, Xiaohui Zhang and Sanjeev Khudanpur.
class NaturalGradientAffineComponent: public AffineComponent {
public:
virtual std::string Type() const { return "NaturalGradientAffineComponent"; }
virtual void Read(std::istream &is, bool binary);
virtual void Write(std::ostream &os, bool binary) const;
void Init(int32 input_dim, int32 output_dim,
BaseFloat param_stddev, BaseFloat bias_stddev, BaseFloat bias_mean,
int32 rank_in, int32 rank_out, int32 update_period,
BaseFloat num_samples_history, BaseFloat alpha,
BaseFloat max_change_per_sample);
void Init(int32 rank_in, int32 rank_out, int32 update_period,
BaseFloat num_samples_history,
BaseFloat alpha, BaseFloat max_change_per_sample,
std::string matrix_filename);
// this constructor does not really initialize, use Init() or Read().
NaturalGradientAffineComponent();
virtual void Resize(int32 input_dim, int32 output_dim);
virtual void InitFromConfig(ConfigLine *cfl);
virtual std::string Info() const;
virtual Component* Copy() const;
virtual void Scale(BaseFloat scale);
virtual void Add(BaseFloat alpha, const Component &other);
// copy constructor
explicit NaturalGradientAffineComponent(
const NaturalGradientAffineComponent &other);
virtual void ZeroStats();
private:
// disallow assignment operator.
NaturalGradientAffineComponent &operator= (
const NaturalGradientAffineComponent&);
// Configs for preconditioner. The input side tends to be better conditioned ->
// smaller rank needed, so make them separately configurable.
int32 rank_in_;
int32 rank_out_;
int32 update_period_;
BaseFloat num_samples_history_;
BaseFloat alpha_;
OnlineNaturalGradient preconditioner_in_;
OnlineNaturalGradient preconditioner_out_;
// If > 0, max_change_per_sample_ is the maximum amount of parameter
// change (in L2 norm) that we allow per sample, averaged over the minibatch.
// This was introduced in order to control instability.
// Instead of the exact L2 parameter change, for
// efficiency purposes we limit a bound on the exact
// change. The limit is applied via a constant <= 1.0
// for each minibatch, A suitable value might be, for
// example, 10 or so; larger if there are more
// parameters.
BaseFloat max_change_per_sample_;
// update_count_ records how many updates we have done.
double update_count_;
// active_scaling_count_ records how many updates we have done,
// where the scaling factor is active (not 1.0).
double active_scaling_count_;
// max_change_scale_stats_ records the sum of scaling factors
// in each update, so we can compute the averaged scaling factor
// in Info().
double max_change_scale_stats_;
// Sets the configs rank, alpha and eta in the preconditioner objects,
// from the class variables.
void SetNaturalGradientConfigs();
virtual void Update(
const std::string &debug_info,
const CuMatrixBase<BaseFloat> &in_value,
const CuMatrixBase<BaseFloat> &out_deriv);
};
/// FixedAffineComponent is an affine transform that is supplied
/// at network initialization time and is not trainable.
class FixedAffineComponent: public Component {
public:
FixedAffineComponent() { }
virtual std::string Type() const { return "FixedAffineComponent"; }
virtual std::string Info() const;
// Copy constructor from AffineComponent-- can be used when we're done
// training a particular part of the model and want to efficiently disable
// further training.
FixedAffineComponent(const AffineComponent &c);
/// matrix should be of size input-dim+1 to output-dim, last col is offset
void Init(const CuMatrixBase<BaseFloat> &matrix);
// The ConfigLine cfl contains just the option matrix=<string>,
// where the string is the filename of a Kaldi-format matrix to read.
virtual void InitFromConfig(ConfigLine *cfl);
virtual int32 Properties() const { return kSimpleComponent|kBackpropAdds; }
virtual int32 InputDim() const { return linear_params_.NumCols(); }
virtual int32 OutputDim() const { return linear_params_.NumRows(); }
virtual void Propagate(const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in,
CuMatrixBase<BaseFloat> *out) const;
virtual void Backprop(const std::string &debug_info,
const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in_value,
const CuMatrixBase<BaseFloat> &, // out_value
const CuMatrixBase<BaseFloat> &out_deriv,
Component *to_update,
CuMatrixBase<BaseFloat> *in_deriv) const;
virtual Component* Copy() const;
virtual void Read(std::istream &is, bool binary);
virtual void Write(std::ostream &os, bool binary) const;
// Function to provide access to linear_params_.
const CuMatrix<BaseFloat> &LinearParams() const { return linear_params_; }
protected:
friend class AffineComponent;
CuMatrix<BaseFloat> linear_params_;
CuVector<BaseFloat> bias_params_;
KALDI_DISALLOW_COPY_AND_ASSIGN(FixedAffineComponent);
};
/// SumGroupComponent is used to sum up groups of posteriors.
/// It's used to introduce a kind of Gaussian-mixture-model-like
/// idea into neural nets. This is basically a degenerate case of
/// MixtureProbComponent; we had to implement it separately to
/// be efficient for CUDA (we can use this one regardless whether
/// we have CUDA or not; it's the normal case we want anyway).
///
/// There are two forms of initialization in a config file: one
/// where the number of elements are specified for each group
/// individually as a vector, and one where only the total input
/// dimension and the output dimension (number of groups) is specified.
/// The second is used when all groups have the same size.
class SumGroupComponent: public Component {
public:
virtual int32 InputDim() const { return input_dim_; }
virtual int32 OutputDim() const { return output_dim_; }
void Init(const std::vector<int32> &sizes); // the vector is of the input dim
// (>= 1) for each output dim.
void Init(int32 input_dim, int32 output_dim);
void GetSizes(std::vector<int32> *sizes) const; // Get a vector saying, for
// each output-dim, how many
// inputs were summed over.
virtual void InitFromConfig(ConfigLine *cfl);
SumGroupComponent() { }
virtual std::string Type() const { return "SumGroupComponent"; }
virtual int32 Properties() const { return kSimpleComponent|kLinearInInput; }
virtual void Propagate(const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in,
CuMatrixBase<BaseFloat> *out) const;
virtual void Backprop(const std::string &debug_info,
const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in_value,
const CuMatrixBase<BaseFloat> &, // out_value
const CuMatrixBase<BaseFloat> &out_deriv,
Component *to_update,
CuMatrixBase<BaseFloat> *in_deriv) const;
virtual Component* Copy() const;
virtual void Read(std::istream &is, bool binary);
virtual void Write(std::ostream &os, bool binary) const;
private:
KALDI_DISALLOW_COPY_AND_ASSIGN(SumGroupComponent);
// Note: Int32Pair is just struct{ int32 first; int32 second }; it's defined
// in cu-matrixdim.h as extern "C" which is needed for the CUDA interface.
CuArray<Int32Pair> indexes_; // for each output index, the (start, end) input
// index.
CuArray<int32> reverse_indexes_; // for each input index, the output index.
int32 input_dim_;
int32 output_dim_;
};
/// FixedScaleComponent applies a fixed per-element scale; it's similar
/// to the Rescale component in the nnet1 setup (and only needed for nnet1
/// model conversion).
class FixedScaleComponent: public Component {
public:
FixedScaleComponent() { }
virtual std::string Type() const { return "FixedScaleComponent"; }
virtual std::string Info() const;
virtual int32 Properties() const {
return kSimpleComponent|kLinearInInput|kPropagateInPlace|kBackpropInPlace;
}
void Init(const CuVectorBase<BaseFloat> &scales);
// The ConfigLine cfl contains only the option scales=<string>,
// where the string is the filename of a Kaldi-format matrix to read.
virtual void InitFromConfig(ConfigLine *cfl);
virtual int32 InputDim() const { return scales_.Dim(); }
virtual int32 OutputDim() const { return scales_.Dim(); }
virtual void Propagate(const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in,
CuMatrixBase<BaseFloat> *out) const;
virtual void Backprop(const std::string &debug_info,
const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &, // in_value
const CuMatrixBase<BaseFloat> &, // out_value
const CuMatrixBase<BaseFloat> &out_deriv,
Component *, // to_update
CuMatrixBase<BaseFloat> *in_deriv) const;
virtual Component* Copy() const;
virtual void Read(std::istream &is, bool binary);
virtual void Write(std::ostream &os, bool binary) const;
protected:
friend class AffineComponent; // necessary for collapse
CuVector<BaseFloat> scales_;
KALDI_DISALLOW_COPY_AND_ASSIGN(FixedScaleComponent);
};
/// FixedBiasComponent applies a fixed per-element bias; it's similar
/// to the AddShift component in the nnet1 setup (and only needed for nnet1
/// model conversion.
class FixedBiasComponent: public Component {
public:
FixedBiasComponent() { }
virtual std::string Type() const { return "FixedBiasComponent"; }
virtual std::string Info() const;
virtual int32 Properties() const {
return kSimpleComponent|kPropagateInPlace|kBackpropInPlace;
}
void Init(const CuVectorBase<BaseFloat> &scales);
// The ConfigLine cfl contains only the option bias=<string>,
// where the string is the filename of a Kaldi-format matrix to read.
virtual void InitFromConfig(ConfigLine *cfl);
virtual int32 InputDim() const { return bias_.Dim(); }
virtual int32 OutputDim() const { return bias_.Dim(); }
using Component::Propagate; // to avoid name hiding
virtual void Propagate(const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in,
CuMatrixBase<BaseFloat> *out) const;
virtual void Backprop(const std::string &debug_info,
const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &, // in_value,
const CuMatrixBase<BaseFloat> &, // out_value
const CuMatrixBase<BaseFloat> &out_deriv,
Component *, // to_update
CuMatrixBase<BaseFloat> *in_deriv) const;
virtual Component* Copy() const;
virtual void Read(std::istream &is, bool binary);
virtual void Write(std::ostream &os, bool binary) const;
protected:
CuVector<BaseFloat> bias_;
KALDI_DISALLOW_COPY_AND_ASSIGN(FixedBiasComponent);
};
// NoOpComponent just duplicates its input. We don't anticipate this being used
// very often, but it may sometimes make your life easier
class NoOpComponent: public NonlinearComponent {
public:
explicit NoOpComponent(const NoOpComponent &other): NonlinearComponent(other) { }
NoOpComponent() { }
virtual std::string Type() const { return "NoOpComponent"; }
virtual int32 Properties() const {
return kSimpleComponent|kLinearInInput|kPropagateInPlace;
}
virtual Component* Copy() const { return new NoOpComponent(*this); }
virtual void Propagate(const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &in,
CuMatrixBase<BaseFloat> *out) const;
virtual void Backprop(const std::string &debug_info,
const ComponentPrecomputedIndexes *indexes,
const CuMatrixBase<BaseFloat> &, //in_value
const CuMatrixBase<BaseFloat> &, // out_value,
const CuMatrixBase<BaseFloat> &out_deriv,
Component *to_update,
CuMatrixBase<BaseFloat> *in_deriv) const;
private:
NoOpComponent &operator = (const NoOpComponent &other); // Disallow.
};
// ClipGradientComponent just duplicates its input, but clips gradients