diff --git a/Makefile.am b/Makefile.am index f47afec6f8..87a05a222b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -519,6 +519,7 @@ libtesseract_lstm_la_CPPFLAGS += -DTESSDATA_PREFIX='"@datadir@"' endif noinst_HEADERS += src/lstm/convolve.h +noinst_HEADERS += src/lstm/dropout.h noinst_HEADERS += src/lstm/fullyconnected.h noinst_HEADERS += src/lstm/functions.h noinst_HEADERS += src/lstm/input.h @@ -541,6 +542,7 @@ noinst_HEADERS += src/lstm/weightmatrix.h noinst_LTLIBRARIES += libtesseract_lstm.la libtesseract_lstm_la_SOURCES = src/lstm/convolve.cpp +libtesseract_lstm_la_SOURCES += src/lstm/dropout.cpp libtesseract_lstm_la_SOURCES += src/lstm/fullyconnected.cpp libtesseract_lstm_la_SOURCES += src/lstm/functions.cpp libtesseract_lstm_la_SOURCES += src/lstm/input.cpp diff --git a/src/ccutil/helpers.h b/src/ccutil/helpers.h index fa2f38ec83..665ea2f8eb 100644 --- a/src/ccutil/helpers.h +++ b/src/ccutil/helpers.h @@ -86,6 +86,10 @@ class TRand { return range * 2.0 * IntRand() / INT32_MAX - range; } // Returns a floating point value in the range [0, range]. + float UnsignedRand(float range) { + return range * IntRand() / INT32_MAX; + } + // Returns a double value in the range [0, range]. double UnsignedRand(double range) { return range * IntRand() / INT32_MAX; } diff --git a/src/lstm/dropout.cpp b/src/lstm/dropout.cpp new file mode 100644 index 0000000000..46c998c2bc --- /dev/null +++ b/src/lstm/dropout.cpp @@ -0,0 +1,189 @@ +/////////////////////////////////////////////////////////////////////// +// File: dropout.cpp +// Description: Dropout layer. +// Author: Stefan Weil +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +/////////////////////////////////////////////////////////////////////// + +#ifdef HAVE_CONFIG_H +# include "config_auto.h" +#endif + +#include "dropout.h" +#include "networkscratch.h" +#include "serialis.h" +#include "tesserrstream.h" // for tesserr + +namespace tesseract { + +Dropout::Dropout(const std::string &name, int ni, float dropout_rate, unsigned dropout_dim) + : Network(NT_DROPOUT, name, ni, ni), + dropout_mask_(), + dropout_rate_(dropout_rate), + dropout_dim_(dropout_dim) { + if (dropout_rate_ < 0 || dropout_rate_ >= 1) { + throw std::invalid_argument("Invalid dropout rate. Must be in [0, 1)."); + } + if (dropout_dim_ > 2) { + throw std::invalid_argument("Invalid dropout dim. Must be 0, 1 or 2."); + } +} + +void Dropout::DebugWeights() { + tesserr << "Dropout layer '" << name_ << "': rate=" << dropout_rate_ << ", dimension=" << dropout_dim_ << '\n'; +} + +// Writes to the given file. Returns false in case of error. +bool Dropout::Serialize(TFile *fp) const { + // dropout_mask_ is runtime data and is not serialized. + return Network::Serialize(fp) && + fp->Serialize(&dropout_rate_) && + fp->Serialize(&dropout_dim_); +} + +// Reads from the given file. Returns false in case of error. +bool Dropout::DeSerialize(TFile *fp) { + if (!fp->DeSerialize(&dropout_rate_)) { + return false; + } + if (!fp->DeSerialize(&dropout_dim_)) { + return false; + } + no_ = ni_; + return true; +} + +// Runs forward propagation of activations on the input line. +// See Network for a detailed discussion of the arguments. +void Dropout::Forward(bool debug, const NetworkIO &input, + const TransposedArray *input_transpose, + NetworkScratch *scratch, NetworkIO *output) { + // Output has the same shape as input (ni_ == no_). + output->Resize(input, no_); + + int width = input.Width(); + int num_features = input.NumFeatures(); + + if (IsTraining() && dropout_rate_ > 0.0f) { + // Inverted dropout: scale retained activations by 1/keep_prob so that + // inference can pass the network output through unchanged. + float keep_prob = 1.0f - dropout_rate_; + float scale = 1.0f / keep_prob; + + if (dropout_dim_ == 2) { + // Feature dropout: one mask value per feature, shared across all timesteps. + dropout_mask_.resize(num_features); + for (int i = 0; i < num_features; ++i) { + float r = randomizer_->UnsignedRand(1.0f); + dropout_mask_[i] = (r < keep_prob) ? 1 : 0; + } + for (int t = 0; t < width; ++t) { + const float *in = input.f(t); + float *out = output->f(t); + for (int i = 0; i < num_features; ++i) + out[i] = dropout_mask_[i] ? in[i] * scale : 0.0f; + } + } else if (dropout_dim_ == 1) { + // Temporal dropout: one mask value per timestep, shared across all features. + dropout_mask_.resize(width); + for (int t = 0; t < width; ++t) { + float r = randomizer_->UnsignedRand(1.0f); + dropout_mask_[t] = (r < keep_prob) ? 1 : 0; + } + for (int t = 0; t < width; ++t) { + const float *in = input.f(t); + float *out = output->f(t); + if (dropout_mask_[t]) { + for (int i = 0; i < num_features; ++i) out[i] = in[i] * scale; + } else { + memset(out, 0, sizeof(float) * num_features); + } + } + } else { + // Element-wise dropout (dim=0, default). + dropout_mask_.resize(static_cast(width) * num_features); + for (int t = 0; t < width; ++t) { + const float *in = input.f(t); + float *out = output->f(t); + char *mask = dropout_mask_.data() + t * num_features; + for (int i = 0; i < num_features; ++i) { + // UnsignedRand(1.0f) returns a value in [0, 1]. + float r = randomizer_->UnsignedRand(1.0f); + mask[i] = (r < keep_prob) ? 1 : 0; + out[i] = mask[i] ? in[i] * scale : 0.0f; + } + } + } + } else { + // Inference mode (or dropout_rate_ == 0): pass input through unchanged. + output->CopyAll(input); + } + +#ifndef GRAPHICS_DISABLED + if (debug) { + DisplayForward(*output); + } +#endif +} + +// Runs backward propagation of errors on the deltas line. +// See Network for a detailed discussion of the arguments. +bool Dropout::Backward(bool debug, const NetworkIO &fwd_deltas, + NetworkScratch *scratch, NetworkIO *back_deltas) { + back_deltas->Resize(fwd_deltas, ni_); + + int width = fwd_deltas.Width(); + int num_features = fwd_deltas.NumFeatures(); + + if (IsTraining() && dropout_rate_ > 0.0f) { + float scale = 1.0f / (1.0f - dropout_rate_); + + if (dropout_dim_ == 2) { + for (int t = 0; t < width; ++t) { + const float *in = fwd_deltas.f(t); + float *out = back_deltas->f(t); + for (int i = 0; i < num_features; ++i) + out[i] = dropout_mask_[i] ? in[i] * scale : 0.0f; + } + } else if (dropout_dim_ == 1) { + for (int t = 0; t < width; ++t) { + const float *in = fwd_deltas.f(t); + float *out = back_deltas->f(t); + if (dropout_mask_[t]) { + for (int i = 0; i < num_features; ++i) out[i] = in[i] * scale; + } else { + memset(out, 0, sizeof(float) * num_features); + } + } + } else { + for (int t = 0; t < width; ++t) { + const float *in = fwd_deltas.f(t); + float *out = back_deltas->f(t); + const char *mask = dropout_mask_.data() + t * num_features; + for (int i = 0; i < num_features; ++i) + out[i] = mask[i] ? in[i] * scale : 0.0f; + } + } + } else { + // Inference mode: pass gradients through unchanged. + back_deltas->CopyAll(fwd_deltas); + } + +#ifndef GRAPHICS_DISABLED + if (debug) { + DisplayBackward(*back_deltas); + } +#endif + return needs_to_backprop_; +} + +} // namespace tesseract. diff --git a/src/lstm/dropout.h b/src/lstm/dropout.h new file mode 100644 index 0000000000..db6ebc3c22 --- /dev/null +++ b/src/lstm/dropout.h @@ -0,0 +1,64 @@ +/////////////////////////////////////////////////////////////////////// +// File: dropout.h +// Description: Standard Dropout layer. +// Author: Stefan Weil +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +/////////////////////////////////////////////////////////////////////// + +#ifndef TESSERACT_LSTM_DROPOUT_H_ +#define TESSERACT_LSTM_DROPOUT_H_ + +#include "network.h" + +namespace tesseract { + +// Dropout. +class Dropout : public Network { +public: + TESS_API + Dropout(const std::string &name, int ni, float dropout_rate, unsigned dropout_dim = 0); + ~Dropout() override = default; + + // Accessors. + std::string spec() const override { + std::string s = "Do" + std::to_string(dropout_rate_); + if (dropout_dim_ != 0) + s += "," + std::to_string(dropout_dim_); + return s; + } + + // Writes to the given file. Returns false in case of error. + bool Serialize(TFile *fp) const override; + // Reads from the given file. Returns false in case of error. + bool DeSerialize(TFile *fp) override; + + // Runs forward propagation of activations on the input line. + // See Network for a detailed discussion of the arguments. + void Forward(bool debug, const NetworkIO &input, const TransposedArray *input_transpose, + NetworkScratch *scratch, NetworkIO *output) override; + + // Runs backward propagation of errors on the deltas line. + // See Network for a detailed discussion of the arguments. + bool Backward(bool debug, const NetworkIO &fwd_deltas, NetworkScratch *scratch, + NetworkIO *back_deltas) override; + +private: + void DebugWeights() override; + + std::vector dropout_mask_; + float dropout_rate_; + unsigned dropout_dim_; // 0=elementwise, 1=temporal, 2=feature/channel +}; + +} // namespace tesseract. + +#endif // TESSERACT_LSTM_DROPOUT_H_ diff --git a/src/lstm/network.cpp b/src/lstm/network.cpp index cfddbfd43a..163ab73829 100644 --- a/src/lstm/network.cpp +++ b/src/lstm/network.cpp @@ -28,6 +28,7 @@ // factory deserializing method: CreateFromFile. #include #include "convolve.h" +#include "dropout.h" #include "fullyconnected.h" #include "input.h" #include "lstm.h" @@ -59,7 +60,7 @@ const int kYWinFrameSize = 80; // layer types in NetworkType without invalidating existing network files. static char const *const kTypeNames[NT_COUNT] = { "Invalid", "Input", - "Convolve", "Maxpool", + "Convolve", "Dropout", "Maxpool", "Parallel", "Replicated", "ParBidiLSTM", "DepParUDLSTM", "Par2dLSTM", "Series", @@ -251,6 +252,9 @@ Network *Network::CreateFromFile(TFile *fp) { case NT_CONVOLVE: network = new Convolve(name, ni, 0, 0); break; + case NT_DROPOUT: + network = new Dropout(name, ni, 0.5f); + break; case NT_INPUT: network = new Input(name, ni, no); break; diff --git a/src/lstm/network.h b/src/lstm/network.h index 353b110c92..d8b238b07d 100644 --- a/src/lstm/network.h +++ b/src/lstm/network.h @@ -42,6 +42,7 @@ enum NetworkType { NT_INPUT, // Inputs from an image. // Plumbing networks combine other networks or rearrange the inputs. NT_CONVOLVE, // Duplicates inputs in a sliding window neighborhood. + NT_DROPOUT, // Dropout random inputs. NT_MAXPOOL, // Chooses the max result from a rectangle. NT_PARALLEL, // Runs networks in parallel. NT_REPLICATED, // Runs identical networks in parallel. diff --git a/src/training/common/networkbuilder.cpp b/src/training/common/networkbuilder.cpp index 5a0d91715c..18a44420e5 100644 --- a/src/training/common/networkbuilder.cpp +++ b/src/training/common/networkbuilder.cpp @@ -19,6 +19,7 @@ #include "networkbuilder.h" #include "convolve.h" +#include "dropout.h" #include "fullyconnected.h" #include "input.h" #include "lstm.h" @@ -28,6 +29,7 @@ #include "reconfig.h" #include "reversed.h" #include "series.h" +#include "tesserrstream.h" // for tesserr #include "unicharset.h" namespace tesseract { @@ -104,6 +106,8 @@ Network *NetworkBuilder::BuildFromString(const StaticShape &input_shape, const c return ParseS(input_shape, str); case 'C': return ParseC(input_shape, str); + case 'D': + return ParseD(input_shape, str); case 'M': return ParseM(input_shape, str); case 'L': @@ -295,6 +299,30 @@ Network *NetworkBuilder::ParseC(const StaticShape &input_shape, const char **str return series; } +// Parses a network that begins with 'D'. +Network *NetworkBuilder::ParseD(const StaticShape &input_shape, const char **str) { + if ((*str)[1] != 'o') { + tesserr << "Invalid Do spec!:" << *str << '\n'; + return nullptr; + } + char *end; + float dropout_rate = strtof(*str + 2, &end); + if (dropout_rate < 0 || dropout_rate >= 1) { + tesserr << "Invalid dropout rate! Must be in [0.0, 1.0): " << dropout_rate << '\n'; + return nullptr; + } + unsigned dropout_dim = 0; + if (*end == ',') { + dropout_dim = static_cast(strtoul(end + 1, &end, 10)); + if (dropout_dim < 1 || dropout_dim > 2) { + tesserr << "Invalid dropout dim! Must be 1 or 2.\n"; + return nullptr; + } + } + *str = end; + return new Dropout("Dropout", input_shape.depth(), dropout_rate, dropout_dim); +} + // Parses a network that begins with 'M'. Network *NetworkBuilder::ParseM(const StaticShape &input_shape, const char **str) { int y = 0, x = 0; diff --git a/src/training/common/networkbuilder.h b/src/training/common/networkbuilder.h index 1ee1900b1c..7f1b2b3cf8 100644 --- a/src/training/common/networkbuilder.h +++ b/src/training/common/networkbuilder.h @@ -83,6 +83,7 @@ class TESS_COMMON_TRAINING_API NetworkBuilder { // C(s|t|r|l|m),, Convolves using a (x,y) window, with no shrinkage, // random infill, producing d outputs, then applies a non-linearity: // s: Sigmoid, t: Tanh, r: Relu, l: Linear, m: Softmax. + // Do Dropout with given rate. // F(s|t|r|l|m) Truly fully-connected with s|t|r|l|m non-linearity and d // outputs. Connects to every x,y,depth position of the input, reducing // height, width to 1, producing a single vector as the output. @@ -136,6 +137,8 @@ class TESS_COMMON_TRAINING_API NetworkBuilder { Network *ParseS(const StaticShape &input_shape, const char **str); // Parses a network that begins with 'C'. Network *ParseC(const StaticShape &input_shape, const char **str); + // Parses a network that begins with 'D'. + Network *ParseD(const StaticShape &input_shape, const char **str); // Parses a network that begins with 'M'. Network *ParseM(const StaticShape &input_shape, const char **str); // Parses an LSTM network, either individual, bi- or quad-directional. diff --git a/unittest/recodebeam_test.cc b/unittest/recodebeam_test.cc index 0fc738576e..867376a53a 100644 --- a/unittest/recodebeam_test.cc +++ b/unittest/recodebeam_test.cc @@ -207,7 +207,7 @@ class RecodeBeamTest : public ::testing::Test { TRand random; for (int t = 0; t < width; ++t) { for (int i = 0; i < num_codes; ++i) { - outputs(t, i) = random.UnsignedRand(0.25); + outputs(t, i) = random.UnsignedRand(0.25f); } } int t = 0;