Implement dropout

Makefile.am

@@ -519,6 +519,7 @@ libtesseract_lstm_la_CPPFLAGS += -DTESSDATA_PREFIX='"@datadir@"'
 endif
 
 noinst_HEADERS += src/lstm/convolve.h
+noinst_HEADERS += src/lstm/dropout.h
 noinst_HEADERS += src/lstm/fullyconnected.h
 noinst_HEADERS += src/lstm/functions.h
 noinst_HEADERS += src/lstm/input.h
@@ -541,6 +542,7 @@ noinst_HEADERS += src/lstm/weightmatrix.h
 noinst_LTLIBRARIES += libtesseract_lstm.la
 
 libtesseract_lstm_la_SOURCES = src/lstm/convolve.cpp
+libtesseract_lstm_la_SOURCES += src/lstm/dropout.cpp
 libtesseract_lstm_la_SOURCES += src/lstm/fullyconnected.cpp
 libtesseract_lstm_la_SOURCES += src/lstm/functions.cpp
 libtesseract_lstm_la_SOURCES += src/lstm/input.cpp

src/lstm/dropout.cpp

@@ -0,0 +1,189 @@
+///////////////////////////////////////////////////////////////////////
+// File:        dropout.cpp
+// Description: Dropout layer.
+// Author:      Stefan Weil
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+///////////////////////////////////////////////////////////////////////
+
+#ifdef HAVE_CONFIG_H
+#  include "config_auto.h"
+#endif
+
+#include "dropout.h"
+#include "networkscratch.h"
+#include "serialis.h"
+#include "tesserrstream.h" // for tesserr
+
+namespace tesseract {
+
+Dropout::Dropout(const std::string &name, int ni, float dropout_rate, int dropout_dim)
+    : Network(NT_DROPOUT, name, ni, ni),
+      dropout_mask_(),
+      dropout_rate_(dropout_rate),
+      dropout_dim_(dropout_dim) {
+  if (dropout_rate_ < 0 || dropout_rate_ >= 1) {
+    throw std::invalid_argument("Invalid dropout rate. Must be in [0, 1).");
+  }
+  if (dropout_dim_ < 0 || dropout_dim_ > 2) {
+    throw std::invalid_argument("Invalid dropout dim. Must be 0, 1 or 2.");
+  }
+}
+
+void Dropout::DebugWeights() {
+  tesserr << "Dropout layer '" << name_ << "': rate=" << dropout_rate_ << '\n';
+}
+
+// Writes to the given file. Returns false in case of error.
+bool Dropout::Serialize(TFile *fp) const {
+  // dropout_mask_ is runtime data and is not serialized.
+  return Network::Serialize(fp) &&
+         fp->Serialize(&dropout_rate_) &&
+         fp->Serialize(&dropout_dim_);
+}
+
+// Reads from the given file. Returns false in case of error.
+bool Dropout::DeSerialize(TFile *fp) {
+  if (!fp->DeSerialize(&dropout_rate_)) {
+    return false;
+  }
+  if (!fp->DeSerialize(&dropout_dim_)) {
+    return false;
+  }
+  no_ = ni_;
+  return true;
+}
+
+// Runs forward propagation of activations on the input line.
+// See Network for a detailed discussion of the arguments.
+void Dropout::Forward(bool debug, const NetworkIO &input,
+                      const TransposedArray *input_transpose,
+                      NetworkScratch *scratch, NetworkIO *output) {
+  // Output has the same shape as input (ni_ == no_).
+  output->Resize(input, no_);
+
+  int width = input.Width();
+  int num_features = input.NumFeatures();
+
+  if (IsTraining() && dropout_rate_ > 0.0f) {
+    // Inverted dropout: scale retained activations by 1/keep_prob so that
+    // inference can pass the network output through unchanged.
+    float keep_prob = 1.0f - dropout_rate_;
+    float scale = 1.0f / keep_prob;
+
+    if (dropout_dim_ == 2) {
+      // Feature dropout: one mask value per feature, shared across all timesteps.
+      dropout_mask_.resize(num_features);
+      for (int i = 0; i < num_features; ++i) {
+        float r = static_cast<float>(randomizer_->UnsignedRand(1.0));
+        dropout_mask_[i] = (r < keep_prob) ? 1 : 0;
+      }
+      for (int t = 0; t < width; ++t) {
+        const float *in = input.f(t);
+        float *out = output->f(t);
+        for (int i = 0; i < num_features; ++i)
+          out[i] = dropout_mask_[i] ? in[i] * scale : 0.0f;
+      }
+    } else if (dropout_dim_ == 1) {
+      // Temporal dropout: one mask value per timestep, shared across all features.
+      dropout_mask_.resize(width);
+      for (int t = 0; t < width; ++t) {
+        float r = static_cast<float>(randomizer_->UnsignedRand(1.0));
+        dropout_mask_[t] = (r < keep_prob) ? 1 : 0;
+      }
+      for (int t = 0; t < width; ++t) {
+        const float *in = input.f(t);
+        float *out = output->f(t);
+        if (dropout_mask_[t]) {
+          for (int i = 0; i < num_features; ++i) out[i] = in[i] * scale;
+        } else {
+          memset(out, 0, sizeof(float) * num_features);
+        }
+      }
+    } else {
+      // Element-wise dropout (dim=0, default).
+      dropout_mask_.resize(width * num_features);
+      for (int t = 0; t < width; ++t) {
+        const float *in = input.f(t);
+        float *out = output->f(t);
+        char *mask = dropout_mask_.data() + t * num_features;
+        for (int i = 0; i < num_features; ++i) {
+          // UnsignedRand(1.0) returns a value in [0, 1].
+          float r = static_cast<float>(randomizer_->UnsignedRand(1.0));
+          mask[i] = (r < keep_prob) ? 1 : 0;
+          out[i] = mask[i] ? in[i] * scale : 0.0f;
+        }
+      }
+    }
+  } else {
+    // Inference mode (or dropout_rate_ == 0): pass input through unchanged.
+    output->CopyAll(input);
+  }
+
+#ifndef GRAPHICS_DISABLED
+  if (debug) {
+    DisplayForward(*output);
+  }
+#endif
+}
+
+// Runs backward propagation of errors on the deltas line.
+// See Network for a detailed discussion of the arguments.
+bool Dropout::Backward(bool debug, const NetworkIO &fwd_deltas,
+                       NetworkScratch *scratch, NetworkIO *back_deltas) {
+  back_deltas->Resize(fwd_deltas, ni_);
+
+  int width = fwd_deltas.Width();
+  int num_features = fwd_deltas.NumFeatures();
+
+  if (IsTraining() && dropout_rate_ > 0.0f) {
+    float scale = 1.0f / (1.0f - dropout_rate_);
+
+    if (dropout_dim_ == 2) {
+      for (int t = 0; t < width; ++t) {
+        const float *in = fwd_deltas.f(t);
+        float *out = back_deltas->f(t);
+        for (int i = 0; i < num_features; ++i)
+          out[i] = dropout_mask_[i] ? in[i] * scale : 0.0f;
+      }
+    } else if (dropout_dim_ == 1) {
+      for (int t = 0; t < width; ++t) {
+        const float *in = fwd_deltas.f(t);
+        float *out = back_deltas->f(t);
+        if (dropout_mask_[t]) {
+          for (int i = 0; i < num_features; ++i) out[i] = in[i] * scale;
+        } else {
+          memset(out, 0, sizeof(float) * num_features);
+        }
+      }
+    } else {
+      for (int t = 0; t < width; ++t) {
+        const float *in = fwd_deltas.f(t);
+        float *out = back_deltas->f(t);
+        const char *mask = dropout_mask_.data() + t * num_features;
+        for (int i = 0; i < num_features; ++i)
+          out[i] = mask[i] ? in[i] * scale : 0.0f;
+      }
+    }
+  } else {
+    // Inference mode: pass gradients through unchanged.
+    back_deltas->CopyAll(fwd_deltas);
+  }
+
+#ifndef GRAPHICS_DISABLED
+  if (debug) {
+    DisplayBackward(*back_deltas);
+  }
+#endif
+  return needs_to_backprop_;
+}
+
+} // namespace tesseract.

src/lstm/dropout.h

@@ -0,0 +1,64 @@
+///////////////////////////////////////////////////////////////////////
+// File:        dropout.h
+// Description: Standard Dropout layer.
+// Author:      Stefan Weil
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+///////////////////////////////////////////////////////////////////////
+
+#ifndef TESSERACT_LSTM_DROPOUT_H_
+#define TESSERACT_LSTM_DROPOUT_H_
+
+#include "network.h"
+
+namespace tesseract {
+
+// Dropout.
+class Dropout : public Network {
+public:
+  TESS_API
+  Dropout(const std::string &name, int ni, float dropout_rate, int dropout_dim = 0);
+  ~Dropout() override = default;
+
+  // Accessors.
+  std::string spec() const override {
+    std::string s = "Do" + std::to_string(dropout_rate_);
+    if (dropout_dim_ != 0)
+      s += "," + std::to_string(dropout_dim_);
+    return s;
+  }
+
+  // Writes to the given file. Returns false in case of error.
+  bool Serialize(TFile *fp) const override;
+  // Reads from the given file. Returns false in case of error.
+  bool DeSerialize(TFile *fp) override;
+
+  // Runs forward propagation of activations on the input line.
+  // See Network for a detailed discussion of the arguments.
+  void Forward(bool debug, const NetworkIO &input, const TransposedArray *input_transpose,
+               NetworkScratch *scratch, NetworkIO *output) override;
+
+  // Runs backward propagation of errors on the deltas line.
+  // See Network for a detailed discussion of the arguments.
+  bool Backward(bool debug, const NetworkIO &fwd_deltas, NetworkScratch *scratch,
+                NetworkIO *back_deltas) override;
+
+private:
+  void DebugWeights() override;
+
+  std::vector<char> dropout_mask_;
+  float dropout_rate_;
+  int dropout_dim_;  // 0=elementwise, 1=temporal, 2=feature/channel
+};
+
+} // namespace tesseract.
+
+#endif // TESSERACT_LSTM_DROPOUT_H_

src/lstm/network.cpp

@@ -28,6 +28,7 @@
 // factory deserializing method: CreateFromFile.
 #include <allheaders.h>
 #include "convolve.h"
+#include "dropout.h"
 #include "fullyconnected.h"
 #include "input.h"
 #include "lstm.h"
@@ -59,7 +60,7 @@ const int kYWinFrameSize = 80;
 // layer types in NetworkType without invalidating existing network files.
 static char const *const kTypeNames[NT_COUNT] = {
     "Invalid",     "Input",
-    "Convolve",    "Maxpool",
+    "Convolve", "Dropout", "Maxpool",
     "Parallel",    "Replicated",
     "ParBidiLSTM", "DepParUDLSTM",
     "Par2dLSTM",   "Series",
@@ -251,6 +252,9 @@ Network *Network::CreateFromFile(TFile *fp) {
     case NT_CONVOLVE:
       network = new Convolve(name, ni, 0, 0);
       break;
+    case NT_DROPOUT:
+      network = new Dropout(name, ni, 0.5f);
+      break;
     case NT_INPUT:
       network = new Input(name, ni, no);
       break;

src/lstm/network.h

@@ -42,6 +42,7 @@ enum NetworkType {
   NT_INPUT, // Inputs from an image.
   // Plumbing networks combine other networks or rearrange the inputs.
   NT_CONVOLVE,    // Duplicates inputs in a sliding window neighborhood.
+  NT_DROPOUT,     // Dropout random inputs.
   NT_MAXPOOL,     // Chooses the max result from a rectangle.
   NT_PARALLEL,    // Runs networks in parallel.
   NT_REPLICATED,  // Runs identical networks in parallel.

src/training/common/networkbuilder.cpp

@@ -19,6 +19,7 @@
 #include "networkbuilder.h"
 
 #include "convolve.h"
+#include "dropout.h"
 #include "fullyconnected.h"
 #include "input.h"
 #include "lstm.h"
@@ -28,6 +29,7 @@
 #include "reconfig.h"
 #include "reversed.h"
 #include "series.h"
+#include "tesserrstream.h"  // for tesserr
 #include "unicharset.h"
 
 namespace tesseract {
@@ -104,6 +106,8 @@ Network *NetworkBuilder::BuildFromString(const StaticShape &input_shape, const c
       return ParseS(input_shape, str);
     case 'C':
       return ParseC(input_shape, str);
+    case 'D':
+      return ParseD(input_shape, str);
     case 'M':
       return ParseM(input_shape, str);
     case 'L':
@@ -295,6 +299,30 @@ Network *NetworkBuilder::ParseC(const StaticShape &input_shape, const char **str
   return series;
 }
 
+// Parses a network that begins with 'D'.
+Network *NetworkBuilder::ParseD(const StaticShape &input_shape, const char **str) {
+  if ((*str)[1] != 'o') {
+    tesserr << "Invalid Do spec!:" << *str << '\n';
+    return nullptr;
+  }
+  char *end;
+  float dropout_rate = strtof(*str + 2, &end);
+  if (dropout_rate < 0 || dropout_rate >= 1) {
+    tesserr << "Invalid dropout rate! Must be in [0.0, 1.0): " << dropout_rate << '\n';
+    return nullptr;
+  }
+  int dropout_dim = 0;
+  if (*end == ',') {
+    dropout_dim = static_cast<int>(strtol(end + 1, &end, 10));
+    if (dropout_dim < 1 || dropout_dim > 2) {
+      tesserr << "Invalid dropout dim! Must be 1 or 2.\n";
+      return nullptr;
+    }
+  }
+  *str = end;
+  return new Dropout("Dropout", input_shape.depth(), dropout_rate, dropout_dim);
+}
+
 // Parses a network that begins with 'M'.
 Network *NetworkBuilder::ParseM(const StaticShape &input_shape, const char **str) {
   int y = 0, x = 0;

src/training/common/networkbuilder.h

@@ -83,6 +83,7 @@ class TESS_COMMON_TRAINING_API NetworkBuilder {
   // C(s|t|r|l|m)<y>,<x>,<d> Convolves using a (x,y) window, with no shrinkage,
   //   random infill, producing d outputs, then applies a non-linearity:
   //   s: Sigmoid, t: Tanh, r: Relu, l: Linear, m: Softmax.
+  // Do<rate> Dropout with given rate.
   // F(s|t|r|l|m)<d> Truly fully-connected with s|t|r|l|m non-linearity and d
   //   outputs. Connects to every x,y,depth position of the input, reducing
   //   height, width to 1, producing a single <d> vector as the output.
@@ -136,6 +137,8 @@ class TESS_COMMON_TRAINING_API NetworkBuilder {
   Network *ParseS(const StaticShape &input_shape, const char **str);
   // Parses a network that begins with 'C'.
   Network *ParseC(const StaticShape &input_shape, const char **str);
+  // Parses a network that begins with 'D'.
+  Network *ParseD(const StaticShape &input_shape, const char **str);
   // Parses a network that begins with 'M'.
   Network *ParseM(const StaticShape &input_shape, const char **str);
   // Parses an LSTM network, either individual, bi- or quad-directional.