From 934f0fbf04dcbb3fcbb158f30c286568f3269c56 Mon Sep 17 00:00:00 2001 From: Stefan Weil Date: Sat, 11 Nov 2023 10:20:39 +0100 Subject: [PATCH 01/14] Add new network layer for dropout Signed-off-by: Stefan Weil --- Makefile.am | 2 + src/lstm/dropout.cpp | 124 +++++++++++++++++++++++++++++++++++++++++++ src/lstm/dropout.h | 62 ++++++++++++++++++++++ src/lstm/network.cpp | 6 ++- src/lstm/network.h | 1 + 5 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 src/lstm/dropout.cpp create mode 100644 src/lstm/dropout.h diff --git a/Makefile.am b/Makefile.am index f47afec6f8..87a05a222b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -519,6 +519,7 @@ libtesseract_lstm_la_CPPFLAGS += -DTESSDATA_PREFIX='"@datadir@"' endif noinst_HEADERS += src/lstm/convolve.h +noinst_HEADERS += src/lstm/dropout.h noinst_HEADERS += src/lstm/fullyconnected.h noinst_HEADERS += src/lstm/functions.h noinst_HEADERS += src/lstm/input.h @@ -541,6 +542,7 @@ noinst_HEADERS += src/lstm/weightmatrix.h noinst_LTLIBRARIES += libtesseract_lstm.la libtesseract_lstm_la_SOURCES = src/lstm/convolve.cpp +libtesseract_lstm_la_SOURCES += src/lstm/dropout.cpp libtesseract_lstm_la_SOURCES += src/lstm/fullyconnected.cpp libtesseract_lstm_la_SOURCES += src/lstm/functions.cpp libtesseract_lstm_la_SOURCES += src/lstm/input.cpp diff --git a/src/lstm/dropout.cpp b/src/lstm/dropout.cpp new file mode 100644 index 0000000000..9c5725d96a --- /dev/null +++ b/src/lstm/dropout.cpp @@ -0,0 +1,124 @@ +/////////////////////////////////////////////////////////////////////// +// File: dropout.cpp +// Description: Dropout layer. +// Author: Stefan Weil +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +/////////////////////////////////////////////////////////////////////// + +#ifdef HAVE_CONFIG_H +# include "config_auto.h" +#endif + +#include "dropout.h" + +#include "networkscratch.h" +#include "serialis.h" + +namespace tesseract { + +Dropout::Dropout(const std::string &name, int ni, float probability, uint8_t dimensions) + : Network(NT_DROPOUT, name, ni, 0), + probability_(probability), + dimensions_(dimensions) +{ +} + +// Writes to the given file. Returns false in case of error. +bool Dropout::Serialize(TFile *fp) const { + return Network::Serialize(fp) && fp->Serialize(&probability_) && fp->Serialize(&dimensions_); +} + +// Reads from the given file. Returns false in case of error. +bool Dropout::DeSerialize(TFile *fp) { + if (!fp->DeSerialize(&probability_)) { + return false; + } + if (!fp->DeSerialize(&dimensions_)) { + return false; + } + no_ = ni_; + return true; +} + +// Runs forward propagation of activations on the input line. +// See NetworkCpp for a detailed discussion of the arguments. +void Dropout::Forward(bool debug, const NetworkIO &input, const TransposedArray *input_transpose, + NetworkScratch *scratch, NetworkIO *output) { +#if 0 + output->Resize(input, no_); + int y_scale = 2 * half_y_ + 1; + StrideMap::Index dest_index(output->stride_map()); + do { + // Stack x_scale groups of y_scale * ni_ inputs together. + int t = dest_index.t(); + int out_ix = 0; + for (int x = -half_x_; x <= half_x_; ++x, out_ix += y_scale * ni_) { + StrideMap::Index x_index(dest_index); + if (!x_index.AddOffset(x, FD_WIDTH)) { + // This x is outside the image. + output->Randomize(t, out_ix, y_scale * ni_, randomizer_); + } else { + int out_iy = out_ix; + for (int y = -half_y_; y <= half_y_; ++y, out_iy += ni_) { + StrideMap::Index y_index(x_index); + if (!y_index.AddOffset(y, FD_HEIGHT)) { + // This y is outside the image. + output->Randomize(t, out_iy, ni_, randomizer_); + } else { + output->CopyTimeStepGeneral(t, out_iy, ni_, input, y_index.t(), 0); + } + } + } + } + } while (dest_index.Increment()); +#endif +#ifndef GRAPHICS_DISABLED + if (debug) { + DisplayForward(*output); + } +#endif +} + +// Runs backward propagation of errors on the deltas line. +// See NetworkCpp for a detailed discussion of the arguments. +bool Dropout::Backward(bool debug, const NetworkIO &fwd_deltas, NetworkScratch *scratch, + NetworkIO *back_deltas) { +#if 0 + back_deltas->Resize(fwd_deltas, ni_); + NetworkScratch::IO delta_sum; + delta_sum.ResizeFloat(fwd_deltas, ni_, scratch); + delta_sum->Zero(); + int y_scale = 2 * half_y_ + 1; + StrideMap::Index src_index(fwd_deltas.stride_map()); + do { + // Stack x_scale groups of y_scale * ni_ inputs together. + int t = src_index.t(); + int out_ix = 0; + for (int x = -half_x_; x <= half_x_; ++x, out_ix += y_scale * ni_) { + StrideMap::Index x_index(src_index); + if (x_index.AddOffset(x, FD_WIDTH)) { + int out_iy = out_ix; + for (int y = -half_y_; y <= half_y_; ++y, out_iy += ni_) { + StrideMap::Index y_index(x_index); + if (y_index.AddOffset(y, FD_HEIGHT)) { + fwd_deltas.AddTimeStepPart(t, out_iy, ni_, delta_sum->f(y_index.t())); + } + } + } + } + } while (src_index.Increment()); + back_deltas->CopyAll(*delta_sum); +#endif + return true; +} + +} // namespace tesseract. diff --git a/src/lstm/dropout.h b/src/lstm/dropout.h new file mode 100644 index 0000000000..bc929b1d53 --- /dev/null +++ b/src/lstm/dropout.h @@ -0,0 +1,62 @@ +/////////////////////////////////////////////////////////////////////// +// File: dropout.h +// Description: Standard Dropout layer. +// Author: Stefan Weil +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +/////////////////////////////////////////////////////////////////////// + +#ifndef TESSERACT_LSTM_DROPOUT_H_ +#define TESSERACT_LSTM_DROPOUT_H_ + +#include "network.h" + +namespace tesseract { + +// Dropout. +class Dropout : public Network { +public: + TESS_API + Dropout(const std::string &name, int ni, float probability, uint8_t dimensions); + ~Dropout() override = default; + + // Accessors. + std::string spec() const override { + return "Do"; + } + + // Writes to the given file. Returns false in case of error. + bool Serialize(TFile *fp) const override; + // Reads from the given file. Returns false in case of error. + bool DeSerialize(TFile *fp) override; + + // Runs forward propagation of activations on the input line. + // See Network for a detailed discussion of the arguments. + void Forward(bool debug, const NetworkIO &input, const TransposedArray *input_transpose, + NetworkScratch *scratch, NetworkIO *output) override; + + // Runs backward propagation of errors on the deltas line. + // See Network for a detailed discussion of the arguments. + bool Backward(bool debug, const NetworkIO &fwd_deltas, NetworkScratch *scratch, + NetworkIO *back_deltas) override; + +private: + void DebugWeights() override { + tprintf("Must override Network::DebugWeights for type %d\n", type_); + } + + float probability_; + uint8_t dimensions_; +}; + +} // namespace tesseract. + +#endif // TESSERACT_LSTM_DROPOUT_H_ diff --git a/src/lstm/network.cpp b/src/lstm/network.cpp index cfddbfd43a..ba41187e66 100644 --- a/src/lstm/network.cpp +++ b/src/lstm/network.cpp @@ -28,6 +28,7 @@ // factory deserializing method: CreateFromFile. #include #include "convolve.h" +#include "dropout.h" #include "fullyconnected.h" #include "input.h" #include "lstm.h" @@ -59,7 +60,7 @@ const int kYWinFrameSize = 80; // layer types in NetworkType without invalidating existing network files. static char const *const kTypeNames[NT_COUNT] = { "Invalid", "Input", - "Convolve", "Maxpool", + "Convolve", "Dropout", "Maxpool", "Parallel", "Replicated", "ParBidiLSTM", "DepParUDLSTM", "Par2dLSTM", "Series", @@ -251,6 +252,9 @@ Network *Network::CreateFromFile(TFile *fp) { case NT_CONVOLVE: network = new Convolve(name, ni, 0, 0); break; + case NT_DROPOUT: + network = new Dropout(name, ni, 0.5f, 1); + break; case NT_INPUT: network = new Input(name, ni, no); break; diff --git a/src/lstm/network.h b/src/lstm/network.h index 353b110c92..d8b238b07d 100644 --- a/src/lstm/network.h +++ b/src/lstm/network.h @@ -42,6 +42,7 @@ enum NetworkType { NT_INPUT, // Inputs from an image. // Plumbing networks combine other networks or rearrange the inputs. NT_CONVOLVE, // Duplicates inputs in a sliding window neighborhood. + NT_DROPOUT, // Dropout random inputs. NT_MAXPOOL, // Chooses the max result from a rectangle. NT_PARALLEL, // Runs networks in parallel. NT_REPLICATED, // Runs identical networks in parallel. From 0cfa2d5f655e55905dcfff2235404bea3feec687 Mon Sep 17 00:00:00 2001 From: Stefan Weil Date: Sat, 11 Nov 2023 15:26:39 +0100 Subject: [PATCH 02/14] dropout (w.i.p.) --- src/lstm/dropout.cpp | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/lstm/dropout.cpp b/src/lstm/dropout.cpp index 9c5725d96a..3ed411751d 100644 --- a/src/lstm/dropout.cpp +++ b/src/lstm/dropout.cpp @@ -18,28 +18,32 @@ # include "config_auto.h" #endif -#include "dropout.h" +#include +#include "dropout.h" #include "networkscratch.h" #include "serialis.h" namespace tesseract { -Dropout::Dropout(const std::string &name, int ni, float probability, uint8_t dimensions) +Dropout::Dropout(const std::string &name, int ni, float dropout_rate, uint8_t dimensions) : Network(NT_DROPOUT, name, ni, 0), - probability_(probability), + dropout_rate_(dropout_rate), dimensions_(dimensions) { + if (dropout_rate_ < 0 || dropout_rate_ >= 1) { + throw std::invalid_argument("Invalid dropout rate. Must be in [0, 1)."); + } } // Writes to the given file. Returns false in case of error. bool Dropout::Serialize(TFile *fp) const { - return Network::Serialize(fp) && fp->Serialize(&probability_) && fp->Serialize(&dimensions_); + return Network::Serialize(fp) && fp->Serialize(&dropout_rate_) && fp->Serialize(&dimensions_); } // Reads from the given file. Returns false in case of error. bool Dropout::DeSerialize(TFile *fp) { - if (!fp->DeSerialize(&probability_)) { + if (!fp->DeSerialize(&dropout_rate_)) { return false; } if (!fp->DeSerialize(&dimensions_)) { @@ -53,8 +57,8 @@ bool Dropout::DeSerialize(TFile *fp) { // See NetworkCpp for a detailed discussion of the arguments. void Dropout::Forward(bool debug, const NetworkIO &input, const TransposedArray *input_transpose, NetworkScratch *scratch, NetworkIO *output) { + *output = input; #if 0 - output->Resize(input, no_); int y_scale = 2 * half_y_ + 1; StrideMap::Index dest_index(output->stride_map()); do { @@ -92,6 +96,21 @@ void Dropout::Forward(bool debug, const NetworkIO &input, const TransposedArray // See NetworkCpp for a detailed discussion of the arguments. bool Dropout::Backward(bool debug, const NetworkIO &fwd_deltas, NetworkScratch *scratch, NetworkIO *back_deltas) { + output->Resize(input, no_); + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_real_distribution dist(0.0f, 1.0f); + + for (unsigned i = 0; i < ni_; i++) { + if (dist(gen) >= dropout_rate_) { + // Keep the neuron + output->push_back(input[i]); + } else { + // Drop the neuron + output->push_back(0.0f); + } + } + #if 0 back_deltas->Resize(fwd_deltas, ni_); NetworkScratch::IO delta_sum; From 0f766d918d728ffcdc08e7afb2cddf7f6bc05920 Mon Sep 17 00:00:00 2001 From: Stefan Weil Date: Sat, 11 Nov 2023 15:29:32 +0100 Subject: [PATCH 03/14] dropout --- src/lstm/dropout.cpp | 6 +++++- src/lstm/dropout.h | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/lstm/dropout.cpp b/src/lstm/dropout.cpp index 3ed411751d..facd4fa0a8 100644 --- a/src/lstm/dropout.cpp +++ b/src/lstm/dropout.cpp @@ -96,7 +96,10 @@ void Dropout::Forward(bool debug, const NetworkIO &input, const TransposedArray // See NetworkCpp for a detailed discussion of the arguments. bool Dropout::Backward(bool debug, const NetworkIO &fwd_deltas, NetworkScratch *scratch, NetworkIO *back_deltas) { - output->Resize(input, no_); + tprintf("%s: missing implementation\n", __FUNCTION__); + +#if 0 + back_deltas->Resize(input, no_); std::random_device rd; std::mt19937 gen(rd()); std::uniform_real_distribution dist(0.0f, 1.0f); @@ -110,6 +113,7 @@ bool Dropout::Backward(bool debug, const NetworkIO &fwd_deltas, NetworkScratch * output->push_back(0.0f); } } +#endif #if 0 back_deltas->Resize(fwd_deltas, ni_); diff --git a/src/lstm/dropout.h b/src/lstm/dropout.h index bc929b1d53..c8b512528f 100644 --- a/src/lstm/dropout.h +++ b/src/lstm/dropout.h @@ -25,12 +25,12 @@ namespace tesseract { class Dropout : public Network { public: TESS_API - Dropout(const std::string &name, int ni, float probability, uint8_t dimensions); + Dropout(const std::string &name, int ni, float dropout_rate, uint8_t dimensions); ~Dropout() override = default; // Accessors. std::string spec() const override { - return "Do"; + return "Do" + std::to_string(dropout_rate_) + "," + std::to_string(dimensions_); } // Writes to the given file. Returns false in case of error. @@ -53,7 +53,7 @@ class Dropout : public Network { tprintf("Must override Network::DebugWeights for type %d\n", type_); } - float probability_; + float dropout_rate_; uint8_t dimensions_; }; From 3e8b1a161ff902c5efe818e0341f5511c8f2e99e Mon Sep 17 00:00:00 2001 From: Stefan Weil Date: Wed, 29 May 2024 06:55:17 +0200 Subject: [PATCH 04/14] Update dropout code (still unfinished) Signed-off-by: Stefan Weil --- src/lstm/dropout.cpp | 32 ++------------------------------ 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/src/lstm/dropout.cpp b/src/lstm/dropout.cpp index facd4fa0a8..f645f67729 100644 --- a/src/lstm/dropout.cpp +++ b/src/lstm/dropout.cpp @@ -58,33 +58,6 @@ bool Dropout::DeSerialize(TFile *fp) { void Dropout::Forward(bool debug, const NetworkIO &input, const TransposedArray *input_transpose, NetworkScratch *scratch, NetworkIO *output) { *output = input; -#if 0 - int y_scale = 2 * half_y_ + 1; - StrideMap::Index dest_index(output->stride_map()); - do { - // Stack x_scale groups of y_scale * ni_ inputs together. - int t = dest_index.t(); - int out_ix = 0; - for (int x = -half_x_; x <= half_x_; ++x, out_ix += y_scale * ni_) { - StrideMap::Index x_index(dest_index); - if (!x_index.AddOffset(x, FD_WIDTH)) { - // This x is outside the image. - output->Randomize(t, out_ix, y_scale * ni_, randomizer_); - } else { - int out_iy = out_ix; - for (int y = -half_y_; y <= half_y_; ++y, out_iy += ni_) { - StrideMap::Index y_index(x_index); - if (!y_index.AddOffset(y, FD_HEIGHT)) { - // This y is outside the image. - output->Randomize(t, out_iy, ni_, randomizer_); - } else { - output->CopyTimeStepGeneral(t, out_iy, ni_, input, y_index.t(), 0); - } - } - } - } - } while (dest_index.Increment()); -#endif #ifndef GRAPHICS_DISABLED if (debug) { DisplayForward(*output); @@ -98,12 +71,12 @@ bool Dropout::Backward(bool debug, const NetworkIO &fwd_deltas, NetworkScratch * NetworkIO *back_deltas) { tprintf("%s: missing implementation\n", __FUNCTION__); -#if 0 - back_deltas->Resize(input, no_); std::random_device rd; std::mt19937 gen(rd()); std::uniform_real_distribution dist(0.0f, 1.0f); + back_deltas->Resize(fwd_deltas, ni_); +#if 0 for (unsigned i = 0; i < ni_; i++) { if (dist(gen) >= dropout_rate_) { // Keep the neuron @@ -116,7 +89,6 @@ bool Dropout::Backward(bool debug, const NetworkIO &fwd_deltas, NetworkScratch * #endif #if 0 - back_deltas->Resize(fwd_deltas, ni_); NetworkScratch::IO delta_sum; delta_sum.ResizeFloat(fwd_deltas, ni_, scratch); delta_sum->Zero(); From aad5646635f589a3fb8d7321214248701708349c Mon Sep 17 00:00:00 2001 From: Stefan Weil Date: Thu, 20 Jun 2024 20:43:52 +0200 Subject: [PATCH 05/14] Update dropout code (still unfinished) Signed-off-by: Stefan Weil --- src/lstm/dropout.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lstm/dropout.cpp b/src/lstm/dropout.cpp index f645f67729..6eb6364b6a 100644 --- a/src/lstm/dropout.cpp +++ b/src/lstm/dropout.cpp @@ -57,7 +57,10 @@ bool Dropout::DeSerialize(TFile *fp) { // See NetworkCpp for a detailed discussion of the arguments. void Dropout::Forward(bool debug, const NetworkIO &input, const TransposedArray *input_transpose, NetworkScratch *scratch, NetworkIO *output) { - *output = input; + if (IsTraining()) { + } else { + *output = input; + } #ifndef GRAPHICS_DISABLED if (debug) { DisplayForward(*output); From a9e390445f08374f2e5bdb8085e32f416c6072fb Mon Sep 17 00:00:00 2001 From: Stefan Weil Date: Thu, 19 Dec 2024 22:29:08 +0100 Subject: [PATCH 06/14] Update dropout code (still unfinished) Signed-off-by: Stefan Weil --- src/lstm/dropout.cpp | 7 ++++++- src/lstm/dropout.h | 4 +--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/lstm/dropout.cpp b/src/lstm/dropout.cpp index 6eb6364b6a..b3c3e50292 100644 --- a/src/lstm/dropout.cpp +++ b/src/lstm/dropout.cpp @@ -23,6 +23,7 @@ #include "dropout.h" #include "networkscratch.h" #include "serialis.h" +#include "tesserrstream.h" // for tesserr namespace tesseract { @@ -36,6 +37,10 @@ Dropout::Dropout(const std::string &name, int ni, float dropout_rate, uint8_t di } } +void Dropout::DebugWeights() { + tesserr << "Must override Network::DebugWeights for type " << type_ << '\n'; +} + // Writes to the given file. Returns false in case of error. bool Dropout::Serialize(TFile *fp) const { return Network::Serialize(fp) && fp->Serialize(&dropout_rate_) && fp->Serialize(&dimensions_); @@ -72,7 +77,7 @@ void Dropout::Forward(bool debug, const NetworkIO &input, const TransposedArray // See NetworkCpp for a detailed discussion of the arguments. bool Dropout::Backward(bool debug, const NetworkIO &fwd_deltas, NetworkScratch *scratch, NetworkIO *back_deltas) { - tprintf("%s: missing implementation\n", __FUNCTION__); + tesserr << __FUNCTION__ << ": missing implementation\n"; std::random_device rd; std::mt19937 gen(rd()); diff --git a/src/lstm/dropout.h b/src/lstm/dropout.h index c8b512528f..c178218a27 100644 --- a/src/lstm/dropout.h +++ b/src/lstm/dropout.h @@ -49,9 +49,7 @@ class Dropout : public Network { NetworkIO *back_deltas) override; private: - void DebugWeights() override { - tprintf("Must override Network::DebugWeights for type %d\n", type_); - } + void DebugWeights() override; float dropout_rate_; uint8_t dimensions_; From 41af4f868c6b3182880e84f270ddd1153a97db42 Mon Sep 17 00:00:00 2001 From: Stefan Weil Date: Thu, 16 Jan 2025 12:49:51 +0100 Subject: [PATCH 07/14] Update dropout code Signed-off-by: Stefan Weil --- src/lstm/dropout.cpp | 48 ++++++++++++++++++++++++++++++++++++++++++-- src/lstm/dropout.h | 1 + 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/lstm/dropout.cpp b/src/lstm/dropout.cpp index b3c3e50292..3e459e1205 100644 --- a/src/lstm/dropout.cpp +++ b/src/lstm/dropout.cpp @@ -62,9 +62,40 @@ bool Dropout::DeSerialize(TFile *fp) { // See NetworkCpp for a detailed discussion of the arguments. void Dropout::Forward(bool debug, const NetworkIO &input, const TransposedArray *input_transpose, NetworkScratch *scratch, NetworkIO *output) { + // Resize output to match input dimensions + output->Resize(input.NumFeatures(), input.TimeSize(), input.BatchSize()); + if (IsTraining()) { + // Initialize the dropout mask + dropout_mask_.Resize(input.NumFeatures(), input.TimeSize(), input.BatchSize()); + + float retain_prob = 1.0f - dropout_rate_; + + // Random number generator setup + std::mt19937 generator; // You may need to use Tesseract's RNG + std::bernoulli_distribution distribution(retain_prob); + + // Apply dropout mask + for (int b = 0; b < input.BatchSize(); ++b) { + for (int t = 0; t < input.TimeSize(); ++t) { + const float* input_features = input.f(b, t); + float* output_features = output->f(b, t); + float* mask_features = dropout_mask_.f(b, t); + for (int i = 0; i < input.NumFeatures(); ++i) { + bool retain = distribution(generator); + mask_features[i] = retain ? 1.0f : 0.0f; + // Scale the activations to maintain expected value + output_features[i] = input_features[i] * mask_features[i] / retain_prob; + } + } + } } else { - *output = input; + // During inference, pass input to output unchanged + output->CopyAll(input); + } + + if (debug) { + tprintf("Dropout Forward Pass Complete.\n"); } #ifndef GRAPHICS_DISABLED if (debug) { @@ -77,6 +108,20 @@ void Dropout::Forward(bool debug, const NetworkIO &input, const TransposedArray // See NetworkCpp for a detailed discussion of the arguments. bool Dropout::Backward(bool debug, const NetworkIO &fwd_deltas, NetworkScratch *scratch, NetworkIO *back_deltas) { + int size = deltas.Size(); + input_deltas->Resize(size); + + if (IsTraining()) { + for (int i = 0; i < size; ++i) { + (*input_deltas)(i) = deltas(i) * dropout_mask_[i]; + } + } else { + for (int i = 0; i < size; ++i) { + (*input_deltas)(i) = deltas(i) * (1.0f - dropout_rate_); + } + } + +#if 0 tesserr << __FUNCTION__ << ": missing implementation\n"; std::random_device rd; @@ -84,7 +129,6 @@ bool Dropout::Backward(bool debug, const NetworkIO &fwd_deltas, NetworkScratch * std::uniform_real_distribution dist(0.0f, 1.0f); back_deltas->Resize(fwd_deltas, ni_); -#if 0 for (unsigned i = 0; i < ni_; i++) { if (dist(gen) >= dropout_rate_) { // Keep the neuron diff --git a/src/lstm/dropout.h b/src/lstm/dropout.h index c178218a27..6b5d2f6c76 100644 --- a/src/lstm/dropout.h +++ b/src/lstm/dropout.h @@ -51,6 +51,7 @@ class Dropout : public Network { private: void DebugWeights() override; + std::vector dropout_mask_; float dropout_rate_; uint8_t dimensions_; }; From 172d6a5b455eb833329ffd12fe114294a6d1a5d0 Mon Sep 17 00:00:00 2001 From: Stefan Weil Date: Thu, 9 Oct 2025 22:09:59 +0200 Subject: [PATCH 08/14] Update dropout code (created with help from qwen3-coder) Signed-off-by: Stefan Weil --- src/lstm/dropout.cpp | 28 +++++++++++++++++++------- src/lstm/dropout.h | 5 ++--- src/lstm/network.cpp | 2 +- src/training/common/networkbuilder.cpp | 20 ++++++++++++++++++ src/training/common/networkbuilder.h | 3 +++ 5 files changed, 47 insertions(+), 11 deletions(-) diff --git a/src/lstm/dropout.cpp b/src/lstm/dropout.cpp index 3e459e1205..ea85f66e9e 100644 --- a/src/lstm/dropout.cpp +++ b/src/lstm/dropout.cpp @@ -27,10 +27,9 @@ namespace tesseract { -Dropout::Dropout(const std::string &name, int ni, float dropout_rate, uint8_t dimensions) +Dropout::Dropout(const std::string &name, int ni, float dropout_rate) : Network(NT_DROPOUT, name, ni, 0), - dropout_rate_(dropout_rate), - dimensions_(dimensions) + dropout_rate_(dropout_rate) { if (dropout_rate_ < 0 || dropout_rate_ >= 1) { throw std::invalid_argument("Invalid dropout rate. Must be in [0, 1)."); @@ -43,7 +42,7 @@ void Dropout::DebugWeights() { // Writes to the given file. Returns false in case of error. bool Dropout::Serialize(TFile *fp) const { - return Network::Serialize(fp) && fp->Serialize(&dropout_rate_) && fp->Serialize(&dimensions_); + return Network::Serialize(fp) && fp->Serialize(&dropout_rate_); } // Reads from the given file. Returns false in case of error. @@ -51,9 +50,6 @@ bool Dropout::DeSerialize(TFile *fp) { if (!fp->DeSerialize(&dropout_rate_)) { return false; } - if (!fp->DeSerialize(&dimensions_)) { - return false; - } no_ = ni_; return true; } @@ -62,6 +58,15 @@ bool Dropout::DeSerialize(TFile *fp) { // See NetworkCpp for a detailed discussion of the arguments. void Dropout::Forward(bool debug, const NetworkIO &input, const TransposedArray *input_transpose, NetworkScratch *scratch, NetworkIO *output) { + if (IsTraining() && dropout_rate_ > 0) { + // Apply dropout: randomly zero out neurons + // Generate random mask, apply to input + } else { + // Inference mode: scale by (1 - dropout_rate) + // or just pass through unchanged + *output = input; + } +#if 0 // Resize output to match input dimensions output->Resize(input.NumFeatures(), input.TimeSize(), input.BatchSize()); @@ -102,12 +107,20 @@ void Dropout::Forward(bool debug, const NetworkIO &input, const TransposedArray DisplayForward(*output); } #endif +#endif } // Runs backward propagation of errors on the deltas line. // See NetworkCpp for a detailed discussion of the arguments. bool Dropout::Backward(bool debug, const NetworkIO &fwd_deltas, NetworkScratch *scratch, NetworkIO *back_deltas) { + if (IsTraining() && dropout_rate_ > 0) { + // Apply same mask from forward pass + // Multiply deltas by the same mask + } else { + *back_deltas = fwd_deltas; + } +#if 0 int size = deltas.Size(); input_deltas->Resize(size); @@ -120,6 +133,7 @@ bool Dropout::Backward(bool debug, const NetworkIO &fwd_deltas, NetworkScratch * (*input_deltas)(i) = deltas(i) * (1.0f - dropout_rate_); } } +#endif #if 0 tesserr << __FUNCTION__ << ": missing implementation\n"; diff --git a/src/lstm/dropout.h b/src/lstm/dropout.h index 6b5d2f6c76..ff84ed4899 100644 --- a/src/lstm/dropout.h +++ b/src/lstm/dropout.h @@ -25,12 +25,12 @@ namespace tesseract { class Dropout : public Network { public: TESS_API - Dropout(const std::string &name, int ni, float dropout_rate, uint8_t dimensions); + Dropout(const std::string &name, int ni, float dropout_rate); ~Dropout() override = default; // Accessors. std::string spec() const override { - return "Do" + std::to_string(dropout_rate_) + "," + std::to_string(dimensions_); + return "Do" + std::to_string(dropout_rate_); } // Writes to the given file. Returns false in case of error. @@ -53,7 +53,6 @@ class Dropout : public Network { std::vector dropout_mask_; float dropout_rate_; - uint8_t dimensions_; }; } // namespace tesseract. diff --git a/src/lstm/network.cpp b/src/lstm/network.cpp index ba41187e66..163ab73829 100644 --- a/src/lstm/network.cpp +++ b/src/lstm/network.cpp @@ -253,7 +253,7 @@ Network *Network::CreateFromFile(TFile *fp) { network = new Convolve(name, ni, 0, 0); break; case NT_DROPOUT: - network = new Dropout(name, ni, 0.5f, 1); + network = new Dropout(name, ni, 0.5f); break; case NT_INPUT: network = new Input(name, ni, no); diff --git a/src/training/common/networkbuilder.cpp b/src/training/common/networkbuilder.cpp index 5a0d91715c..997bfb1cf2 100644 --- a/src/training/common/networkbuilder.cpp +++ b/src/training/common/networkbuilder.cpp @@ -19,6 +19,7 @@ #include "networkbuilder.h" #include "convolve.h" +#include "dropout.h" #include "fullyconnected.h" #include "input.h" #include "lstm.h" @@ -28,6 +29,7 @@ #include "reconfig.h" #include "reversed.h" #include "series.h" +#include "tesserrstream.h" // for tesserr #include "unicharset.h" namespace tesseract { @@ -104,6 +106,8 @@ Network *NetworkBuilder::BuildFromString(const StaticShape &input_shape, const c return ParseS(input_shape, str); case 'C': return ParseC(input_shape, str); + case 'D': + return ParseD(input_shape, str); case 'M': return ParseM(input_shape, str); case 'L': @@ -295,6 +299,22 @@ Network *NetworkBuilder::ParseC(const StaticShape &input_shape, const char **str return series; } +// Parses a network that begins with 'D'. +Network *NetworkBuilder::ParseD(const StaticShape &input_shape, const char **str) { + if ((*str)[1] != 'o') { + tesserr << "Invalid Do spec!:" << *str << '\n'; + return nullptr; + } + char *end; + float dropout_rate = strtof(*str + 2, &end); + if (dropout_rate < 0 || dropout_rate > 1) { + tesserr << "Invalid dropout rate! Must be between 0.0 and 1.0: " << dropout_rate << '\n'; + return nullptr; + } + *str = end; + return new Dropout("Dropout", input_shape.depth(), dropout_rate); +} + // Parses a network that begins with 'M'. Network *NetworkBuilder::ParseM(const StaticShape &input_shape, const char **str) { int y = 0, x = 0; diff --git a/src/training/common/networkbuilder.h b/src/training/common/networkbuilder.h index 1ee1900b1c..7f1b2b3cf8 100644 --- a/src/training/common/networkbuilder.h +++ b/src/training/common/networkbuilder.h @@ -83,6 +83,7 @@ class TESS_COMMON_TRAINING_API NetworkBuilder { // C(s|t|r|l|m),, Convolves using a (x,y) window, with no shrinkage, // random infill, producing d outputs, then applies a non-linearity: // s: Sigmoid, t: Tanh, r: Relu, l: Linear, m: Softmax. + // Do Dropout with given rate. // F(s|t|r|l|m) Truly fully-connected with s|t|r|l|m non-linearity and d // outputs. Connects to every x,y,depth position of the input, reducing // height, width to 1, producing a single vector as the output. @@ -136,6 +137,8 @@ class TESS_COMMON_TRAINING_API NetworkBuilder { Network *ParseS(const StaticShape &input_shape, const char **str); // Parses a network that begins with 'C'. Network *ParseC(const StaticShape &input_shape, const char **str); + // Parses a network that begins with 'D'. + Network *ParseD(const StaticShape &input_shape, const char **str); // Parses a network that begins with 'M'. Network *ParseM(const StaticShape &input_shape, const char **str); // Parses an LSTM network, either individual, bi- or quad-directional. From fec422ffc3815d16b3212fa75d370543226b8be5 Mon Sep 17 00:00:00 2001 From: Stefan Weil Date: Fri, 10 Oct 2025 07:28:04 +0200 Subject: [PATCH 09/14] Fix range check for dropout rate Signed-off-by: Stefan Weil --- src/training/common/networkbuilder.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/training/common/networkbuilder.cpp b/src/training/common/networkbuilder.cpp index 997bfb1cf2..edf2970b2b 100644 --- a/src/training/common/networkbuilder.cpp +++ b/src/training/common/networkbuilder.cpp @@ -307,8 +307,8 @@ Network *NetworkBuilder::ParseD(const StaticShape &input_shape, const char **str } char *end; float dropout_rate = strtof(*str + 2, &end); - if (dropout_rate < 0 || dropout_rate > 1) { - tesserr << "Invalid dropout rate! Must be between 0.0 and 1.0: " << dropout_rate << '\n'; + if (dropout_rate < 0 || dropout_rate >= 1) { + tesserr << "Invalid dropout rate! Must be in [0.0, 1.0): " << dropout_rate << '\n'; return nullptr; } *str = end; From cac70b2b3f21be1337845fc712d9cd74ed71ec0a Mon Sep 17 00:00:00 2001 From: Stefan Weil Date: Sun, 12 Oct 2025 19:51:21 +0200 Subject: [PATCH 10/14] Update dummy dropout code Signed-off-by: Stefan Weil --- src/lstm/dropout.cpp | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/src/lstm/dropout.cpp b/src/lstm/dropout.cpp index ea85f66e9e..83d528f509 100644 --- a/src/lstm/dropout.cpp +++ b/src/lstm/dropout.cpp @@ -18,8 +18,6 @@ # include "config_auto.h" #endif -#include - #include "dropout.h" #include "networkscratch.h" #include "serialis.h" @@ -28,7 +26,8 @@ namespace tesseract { Dropout::Dropout(const std::string &name, int ni, float dropout_rate) - : Network(NT_DROPOUT, name, ni, 0), + : Network(NT_DROPOUT, name, ni, ni), + dropout_mask_(), dropout_rate_(dropout_rate) { if (dropout_rate_ < 0 || dropout_rate_ >= 1) { @@ -37,11 +36,14 @@ Dropout::Dropout(const std::string &name, int ni, float dropout_rate) } void Dropout::DebugWeights() { + // Dropout doesn't typically have weights to display like other layers. tesserr << "Must override Network::DebugWeights for type " << type_ << '\n'; + tesserr << "Dropout layer '" << name_ << "': rate=" << dropout_rate_ << '\n'; } // Writes to the given file. Returns false in case of error. bool Dropout::Serialize(TFile *fp) const { + // Note: dropout_mask_ is runtime data, not serialized. return Network::Serialize(fp) && fp->Serialize(&dropout_rate_); } @@ -58,9 +60,38 @@ bool Dropout::DeSerialize(TFile *fp) { // See NetworkCpp for a detailed discussion of the arguments. void Dropout::Forward(bool debug, const NetworkIO &input, const TransposedArray *input_transpose, NetworkScratch *scratch, NetworkIO *output) { + // Resize output to match input dimensions. + // Start by copying input structure and potentially data. + *output = input; + if (IsTraining() && dropout_rate_ > 0) { +#if 0 // Apply dropout: randomly zero out neurons - // Generate random mask, apply to input + float keep_prob = 1.0f - static_cast(dropout_rate_); + int32_t batch_size = input.BatchSize(); + int32_t total_elements_per_sample = input.Width() * input.Height() * input.Depth(); + int32_t total_elements = batch_size * total_elements_per_sample; + int num_elements = height * width; + + // Resize mask storage + dropout_mask_.resize(num_elements, 0); + + const float* input_data = input.f(0); + float* output_data = output->f(0); + + // Generate mask and apply dropout + for (int i = 0; i < num_elements; ++i) { + float random_val = static_cast(random_.UnsignedRand(1000000)) / 1000000.0f; + + if (random_val < keep_prob) { + dropout_mask_[i] = 1; // Keep neuron + output_data[i] = input_data[i] / keep_prob; + } else { + dropout_mask_[i] = 0; // Drop neuron + output_data[i] = 0.0f; + } + } +#endif } else { // Inference mode: scale by (1 - dropout_rate) // or just pass through unchanged From bf0a5967395fb91015666bde49ab9768cceab65c Mon Sep 17 00:00:00 2001 From: Stefan Weil Date: Mon, 25 May 2026 23:10:48 +0200 Subject: [PATCH 11/14] lstm: Implement Dropout layer Forward and Backward passes Uses inverted dropout: during training, retained activations are scaled by 1/(1-dropout_rate) so that no rescaling is needed at inference time. The dropout mask is stored during Forward and reused in Backward. The base-class randomizer_ (TRand*) is used for random number generation. Signed-off-by: Stefan Weil --- src/lstm/dropout.cpp | 199 +++++++++++++------------------------------ 1 file changed, 60 insertions(+), 139 deletions(-) diff --git a/src/lstm/dropout.cpp b/src/lstm/dropout.cpp index 83d528f509..c7cf34d09d 100644 --- a/src/lstm/dropout.cpp +++ b/src/lstm/dropout.cpp @@ -21,29 +21,24 @@ #include "dropout.h" #include "networkscratch.h" #include "serialis.h" -#include "tesserrstream.h" // for tesserr +#include "tesserrstream.h" // for tesserr namespace tesseract { Dropout::Dropout(const std::string &name, int ni, float dropout_rate) - : Network(NT_DROPOUT, name, ni, ni), - dropout_mask_(), - dropout_rate_(dropout_rate) -{ + : Network(NT_DROPOUT, name, ni, ni), dropout_mask_(), dropout_rate_(dropout_rate) { if (dropout_rate_ < 0 || dropout_rate_ >= 1) { throw std::invalid_argument("Invalid dropout rate. Must be in [0, 1)."); } } void Dropout::DebugWeights() { - // Dropout doesn't typically have weights to display like other layers. - tesserr << "Must override Network::DebugWeights for type " << type_ << '\n'; tesserr << "Dropout layer '" << name_ << "': rate=" << dropout_rate_ << '\n'; } // Writes to the given file. Returns false in case of error. bool Dropout::Serialize(TFile *fp) const { - // Note: dropout_mask_ is runtime data, not serialized. + // dropout_mask_ is runtime data and is not serialized. return Network::Serialize(fp) && fp->Serialize(&dropout_rate_); } @@ -57,160 +52,86 @@ bool Dropout::DeSerialize(TFile *fp) { } // Runs forward propagation of activations on the input line. -// See NetworkCpp for a detailed discussion of the arguments. -void Dropout::Forward(bool debug, const NetworkIO &input, const TransposedArray *input_transpose, - NetworkScratch *scratch, NetworkIO *output) { - // Resize output to match input dimensions. - // Start by copying input structure and potentially data. - *output = input; - - if (IsTraining() && dropout_rate_ > 0) { -#if 0 - // Apply dropout: randomly zero out neurons - float keep_prob = 1.0f - static_cast(dropout_rate_); - int32_t batch_size = input.BatchSize(); - int32_t total_elements_per_sample = input.Width() * input.Height() * input.Depth(); - int32_t total_elements = batch_size * total_elements_per_sample; - int num_elements = height * width; - - // Resize mask storage - dropout_mask_.resize(num_elements, 0); - - const float* input_data = input.f(0); - float* output_data = output->f(0); - - // Generate mask and apply dropout - for (int i = 0; i < num_elements; ++i) { - float random_val = static_cast(random_.UnsignedRand(1000000)) / 1000000.0f; - - if (random_val < keep_prob) { - dropout_mask_[i] = 1; // Keep neuron - output_data[i] = input_data[i] / keep_prob; - } else { - dropout_mask_[i] = 0; // Drop neuron - output_data[i] = 0.0f; - } - } -#endif - } else { - // Inference mode: scale by (1 - dropout_rate) - // or just pass through unchanged - *output = input; - } -#if 0 - // Resize output to match input dimensions - output->Resize(input.NumFeatures(), input.TimeSize(), input.BatchSize()); - - if (IsTraining()) { - // Initialize the dropout mask - dropout_mask_.Resize(input.NumFeatures(), input.TimeSize(), input.BatchSize()); - - float retain_prob = 1.0f - dropout_rate_; - - // Random number generator setup - std::mt19937 generator; // You may need to use Tesseract's RNG - std::bernoulli_distribution distribution(retain_prob); - - // Apply dropout mask - for (int b = 0; b < input.BatchSize(); ++b) { - for (int t = 0; t < input.TimeSize(); ++t) { - const float* input_features = input.f(b, t); - float* output_features = output->f(b, t); - float* mask_features = dropout_mask_.f(b, t); - for (int i = 0; i < input.NumFeatures(); ++i) { - bool retain = distribution(generator); - mask_features[i] = retain ? 1.0f : 0.0f; - // Scale the activations to maintain expected value - output_features[i] = input_features[i] * mask_features[i] / retain_prob; +// See Network for a detailed discussion of the arguments. +void Dropout::Forward(bool debug, const NetworkIO &input, + const TransposedArray *input_transpose, NetworkScratch *scratch, + NetworkIO *output) { + // Output has the same shape as input (ni_ == no_). + output->Resize(input, no_); + + int width = input.Width(); + int num_features = input.NumFeatures(); + + if (IsTraining() && dropout_rate_ > 0.0f) { + // Inverted dropout: scale retained activations by 1/keep_prob so that + // inference can pass the network output through unchanged. + float keep_prob = 1.0f - dropout_rate_; + float scale = 1.0f / keep_prob; + + int num_elements = width * num_features; + dropout_mask_.resize(num_elements); + + for (int t = 0; t < width; ++t) { + const float *in = input.f(t); + float *out = output->f(t); + char *mask = dropout_mask_.data() + t * num_features; + for (int i = 0; i < num_features; ++i) { + // UnsignedRand(1.0) returns a value in [0, 1]. + float r = static_cast(randomizer_->UnsignedRand(1.0)); + if (r < keep_prob) { + mask[i] = 1; + out[i] = in[i] * scale; + } else { + mask[i] = 0; + out[i] = 0.0f; } } } } else { - // During inference, pass input to output unchanged + // Inference mode (or dropout_rate_ == 0): pass input through unchanged. output->CopyAll(input); } - if (debug) { - tprintf("Dropout Forward Pass Complete.\n"); - } #ifndef GRAPHICS_DISABLED if (debug) { DisplayForward(*output); } #endif -#endif } // Runs backward propagation of errors on the deltas line. -// See NetworkCpp for a detailed discussion of the arguments. +// See Network for a detailed discussion of the arguments. bool Dropout::Backward(bool debug, const NetworkIO &fwd_deltas, NetworkScratch *scratch, - NetworkIO *back_deltas) { - if (IsTraining() && dropout_rate_ > 0) { - // Apply same mask from forward pass - // Multiply deltas by the same mask - } else { - *back_deltas = fwd_deltas; - } -#if 0 - int size = deltas.Size(); - input_deltas->Resize(size); - - if (IsTraining()) { - for (int i = 0; i < size; ++i) { - (*input_deltas)(i) = deltas(i) * dropout_mask_[i]; - } - } else { - for (int i = 0; i < size; ++i) { - (*input_deltas)(i) = deltas(i) * (1.0f - dropout_rate_); - } - } -#endif + NetworkIO *back_deltas) { + back_deltas->Resize(fwd_deltas, ni_); -#if 0 - tesserr << __FUNCTION__ << ": missing implementation\n"; + int width = fwd_deltas.Width(); + int num_features = fwd_deltas.NumFeatures(); - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_real_distribution dist(0.0f, 1.0f); + if (IsTraining() && dropout_rate_ > 0.0f) { + // Apply the same inverted-dropout mask that was used in Forward. + float keep_prob = 1.0f - dropout_rate_; + float scale = 1.0f / keep_prob; - back_deltas->Resize(fwd_deltas, ni_); - for (unsigned i = 0; i < ni_; i++) { - if (dist(gen) >= dropout_rate_) { - // Keep the neuron - output->push_back(input[i]); - } else { - // Drop the neuron - output->push_back(0.0f); + for (int t = 0; t < width; ++t) { + const float *in = fwd_deltas.f(t); + float *out = back_deltas->f(t); + const char *mask = dropout_mask_.data() + t * num_features; + for (int i = 0; i < num_features; ++i) { + out[i] = mask[i] ? in[i] * scale : 0.0f; + } } + } else { + // Inference mode: pass gradients through unchanged. + back_deltas->CopyAll(fwd_deltas); } -#endif -#if 0 - NetworkScratch::IO delta_sum; - delta_sum.ResizeFloat(fwd_deltas, ni_, scratch); - delta_sum->Zero(); - int y_scale = 2 * half_y_ + 1; - StrideMap::Index src_index(fwd_deltas.stride_map()); - do { - // Stack x_scale groups of y_scale * ni_ inputs together. - int t = src_index.t(); - int out_ix = 0; - for (int x = -half_x_; x <= half_x_; ++x, out_ix += y_scale * ni_) { - StrideMap::Index x_index(src_index); - if (x_index.AddOffset(x, FD_WIDTH)) { - int out_iy = out_ix; - for (int y = -half_y_; y <= half_y_; ++y, out_iy += ni_) { - StrideMap::Index y_index(x_index); - if (y_index.AddOffset(y, FD_HEIGHT)) { - fwd_deltas.AddTimeStepPart(t, out_iy, ni_, delta_sum->f(y_index.t())); - } - } - } - } - } while (src_index.Increment()); - back_deltas->CopyAll(*delta_sum); +#ifndef GRAPHICS_DISABLED + if (debug) { + DisplayBackward(*back_deltas); + } #endif - return true; + return needs_to_backprop_; } } // namespace tesseract. From 0fc02f30c7f3ead445e331795e3e49c7aed78646 Mon Sep 17 00:00:00 2001 From: Stefan Weil Date: Tue, 26 May 2026 16:31:40 +0200 Subject: [PATCH 12/14] lstm: Add dimension parameter for Dropout layer Signed-off-by: Stefan Weil --- src/lstm/dropout.cpp | 112 ++++++++++++++++++------- src/lstm/dropout.h | 8 +- src/training/common/networkbuilder.cpp | 10 ++- 3 files changed, 97 insertions(+), 33 deletions(-) diff --git a/src/lstm/dropout.cpp b/src/lstm/dropout.cpp index c7cf34d09d..4063c234f9 100644 --- a/src/lstm/dropout.cpp +++ b/src/lstm/dropout.cpp @@ -25,11 +25,17 @@ namespace tesseract { -Dropout::Dropout(const std::string &name, int ni, float dropout_rate) - : Network(NT_DROPOUT, name, ni, ni), dropout_mask_(), dropout_rate_(dropout_rate) { +Dropout::Dropout(const std::string &name, int ni, float dropout_rate, int dropout_dim) + : Network(NT_DROPOUT, name, ni, ni), + dropout_mask_(), + dropout_rate_(dropout_rate), + dropout_dim_(dropout_dim) { if (dropout_rate_ < 0 || dropout_rate_ >= 1) { throw std::invalid_argument("Invalid dropout rate. Must be in [0, 1)."); } + if (dropout_dim_ < 0 || dropout_dim_ > 2) { + throw std::invalid_argument("Invalid dropout dim. Must be 0, 1 or 2."); + } } void Dropout::DebugWeights() { @@ -39,7 +45,9 @@ void Dropout::DebugWeights() { // Writes to the given file. Returns false in case of error. bool Dropout::Serialize(TFile *fp) const { // dropout_mask_ is runtime data and is not serialized. - return Network::Serialize(fp) && fp->Serialize(&dropout_rate_); + return Network::Serialize(fp) && + fp->Serialize(&dropout_rate_) && + fp->Serialize(&dropout_dim_); } // Reads from the given file. Returns false in case of error. @@ -47,6 +55,9 @@ bool Dropout::DeSerialize(TFile *fp) { if (!fp->DeSerialize(&dropout_rate_)) { return false; } + if (!fp->DeSerialize(&dropout_dim_)) { + return false; + } no_ = ni_; return true; } @@ -54,8 +65,8 @@ bool Dropout::DeSerialize(TFile *fp) { // Runs forward propagation of activations on the input line. // See Network for a detailed discussion of the arguments. void Dropout::Forward(bool debug, const NetworkIO &input, - const TransposedArray *input_transpose, NetworkScratch *scratch, - NetworkIO *output) { + const TransposedArray *input_transpose, + NetworkScratch *scratch, NetworkIO *output) { // Output has the same shape as input (ni_ == no_). output->Resize(input, no_); @@ -68,22 +79,47 @@ void Dropout::Forward(bool debug, const NetworkIO &input, float keep_prob = 1.0f - dropout_rate_; float scale = 1.0f / keep_prob; - int num_elements = width * num_features; - dropout_mask_.resize(num_elements); - - for (int t = 0; t < width; ++t) { - const float *in = input.f(t); - float *out = output->f(t); - char *mask = dropout_mask_.data() + t * num_features; + if (dropout_dim_ == 2) { + // Feature dropout: one mask value per feature, shared across all timesteps. + dropout_mask_.resize(num_features); for (int i = 0; i < num_features; ++i) { - // UnsignedRand(1.0) returns a value in [0, 1]. float r = static_cast(randomizer_->UnsignedRand(1.0)); - if (r < keep_prob) { - mask[i] = 1; - out[i] = in[i] * scale; + dropout_mask_[i] = (r < keep_prob) ? 1 : 0; + } + for (int t = 0; t < width; ++t) { + const float *in = input.f(t); + float *out = output->f(t); + for (int i = 0; i < num_features; ++i) + out[i] = dropout_mask_[i] ? in[i] * scale : 0.0f; + } + } else if (dropout_dim_ == 1) { + // Temporal dropout: one mask value per timestep, shared across all features. + dropout_mask_.resize(width); + for (int t = 0; t < width; ++t) { + float r = static_cast(randomizer_->UnsignedRand(1.0)); + dropout_mask_[t] = (r < keep_prob) ? 1 : 0; + } + for (int t = 0; t < width; ++t) { + const float *in = input.f(t); + float *out = output->f(t); + if (dropout_mask_[t]) { + for (int i = 0; i < num_features; ++i) out[i] = in[i] * scale; } else { - mask[i] = 0; - out[i] = 0.0f; + memset(out, 0, sizeof(float) * num_features); + } + } + } else { + // Element-wise dropout (dim=0, default). + dropout_mask_.resize(width * num_features); + for (int t = 0; t < width; ++t) { + const float *in = input.f(t); + float *out = output->f(t); + char *mask = dropout_mask_.data() + t * num_features; + for (int i = 0; i < num_features; ++i) { + // UnsignedRand(1.0) returns a value in [0, 1]. + float r = static_cast(randomizer_->UnsignedRand(1.0)); + mask[i] = (r < keep_prob) ? 1 : 0; + out[i] = mask[i] ? in[i] * scale : 0.0f; } } } @@ -101,24 +137,40 @@ void Dropout::Forward(bool debug, const NetworkIO &input, // Runs backward propagation of errors on the deltas line. // See Network for a detailed discussion of the arguments. -bool Dropout::Backward(bool debug, const NetworkIO &fwd_deltas, NetworkScratch *scratch, - NetworkIO *back_deltas) { +bool Dropout::Backward(bool debug, const NetworkIO &fwd_deltas, + NetworkScratch *scratch, NetworkIO *back_deltas) { back_deltas->Resize(fwd_deltas, ni_); int width = fwd_deltas.Width(); int num_features = fwd_deltas.NumFeatures(); if (IsTraining() && dropout_rate_ > 0.0f) { - // Apply the same inverted-dropout mask that was used in Forward. - float keep_prob = 1.0f - dropout_rate_; - float scale = 1.0f / keep_prob; - - for (int t = 0; t < width; ++t) { - const float *in = fwd_deltas.f(t); - float *out = back_deltas->f(t); - const char *mask = dropout_mask_.data() + t * num_features; - for (int i = 0; i < num_features; ++i) { - out[i] = mask[i] ? in[i] * scale : 0.0f; + float scale = 1.0f / (1.0f - dropout_rate_); + + if (dropout_dim_ == 2) { + for (int t = 0; t < width; ++t) { + const float *in = fwd_deltas.f(t); + float *out = back_deltas->f(t); + for (int i = 0; i < num_features; ++i) + out[i] = dropout_mask_[i] ? in[i] * scale : 0.0f; + } + } else if (dropout_dim_ == 1) { + for (int t = 0; t < width; ++t) { + const float *in = fwd_deltas.f(t); + float *out = back_deltas->f(t); + if (dropout_mask_[t]) { + for (int i = 0; i < num_features; ++i) out[i] = in[i] * scale; + } else { + memset(out, 0, sizeof(float) * num_features); + } + } + } else { + for (int t = 0; t < width; ++t) { + const float *in = fwd_deltas.f(t); + float *out = back_deltas->f(t); + const char *mask = dropout_mask_.data() + t * num_features; + for (int i = 0; i < num_features; ++i) + out[i] = mask[i] ? in[i] * scale : 0.0f; } } } else { diff --git a/src/lstm/dropout.h b/src/lstm/dropout.h index ff84ed4899..b0f3eaba7c 100644 --- a/src/lstm/dropout.h +++ b/src/lstm/dropout.h @@ -25,12 +25,15 @@ namespace tesseract { class Dropout : public Network { public: TESS_API - Dropout(const std::string &name, int ni, float dropout_rate); + Dropout(const std::string &name, int ni, float dropout_rate, int dropout_dim = 0); ~Dropout() override = default; // Accessors. std::string spec() const override { - return "Do" + std::to_string(dropout_rate_); + std::string s = "Do" + std::to_string(dropout_rate_); + if (dropout_dim_ != 0) + s += "," + std::to_string(dropout_dim_); + return s; } // Writes to the given file. Returns false in case of error. @@ -53,6 +56,7 @@ class Dropout : public Network { std::vector dropout_mask_; float dropout_rate_; + int dropout_dim_; // 0=elementwise, 1=temporal, 2=feature/channel }; } // namespace tesseract. diff --git a/src/training/common/networkbuilder.cpp b/src/training/common/networkbuilder.cpp index edf2970b2b..8ff017cc96 100644 --- a/src/training/common/networkbuilder.cpp +++ b/src/training/common/networkbuilder.cpp @@ -311,8 +311,16 @@ Network *NetworkBuilder::ParseD(const StaticShape &input_shape, const char **str tesserr << "Invalid dropout rate! Must be in [0.0, 1.0): " << dropout_rate << '\n'; return nullptr; } + int dropout_dim = 0; + if (*end == ',') { + dropout_dim = static_cast(strtol(end + 1, &end, 10)); + if (dropout_dim < 1 || dropout_dim > 2) { + tesserr << "Invalid dropout dim! Must be 1 or 2.\n"; + return nullptr; + } + } *str = end; - return new Dropout("Dropout", input_shape.depth(), dropout_rate); + return new Dropout("Dropout", input_shape.depth(), dropout_rate, dropout_dim); } // Parses a network that begins with 'M'. From 90bff0394b2af0f254516a0a400682bf3b488703 Mon Sep 17 00:00:00 2001 From: Stefan Weil Date: Mon, 1 Jun 2026 07:17:57 +0200 Subject: [PATCH 13/14] Add float variant of helper function UnsignedRand Signed-off-by: Stefan Weil --- src/ccutil/helpers.h | 4 ++++ unittest/recodebeam_test.cc | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ccutil/helpers.h b/src/ccutil/helpers.h index fa2f38ec83..665ea2f8eb 100644 --- a/src/ccutil/helpers.h +++ b/src/ccutil/helpers.h @@ -86,6 +86,10 @@ class TRand { return range * 2.0 * IntRand() / INT32_MAX - range; } // Returns a floating point value in the range [0, range]. + float UnsignedRand(float range) { + return range * IntRand() / INT32_MAX; + } + // Returns a double value in the range [0, range]. double UnsignedRand(double range) { return range * IntRand() / INT32_MAX; } diff --git a/unittest/recodebeam_test.cc b/unittest/recodebeam_test.cc index 0fc738576e..867376a53a 100644 --- a/unittest/recodebeam_test.cc +++ b/unittest/recodebeam_test.cc @@ -207,7 +207,7 @@ class RecodeBeamTest : public ::testing::Test { TRand random; for (int t = 0; t < width; ++t) { for (int i = 0; i < num_codes; ++i) { - outputs(t, i) = random.UnsignedRand(0.25); + outputs(t, i) = random.UnsignedRand(0.25f); } } int t = 0; From 1aaae5157d5c5bb8d3f96b47fd511e84766c15d0 Mon Sep 17 00:00:00 2001 From: Stefan Weil Date: Mon, 1 Jun 2026 07:21:45 +0200 Subject: [PATCH 14/14] Use unsigned dimension for dropout Signed-off-by: Stefan Weil --- src/lstm/dropout.cpp | 16 ++++++++-------- src/lstm/dropout.h | 4 ++-- src/training/common/networkbuilder.cpp | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/lstm/dropout.cpp b/src/lstm/dropout.cpp index 4063c234f9..46c998c2bc 100644 --- a/src/lstm/dropout.cpp +++ b/src/lstm/dropout.cpp @@ -25,7 +25,7 @@ namespace tesseract { -Dropout::Dropout(const std::string &name, int ni, float dropout_rate, int dropout_dim) +Dropout::Dropout(const std::string &name, int ni, float dropout_rate, unsigned dropout_dim) : Network(NT_DROPOUT, name, ni, ni), dropout_mask_(), dropout_rate_(dropout_rate), @@ -33,13 +33,13 @@ Dropout::Dropout(const std::string &name, int ni, float dropout_rate, int dropou if (dropout_rate_ < 0 || dropout_rate_ >= 1) { throw std::invalid_argument("Invalid dropout rate. Must be in [0, 1)."); } - if (dropout_dim_ < 0 || dropout_dim_ > 2) { + if (dropout_dim_ > 2) { throw std::invalid_argument("Invalid dropout dim. Must be 0, 1 or 2."); } } void Dropout::DebugWeights() { - tesserr << "Dropout layer '" << name_ << "': rate=" << dropout_rate_ << '\n'; + tesserr << "Dropout layer '" << name_ << "': rate=" << dropout_rate_ << ", dimension=" << dropout_dim_ << '\n'; } // Writes to the given file. Returns false in case of error. @@ -83,7 +83,7 @@ void Dropout::Forward(bool debug, const NetworkIO &input, // Feature dropout: one mask value per feature, shared across all timesteps. dropout_mask_.resize(num_features); for (int i = 0; i < num_features; ++i) { - float r = static_cast(randomizer_->UnsignedRand(1.0)); + float r = randomizer_->UnsignedRand(1.0f); dropout_mask_[i] = (r < keep_prob) ? 1 : 0; } for (int t = 0; t < width; ++t) { @@ -96,7 +96,7 @@ void Dropout::Forward(bool debug, const NetworkIO &input, // Temporal dropout: one mask value per timestep, shared across all features. dropout_mask_.resize(width); for (int t = 0; t < width; ++t) { - float r = static_cast(randomizer_->UnsignedRand(1.0)); + float r = randomizer_->UnsignedRand(1.0f); dropout_mask_[t] = (r < keep_prob) ? 1 : 0; } for (int t = 0; t < width; ++t) { @@ -110,14 +110,14 @@ void Dropout::Forward(bool debug, const NetworkIO &input, } } else { // Element-wise dropout (dim=0, default). - dropout_mask_.resize(width * num_features); + dropout_mask_.resize(static_cast(width) * num_features); for (int t = 0; t < width; ++t) { const float *in = input.f(t); float *out = output->f(t); char *mask = dropout_mask_.data() + t * num_features; for (int i = 0; i < num_features; ++i) { - // UnsignedRand(1.0) returns a value in [0, 1]. - float r = static_cast(randomizer_->UnsignedRand(1.0)); + // UnsignedRand(1.0f) returns a value in [0, 1]. + float r = randomizer_->UnsignedRand(1.0f); mask[i] = (r < keep_prob) ? 1 : 0; out[i] = mask[i] ? in[i] * scale : 0.0f; } diff --git a/src/lstm/dropout.h b/src/lstm/dropout.h index b0f3eaba7c..db6ebc3c22 100644 --- a/src/lstm/dropout.h +++ b/src/lstm/dropout.h @@ -25,7 +25,7 @@ namespace tesseract { class Dropout : public Network { public: TESS_API - Dropout(const std::string &name, int ni, float dropout_rate, int dropout_dim = 0); + Dropout(const std::string &name, int ni, float dropout_rate, unsigned dropout_dim = 0); ~Dropout() override = default; // Accessors. @@ -56,7 +56,7 @@ class Dropout : public Network { std::vector dropout_mask_; float dropout_rate_; - int dropout_dim_; // 0=elementwise, 1=temporal, 2=feature/channel + unsigned dropout_dim_; // 0=elementwise, 1=temporal, 2=feature/channel }; } // namespace tesseract. diff --git a/src/training/common/networkbuilder.cpp b/src/training/common/networkbuilder.cpp index 8ff017cc96..18a44420e5 100644 --- a/src/training/common/networkbuilder.cpp +++ b/src/training/common/networkbuilder.cpp @@ -311,9 +311,9 @@ Network *NetworkBuilder::ParseD(const StaticShape &input_shape, const char **str tesserr << "Invalid dropout rate! Must be in [0.0, 1.0): " << dropout_rate << '\n'; return nullptr; } - int dropout_dim = 0; + unsigned dropout_dim = 0; if (*end == ',') { - dropout_dim = static_cast(strtol(end + 1, &end, 10)); + dropout_dim = static_cast(strtoul(end + 1, &end, 10)); if (dropout_dim < 1 || dropout_dim > 2) { tesserr << "Invalid dropout dim! Must be 1 or 2.\n"; return nullptr;