Feature/seas6 sprint

src/multio/action/encode-mtg2/EncodeMtg2.cc

@@ -11,6 +11,7 @@
 #include "EncodeMtg2.h"
 
 #include <iostream>
+#include <unordered_set>
 
 #include "eckit/config/LocalConfiguration.h"
 #include "eckit/exception/Exceptions.h"
@@ -34,11 +35,61 @@ namespace dm = multio::datamod;
 using message::Message;
 using message::Peer;
 
+namespace {
+
+const std::unordered_set<std::string>& supportedMars2gribOptions() {
+    static const auto supported = std::unordered_set<std::string>{"applyChecks",
+                                                                  "enableOverride",
+                                                                  "enableBitsPerValueCompression",
+                                                                  "normalizeMars",
+                                                                  "normalizeMisc",
+                                                                  "fixMarsGrid",
+                                                                  "skipSection3",
+                                                                  "allowDefaultTimeIncrement",
+                                                                  "allowZeroLengthFsWindow",
+                                                                  "allowNonEnumeratedPositiveIntegerTimespanHours",
+                                                                  "allowRedundantTimeIncrement",
+                                                                  "allowMissingTimespanForInstantProduct",
+                                                                  "allowMissingTimespanForStatisticalProduct"};
+    return supported;
+}
+
+std::unique_ptr<metkit::mars2grib::Mars2Grib> makeEncoder(const ComponentConfiguration& compConf) {
+    const auto& actionConf = compConf.parsedConfig();
+
+    if (!actionConf.has("mars2grib-options")) {
+        return std::make_unique<metkit::mars2grib::Mars2Grib>();
+    }
+
+    if (!actionConf.isSubConfiguration("mars2grib-options")) {
+        throw EncodeMtg2Exception("encode-mtg2 option 'mars2grib-options' must be a configuration section", Here());
+    }
+
+    const auto rawEncoderConf = actionConf.getSubConfiguration("mars2grib-options");
+    eckit::LocalConfiguration validatedEncoderConf{};
+
+    for (const auto& key : rawEncoderConf.keys()) {
+        if (supportedMars2gribOptions().find(key) == supportedMars2gribOptions().end()) {
+            throw EncodeMtg2Exception("Unsupported mars2grib option 'mars2grib-options." + key + "'", Here());
+        }
+
+        if (!rawEncoderConf.isBoolean(key)) {
+            throw EncodeMtg2Exception("mars2grib option 'mars2grib-options." + key + "' must be boolean", Here());
+        }
+
+        validatedEncoderConf.set(key, rawEncoderConf.getBool(key));
+    }
+
+    return std::make_unique<metkit::mars2grib::Mars2Grib>(validatedEncoderConf);
+}
+
+}  // namespace
+
 
 EncodeMtg2::EncodeMtg2(const ComponentConfiguration& compConf) :
     ChainedAction{compConf},
     opts_{cf::parseActionConfig<EncodeMtg2Options>(compConf)},
-    encoder_{},
+    encoder_{makeEncoder(compConf)},
     cache_{opts_.cached ? std::optional<Cache>{Cache{}} : std::optional<Cache>{}} {}
 
 
@@ -49,6 +100,7 @@ std::unique_ptr<metkit::codes::CodesHandle> encode(metkit::mars2grib::Mars2Grib&
     const auto mars = dm::dumpRecord<eckit::LocalConfiguration>(marsRec);
     const auto misc = dm::dumpUnscopedRecord<eckit::LocalConfiguration>(miscRec);
 
+
     if (!cache) {
         return encoder.encode(values, size, mars, misc);
     }
@@ -116,7 +168,7 @@ void EncodeMtg2::executeImpl(Message msg) {
         }
 
         // Call the GRIB2 encoder in metkit
-        const auto sample = encode(encoder_, cache_, values, size, marsRec, miscRec);
+        const auto sample = encode(*encoder_, cache_, values, size, marsRec, miscRec);
 
         eckit::Buffer buf{sample->messageSize()};
         sample->copyInto(reinterpret_cast<uint8_t*>(buf.data()), buf.size());

src/multio/action/encode-mtg2/EncodeMtg2.h

@@ -10,6 +10,7 @@
 
 #pragma once
 
+#include <memory>
 #include <optional>
 
 #include "metkit/mars2grib/api/Mars2Grib.h"
@@ -44,15 +45,11 @@ class EncodeMtg2 : public ChainedAction {
     void executeImpl(message::Message msg) override;
 
 private:
-    // Internal constructor delegate with prepared configuration for specific
-    // encoder
-    explicit EncodeMtg2(const ComponentConfiguration& compConf, const eckit::LocalConfiguration& encoderConf);
-
     void print(std::ostream& os) const override;
 
     // TODO pgeier this option will be renamed and the action should get it own struct with parsing capabilities again
     EncodeMtg2Options opts_;
-    metkit::mars2grib::Mars2Grib encoder_;
+    std::unique_ptr<metkit::mars2grib::Mars2Grib> encoder_;
 
     std::optional<Cache> cache_;
 };

src/multio/action/encode-mtg2/fakeDoubleLoop.cc

@@ -19,6 +19,7 @@
 #include "eckit/config/LocalConfiguration.h"
 #include "eckit/config/YAMLConfiguration.h"
 #include "eckit/filesystem/PathName.h"
+#include "eckit/types/DateTime.h"
 
 namespace multio::action::encode_mtg2::fake_double_loop {
 
@@ -34,7 +35,9 @@ enum class TypeOfStatisticalProcessing : std::int64_t
     Minimum = 3,
     Difference = 4,
     StandardDeviation = 6,
-    InverseDifference = 8
+    InverseDifference = 8,
+    Severity = 100,
+    Mode = 101
 };
 
 
@@ -54,6 +57,10 @@ TypeOfStatisticalProcessing typeOfStatisticalProcessingFromInt(std::int64_t valu
             return TypeOfStatisticalProcessing::StandardDeviation;
         case 8:
             return TypeOfStatisticalProcessing::InverseDifference;
+        case 100:
+            return TypeOfStatisticalProcessing::Severity;
+        case 101:
+            return TypeOfStatisticalProcessing::Mode;
         default: {
             std::ostringstream os;
             os << "Unknown typeOfStatisticalProcessing value: " << value;
@@ -74,6 +81,8 @@ bool isValidStattypeOperation(TypeOfStatisticalProcessing operation) {
         case TypeOfStatisticalProcessing::Accumulation:
         case TypeOfStatisticalProcessing::Difference:
         case TypeOfStatisticalProcessing::InverseDifference:
+        case TypeOfStatisticalProcessing::Severity:
+        case TypeOfStatisticalProcessing::Mode:
             return false;
     }
 
@@ -95,6 +104,8 @@ std::string stattypeOperationCode(TypeOfStatisticalProcessing operation) {
         case TypeOfStatisticalProcessing::Accumulation:
         case TypeOfStatisticalProcessing::Difference:
         case TypeOfStatisticalProcessing::InverseDifference:
+        case TypeOfStatisticalProcessing::Severity:
+        case TypeOfStatisticalProcessing::Mode:
             break;
     }
 
@@ -137,7 +148,8 @@ class LocalStatisticsOperationMapping {
         for (const auto& mapping : mappingConf.getSubConfigurations()) {
             const auto param = mapping.getInt64("param");
             const auto typeOfStatisticalProcessing = mapping.getInt64("typeOfStatisticalProcessing");
-
+            //            std::cout << "MIVAL: Mapping param: " << param << " to typeOfStatisticalProcessing: "
+            //                      << typeOfStatisticalProcessing << std::endl;
             operationMappings.emplace(param, typeOfStatisticalProcessingFromInt(typeOfStatisticalProcessing));
         }
 
@@ -181,6 +193,13 @@ bool requiresFakeDoubleLoopRepresentation(const dm::FullMarsRecord& marsRec) {
     return false;
 }
 
+bool isSeasonal(const dm::FullMarsRecord& marsRec) {
+    std::string klass = marsRec.klass.get();
+    std::string stream = marsRec.stream.get();
+
+    return (klass == "od" || klass == "rd" || klass == "c3") && (stream == "sfmd" || stream == "shmd");
+}
+
 std::optional<std::string> operationCodeFromParam(std::int64_t param) {
 
     // This is the full
@@ -228,7 +247,7 @@ std::string reconstructStatType(const dm::FullMarsRecord& marsRec) {
 
     // Create statType by concatenating operation code and period code
     if (operationCode && periodCode) {
-        return *operationCode + *periodCode;
+        return *periodCode + *operationCode;
     }
     else {
         std::ostringstream os;
@@ -238,7 +257,49 @@ std::string reconstructStatType(const dm::FullMarsRecord& marsRec) {
     }
 }
 
-}  // namespace detail
+long computeFcmonth(const dm::FullMarsRecord& marsRec) {
+    if (!marsRec.step.isSet()) {
+        throw eckit::SeriousBug("Cannot compute fcmonth for seasonal record without step", Here());
+    }
+
+    const eckit::Date epochDate{marsRec.date.get()};
+    const long epochTime = marsRec.time.get();
+    const auto epochHour = epochTime / 10000;
+    const auto epochMinute = (epochTime % 10000) / 100;
+    const eckit::DateTime epochDateTime{epochDate, eckit::Time{epochHour, epochMinute, 0}};
+    const eckit::DateTime currentDateTime
+        = epochDateTime + static_cast<eckit::Second>(marsRec.step.get().toHours() * 3600);
+
+    const auto isBeginningOfMonth = [](const eckit::DateTime& dt) {
+        return dt.date().day() == 1 && dt.time().hours() == 0 && dt.time().minutes() == 0 && dt.time().seconds() == 0;
+    };
+
+    if (!isBeginningOfMonth(epochDateTime)) {
+        std::ostringstream os;
+        os << "Cannot compute fcmonth: epochDateTime is not at the beginning of a month: " << epochDateTime;
+        throw eckit::SeriousBug(os.str(), Here());
+    }
+
+    if (!isBeginningOfMonth(currentDateTime)) {
+        std::ostringstream os;
+        os << "Cannot compute fcmonth: currentDateTime is not at the beginning of a month: " << currentDateTime;
+        throw eckit::SeriousBug(os.str(), Here());
+    }
+
+    const long fcmonth = static_cast<long>((currentDateTime.date().year() - epochDateTime.date().year()) * 12
+                                           + (currentDateTime.date().month() - epochDateTime.date().month()));
+
+    if (fcmonth < 0) {
+        std::ostringstream os;
+        os << "Cannot compute fcmonth: currentDateTime precedes epochDateTime: " << currentDateTime << " < "
+           << epochDateTime;
+        throw eckit::SeriousBug(os.str(), Here());
+    }
+
+    return fcmonth;
+}
+
+} // namespace detail
 
 void fakeDoubleLoop(dm::FullMarsRecord& marsRec) {
 
@@ -249,6 +310,11 @@ void fakeDoubleLoop(dm::FullMarsRecord& marsRec) {
             marsRec.timespan.set(dm::TypeParser<dm::TimeSpan>::parse("none"));
         }
     }
+    if (detail::isSeasonal(marsRec)) {
+        const long fcmonth = detail::computeFcmonth(marsRec);
+        marsRec.fcmonth.set(fcmonth);
+        marsRec.step.unset();
+    }
 }
 
-}  // namespace multio::action::encode_mtg2::fake_double_loop
+}  // namespace multio::action::encode_mtg2::fake_double_loop

src/multio/action/print/Print.cc

@@ -11,16 +11,34 @@
 #include "Print.h"
 
 #include <fstream>
+#include <iomanip>
+#include <iostream>
 
 #include "eckit/config/Configuration.h"
+#include "eckit/config/LocalConfiguration.h"
 #include "eckit/exception/Exceptions.h"
 #include "eckit/log/Log.h"
 
+#include "multio/datamod/MarsMiscGeo.h"
+#include "multio/datamod/core/EntryDumper.h"
+
+#include "multio/datamod/ContainerInterop.h"
+#include "multio/datamod/MarsMiscGeo.h"
+#include "multio/datamod/core/EntryDumper.h"
+#include "multio/datamod/core/EntryParser.h"
+#include "multio/datamod/core/Record.h"
+#include "multio/message/Parametrization.h"
+#include "multio/util/PrecisionTag.h"
+
 namespace multio::action::print {
 
+namespace dm = multio::datamod;
+
 Print::Print(const ComponentConfiguration& compConf) : ChainedAction(compConf) {
     stream_ = compConf.parsedConfig().getString("stream", "info");
     onlyFields_ = compConf.parsedConfig().getBool("only-fields", false);
+    marsStream_ = (stream_ == "mars");
+    count_ = 1;
 
     if (stream_ == "info") {
         os_ = &eckit::Log::info();
@@ -31,23 +49,69 @@ Print::Print(const ComponentConfiguration& compConf) : ChainedAction(compConf) {
     else if (stream_ == "cout") {
         os_ = &std::cout;
     }
+    else if (stream_ == "mars") {
+        os_ = &std::cout;
+    }
     else {
         os_ = &eckit::Log::debug();
     }
 
     prefix_ = compConf.parsedConfig().getString("prefix", "");
 }
 
+void Print::printPrefix(std::ostream& os) const {
+    if (!prefix_.empty()) {
+        os << prefix_ << ": ";
+    }
+}
+
+void Print::printMars(std::ostream& os, const message::Message& msg) const {
+    if (msg.tag() == message::Message::Tag::Field) {
+        auto mars = dm::readRecord<dm::FullMarsRecord>(msg.metadata());
+        auto md = dm::dumpRecord<message::Metadata>(mars);
+
+        // printPrefix(os);
+        os << prefix_ << ": Field: " << std::setw(6) << count_++ << " :: \"mars\":";
+        os << md << std::endl;
+        return;
+    }
+
+    if (msg.tag() == message::Message::Tag::Flush) {
+        count_ = 1;
+        // printPrefix(os);
+        long flushKind = msg.metadata().getOpt<long>("flushKind").value_or(-1);
+        if (flushKind == 1) {
+            long step = msg.metadata().getOpt<long>("step").value_or(-1);
+            os << prefix_ << ": Flush: step=" << step << std::endl;
+        }
+        else {
+            os << prefix_ << ": Flush: " << flushKind << std::endl;
+        }
+        os << std::endl << std::endl;
+    }
+}
+
 void Print::executeImpl(message::Message msg) {
     ASSERT(os_);
     bool doOutput = onlyFields_ ? (msg.tag() == message::Message::Tag::Field) : true;
     if (doOutput) {
-        if (!prefix_.empty()) {
-            std::cout << prefix_ << ": ";
+        if (marsStream_) {
+            printMars(*os_, msg);
+        }
+        else {
+            printPrefix(*os_);
+            *os_ << msg << std::endl;
         }
-        std::cout << msg << std::endl;
     }
-    executeNext(std::move(msg));
+    // try {
+        executeNext(std::move(msg));
+    // }
+    // catch (...) {
+    //     std::cerr << "Received \"mars\":";
+    //     printMars(std::cerr, msg);
+    //     std::cerr << "# =======================================================================================" << std::endl;
+    //     std::cerr << std::endl << std::endl << std::endl << std::endl << std::endl << std::endl << std::endl;
+    // }
 }
 
 void Print::print(std::ostream& os) const {

src/multio/action/print/Print.h

@@ -30,9 +30,13 @@ class Print : public ChainedAction {
 
 private:
     void print(std::ostream& os) const override;
+    void printPrefix(std::ostream& os) const;
+    void printMars(std::ostream& os, const message::Message& msg) const;
 
     bool onlyFields_;
     std::string stream_;
+    bool marsStream_;
+    mutable long count_;
 
     std::ostream* os_;
     std::string prefix_;