From 80579dbff27e7348f0671442d0177cea466243e7 Mon Sep 17 00:00:00 2001
From: Dan Paulat <dpaulat@gmail.com>
Date: Mon, 24 Jan 2022 16:29:40 -0600
Subject: [PATCH] Support for overview block in text product, and relax
 formatting guidelines

---
 test/data                                     |  2 +-
 .../scwx/awips/text_product_file.test.cpp     |  4 ++-
 .../scwx/awips/text_product_message.cpp       | 35 ++++++++++++++++++-
 wxdata/source/scwx/awips/wmo_header.cpp       |  8 +++--
 4 files changed, 44 insertions(+), 5 deletions(-)

diff --git a/test/data b/test/data
index b8c76bd2..a1b78dcb 160000
--- a/test/data
+++ b/test/data
@@ -1 +1 @@
-Subproject commit b8c76bd23f636f9dad08efafa348dcbfe878f0b1
+Subproject commit a1b78dcb40b85271270afd7dc9d88fe8594471b2
diff --git a/test/source/scwx/awips/text_product_file.test.cpp b/test/source/scwx/awips/text_product_file.test.cpp
index 52426a1d..251a1c01 100644
--- a/test/source/scwx/awips/text_product_file.test.cpp
+++ b/test/source/scwx/awips/text_product_file.test.cpp
@@ -33,7 +33,9 @@ INSTANTIATE_TEST_SUITE_P(
    testing::Values("/warnings/warnings_20210604_21.txt",
                    "/warnings/warnings_20210606_15.txt",
                    "/warnings/warnings_20210606_22-59.txt",
-                   "/nexrad/level3/KLSX_NOUS63_FTMLSX_202201041404"));
+                   "/nexrad/level3/KLSX_NOUS63_FTMLSX_202201041404",
+                   "/text/PGUM_WHPQ41_CFWPQ1_202201231710.nids",
+                   "/text/PGUM_WHPQ42_CFWPQ2_202201231924.nids"));
 
 TEST(TextProductFile, Update)
 {
diff --git a/wxdata/source/scwx/awips/text_product_message.cpp b/wxdata/source/scwx/awips/text_product_message.cpp
index e205c066..a8a7f768 100644
--- a/wxdata/source/scwx/awips/text_product_message.cpp
+++ b/wxdata/source/scwx/awips/text_product_message.cpp
@@ -75,6 +75,7 @@ static std::vector<std::string>     ParseProductContent(std::istream& is);
 void                                SkipBlankLines(std::istream& is);
 bool                                TryParseEndOfProduct(std::istream& is);
 static std::vector<std::string>     TryParseMndHeader(std::istream& is);
+static std::vector<std::string>     TryParseOverviewBlock(std::istream& is);
 static std::optional<SegmentHeader> TryParseSegmentHeader(std::istream& is);
 static std::optional<Vtec>          TryParseVtecString(std::istream& is);
 
@@ -86,6 +87,7 @@ public:
 
    std::shared_ptr<WmoHeader>            wmoHeader_;
    std::vector<std::string>              mndHeader_;
+   std::vector<std::string>              overviewBlock_;
    std::vector<std::shared_ptr<Segment>> segments_;
 };
 
@@ -136,6 +138,13 @@ bool TextProductMessage::Parse(std::istream& is)
 
          p->mndHeader_ = TryParseMndHeader(is);
          SkipBlankLines(is);
+
+         // Optional overview block appears between MND and segment header
+         if (!segment->header_.has_value())
+         {
+            p->overviewBlock_ = TryParseOverviewBlock(is);
+            SkipBlankLines(is);
+         }
       }
 
       if (!segment->header_.has_value())
@@ -165,7 +174,10 @@ std::vector<std::string> ParseProductContent(std::istream& is)
    {
       util::getline(is, line);
 
-      productContent.push_back(line);
+      if (!productContent.empty() || !line.starts_with("$$"))
+      {
+         productContent.push_back(line);
+      }
 
       if (line.starts_with("$$"))
       {
@@ -263,6 +275,27 @@ std::vector<std::string> TryParseMndHeader(std::istream& is)
    return mndHeader;
 }
 
+std::vector<std::string> TryParseOverviewBlock(std::istream& is)
+{
+   // Optional overview block contains text in the following format:
+   // ...OVERVIEW HEADLINE... /OPTIONAL/
+   // .OVERVIEW WITH GENERAL INFORMATION / OPTIONAL /
+   // Key off the block beginning with .
+   std::vector<std::string> overviewBlock;
+   std::string              line;
+
+   if (is.peek() == '.')
+   {
+      while (!is.eof() && is.peek() != '\r')
+      {
+         util::getline(is, line);
+         overviewBlock.push_back(line);
+      }
+   }
+
+   return overviewBlock;
+}
+
 std::optional<SegmentHeader> TryParseSegmentHeader(std::istream& is)
 {
    // UGC takes the form SSFNNN-NNN>NNN-SSFNNN-DDHHMM- (NWSI 10-1702)
diff --git a/wxdata/source/scwx/awips/wmo_header.cpp b/wxdata/source/scwx/awips/wmo_header.cpp
index 7f3d3466..19bae36d 100644
--- a/wxdata/source/scwx/awips/wmo_header.cpp
+++ b/wxdata/source/scwx/awips/wmo_header.cpp
@@ -148,7 +148,7 @@ bool WmoHeader::Parse(std::istream& is)
          << logPrefix_ << "Start of Heading Line is malformed";
       headerValid = false;
    }
-   else if (!sequenceLine.empty() && !sequenceLine.ends_with(" \r\r"))
+   else if (!sequenceLine.empty() && !sequenceLine.ends_with("\r\r"))
    {
       BOOST_LOG_TRIVIAL(debug) << logPrefix_ << "Sequence Line is malformed";
       headerValid = false;
@@ -170,7 +170,11 @@ bool WmoHeader::Parse(std::istream& is)
       // Remove delimiters from the end of the line
       if (!sequenceLine.empty())
       {
-         sequenceLine.erase(sequenceLine.length() - 3);
+         sequenceLine.erase(sequenceLine.length() - 2);
+         while (sequenceLine.ends_with(' '))
+         {
+            sequenceLine.erase(sequenceLine.length() - 1);
+         }
       }
 
       wmoLine.erase(wmoLine.length() - 2);