Support zlib compressed level 3 files

test/data

@@ -1 +1 @@
-Subproject commit 4e9a2804d977acebbebdcfb73efa9dec190b28e5
+Subproject commit 5de70eccee20d5d8358acb4ebcc209259658fddd

test/source/scwx/wsr88d/level3_file.test.cpp

@@ -73,6 +73,7 @@ INSTANTIATE_TEST_SUITE_P(
       std::pair<int16_t, std::string> {176, "KLSX_SDUS83_DPRLSX_202112110140"},
       std::pair<int16_t, std::string> {177, "KLSX_SDUS83_HHCLSX_202112110140"},
       std::pair<int16_t, std::string> {99, "Level3_LSX_N1U_20211228_0446.nids"},
+      std::pair<int16_t, std::string> {37, "Level3_STL_NCR_20211211_0200.nids"},
       std::pair<int16_t, std::string> {180,
                                        "Level3_STL_TZ0_20211211_0200.nids"},
       std::pair<int16_t, std::string> {182,

wxdata/include/scwx/wsr88d/rpg/ccb_header.hpp

@@ -0,0 +1,60 @@
+#pragma once
+
+#include <memory>
+#include <string>
+
+namespace scwx
+{
+namespace wsr88d
+{
+namespace rpg
+{
+
+class CcbHeaderImpl;
+
+/**
+ * @brief The Communication Control Block Header is defined in the Interface
+ * Control Document (ICD) for the National Weather Service Telecommunications
+ * Gateway (NWSTG).
+ *
+ * <https://web.archive.org/web/20010706211204/http://www.nws.noaa.gov/pub/noaaport/nwstgicd.pdf>
+ */
+class CcbHeader
+{
+public:
+   explicit CcbHeader();
+   ~CcbHeader();
+
+   CcbHeader(const CcbHeader&) = delete;
+   CcbHeader& operator=(const CcbHeader&) = delete;
+
+   CcbHeader(CcbHeader&&) noexcept;
+   CcbHeader& operator=(CcbHeader&&) noexcept;
+
+   uint16_t    ff() const;
+   uint16_t    ccb_length() const;
+   uint8_t     mode() const;
+   uint8_t     submode() const;
+   char        precedence() const;
+   char        classification() const;
+   std::string message_originator() const;
+   uint8_t     category() const;
+   uint8_t     subcategory() const;
+   uint16_t    user_defined() const;
+   uint8_t     year() const;
+   uint8_t     month() const;
+   uint8_t     tor_day() const;
+   uint8_t     tor_hour() const;
+   uint8_t     tor_minute() const;
+   uint8_t     number_of_destinations() const;
+   std::string message_destination(uint8_t i) const;
+
+   bool Parse(std::istream& is);
+
+private:
+   std::unique_ptr<CcbHeaderImpl> p;
+};
+
+} // namespace rpg
+} // namespace wsr88d
+} // namespace scwx

wxdata/source/scwx/wsr88d/level3_file.cpp

@@ -1,4 +1,5 @@
 #include <scwx/wsr88d/level3_file.hpp>
+#include <scwx/wsr88d/rpg/ccb_header.hpp>
 #include <scwx/wsr88d/rpg/level3_message_header.hpp>
 #include <scwx/wsr88d/rpg/product_description_block.hpp>
 #include <scwx/wsr88d/rpg/product_symbology_block.hpp>
@@ -11,6 +12,7 @@
 #include <boost/iostreams/copy.hpp>
 #include <boost/iostreams/filtering_streambuf.hpp>
 #include <boost/iostreams/filter/bzip2.hpp>
+#include <boost/iostreams/filter/zlib.hpp>
 #include <boost/log/trivial.hpp>
 
 namespace scwx
@@ -25,6 +27,7 @@ class Level3FileImpl
 public:
    explicit Level3FileImpl() :
        wmoHeader_ {},
+       ccbHeader_ {},
        messageHeader_ {},
        descriptionBlock_ {},
        symbologyBlock_ {},
@@ -32,11 +35,15 @@ public:
        tabularBlock_ {} {};
    ~Level3FileImpl() = default;
 
+   bool DecompressFile(std::istream& is, std::stringstream& ss);
+   bool LoadFileData(std::istream& is);
    bool LoadBlocks(std::istream& is);
 
-   rpg::WmoHeader               wmoHeader_;
+   rpg::WmoHeader                  wmoHeader_;
-   rpg::Level3MessageHeader     messageHeader_;
+   std::shared_ptr<rpg::CcbHeader> ccbHeader_;
-   rpg::ProductDescriptionBlock descriptionBlock_;
+   std::shared_ptr<rpg::WmoHeader> innerHeader_;
+   rpg::Level3MessageHeader        messageHeader_;
+   rpg::ProductDescriptionBlock    descriptionBlock_;
 
    std::shared_ptr<rpg::ProductSymbologyBlock> symbologyBlock_;
    std::shared_ptr<void>                       graphicBlock_;
@@ -89,22 +96,107 @@ bool Level3File::LoadData(std::istream& is)
       BOOST_LOG_TRIVIAL(debug)
          << logPrefix_ << "Category:  " << p->wmoHeader_.product_category();
 
-      dataValid = p->messageHeader_.Parse(is);
+      // If the header is compressed
+      if (is.peek() == 0x78)
+      {
+         std::stringstream ss;
+
+         dataValid = p->DecompressFile(is, ss);
+
+         if (dataValid)
+         {
+            dataValid = p->LoadFileData(ss);
+         }
+      }
+      else
+      {
+         dataValid = p->LoadFileData(is);
+      }
    }
 
+   return dataValid;
+}
+
+bool Level3FileImpl::DecompressFile(std::istream& is, std::stringstream& ss)
+{
+   bool dataValid = true;
+
+   std::streampos  dataStart          = is.tellg();
+   std::streamsize totalBytesCopied   = 0;
+   int             totalBytesConsumed = 0;
+
+   while (dataValid && is.peek() == 0x78)
+      try
+      {
+         boost::iostreams::filtering_streambuf<boost::iostreams::input> in;
+         boost::iostreams::zlib_decompressor zlibDecompressor;
+         in.push(zlibDecompressor);
+         in.push(is);
+
+         std::streamsize bytesCopied   = boost::iostreams::copy(in, ss);
+         int             bytesConsumed = zlibDecompressor.filter().total_in();
+
+         totalBytesCopied += bytesCopied;
+         totalBytesConsumed += bytesConsumed;
+
+         is.seekg(dataStart + static_cast<std::streamoff>(totalBytesConsumed),
+                  std::ios_base::beg);
+
+         if (bytesConsumed <= 0)
+         {
+            // Not sure this will ever occur, but will prevent an infinite loop
+            break;
+         }
+      }
+      catch (const boost::iostreams::zlib_error& ex)
+      {
+         int error = ex.error();
+
+         BOOST_LOG_TRIVIAL(warning)
+            << logPrefix_ << "Error decompressing data: " << ex.what();
+
+         dataValid = false;
+      }
+
+   if (dataValid)
+   {
+      BOOST_LOG_TRIVIAL(trace)
+         << logPrefix_ << "Input data consumed = " << totalBytesCopied
+         << " bytes";
+      BOOST_LOG_TRIVIAL(trace)
+         << logPrefix_ << "Decompressed data size = " << totalBytesConsumed
+         << " bytes";
+
+      ccbHeader_ = std::make_shared<rpg::CcbHeader>();
+      dataValid  = ccbHeader_->Parse(ss);
+   }
+
+   if (dataValid)
+   {
+      innerHeader_ = std::make_shared<rpg::WmoHeader>();
+      dataValid    = innerHeader_->Parse(ss);
+   }
+
+   return dataValid;
+}
+
+bool Level3FileImpl::LoadFileData(std::istream& is)
+{
+   bool dataValid = messageHeader_.Parse(is);
+
    if (dataValid)
    {
       BOOST_LOG_TRIVIAL(debug)
-         << logPrefix_ << "Code:      " << p->messageHeader_.message_code();
+         << logPrefix_ << "Code:      " << messageHeader_.message_code();
 
-      dataValid = p->descriptionBlock_.Parse(is);
+      dataValid = descriptionBlock_.Parse(is);
    }
 
    if (dataValid)
    {
-      if (p->descriptionBlock_.IsCompressionEnabled())
+      if (descriptionBlock_.IsCompressionEnabled())
       {
-         size_t messageLength = p->messageHeader_.length_of_message();
+         size_t messageLength = messageHeader_.length_of_message();
          size_t prefixLength =
             rpg::Level3MessageHeader::SIZE + rpg::ProductDescriptionBlock::SIZE;
          size_t recordSize =
@@ -123,7 +215,7 @@ bool Level3File::LoadData(std::istream& is)
                << logPrefix_ << "Decompressed data size = " << bytesCopied
                << " bytes";
 
-            dataValid = p->LoadBlocks(ss);
+            dataValid = LoadBlocks(ss);
          }
          catch (const boost::iostreams::bzip2_error& ex)
          {
@@ -136,7 +228,7 @@ bool Level3File::LoadData(std::istream& is)
       }
       else
       {
-         dataValid = p->LoadBlocks(is);
+         dataValid = LoadBlocks(is);
       }
    }
 

wxdata/source/scwx/wsr88d/rpg/ccb_header.cpp

@@ -0,0 +1,209 @@
+#include <scwx/wsr88d/rpg/ccb_header.hpp>
+#include <scwx/util/streams.hpp>
+
+#include <istream>
+#include <sstream>
+#include <string>
+
+#include <boost/log/trivial.hpp>
+
+#ifdef WIN32
+#   include <WinSock2.h>
+#else
+#   include <arpa/inet.h>
+#endif
+
+namespace scwx
+{
+namespace wsr88d
+{
+namespace rpg
+{
+
+static const std::string logPrefix_ = "[scwx::wsr88d::rpg::ccb_header] ";
+
+class CcbHeaderImpl
+{
+public:
+   explicit CcbHeaderImpl() :
+       ff_ {0},
+       ccbLength_ {0},
+       mode_ {0},
+       submode_ {0},
+       precedence_ {0},
+       classification_ {0},
+       messageOriginator_ {},
+       category_ {0},
+       subcategory_ {0},
+       userDefined_ {0},
+       year_ {0},
+       month_ {0},
+       torDay_ {0},
+       torHour_ {0},
+       torMinute_ {0},
+       numberOfDestinations_ {0},
+       messageDestination_ {}
+   {
+   }
+   ~CcbHeaderImpl() = default;
+
+   uint16_t                 ff_;
+   uint16_t                 ccbLength_;
+   uint8_t                  mode_;
+   uint8_t                  submode_;
+   char                     precedence_;
+   char                     classification_;
+   std::string              messageOriginator_;
+   uint8_t                  category_;
+   uint8_t                  subcategory_;
+   uint16_t                 userDefined_;
+   uint8_t                  year_;
+   uint8_t                  month_;
+   uint8_t                  torDay_;
+   uint8_t                  torHour_;
+   uint8_t                  torMinute_;
+   uint8_t                  numberOfDestinations_;
+   std::vector<std::string> messageDestination_;
+};
+
+CcbHeader::CcbHeader() : p(std::make_unique<CcbHeaderImpl>()) {}
+CcbHeader::~CcbHeader() = default;
+
+CcbHeader::CcbHeader(CcbHeader&&) noexcept = default;
+CcbHeader& CcbHeader::operator=(CcbHeader&&) noexcept = default;
+
+uint16_t CcbHeader::ff() const
+{
+   return p->ff_;
+}
+
+uint16_t CcbHeader::ccb_length() const
+{
+   return p->ccbLength_;
+}
+
+uint8_t CcbHeader::mode() const
+{
+   return p->mode_;
+}
+
+uint8_t CcbHeader::submode() const
+{
+   return p->submode_;
+}
+
+char CcbHeader::precedence() const
+{
+   return p->precedence_;
+}
+
+char CcbHeader::classification() const
+{
+   return p->classification_;
+}
+
+std::string CcbHeader::message_originator() const
+{
+   return p->messageOriginator_;
+}
+
+uint8_t CcbHeader::category() const
+{
+   return p->category_;
+}
+
+uint8_t CcbHeader::subcategory() const
+{
+   return p->subcategory_;
+}
+
+uint16_t CcbHeader::user_defined() const
+{
+   return p->userDefined_;
+}
+
+uint8_t CcbHeader::year() const
+{
+   return p->year_;
+}
+
+uint8_t CcbHeader::month() const
+{
+   return p->month_;
+}
+
+uint8_t CcbHeader::tor_day() const
+{
+   return p->torDay_;
+}
+
+uint8_t CcbHeader::tor_hour() const
+{
+   return p->torHour_;
+}
+
+uint8_t CcbHeader::tor_minute() const
+{
+   return p->torMinute_;
+}
+
+uint8_t CcbHeader::number_of_destinations() const
+{
+   return p->numberOfDestinations_;
+}
+
+std::string CcbHeader::message_destination(uint8_t i) const
+{
+   return p->messageDestination_[i];
+}
+
+bool CcbHeader::Parse(std::istream& is)
+{
+   bool headerValid = true;
+
+   is.read(reinterpret_cast<char*>(&p->ccbLength_), 2);
+
+   p->ccbLength_ = ntohs(p->ccbLength_);
+
+   p->ff_        = p->ccbLength_ >> 14;
+   p->ccbLength_ = p->ccbLength_ & 0x3fff;
+
+   p->messageOriginator_.resize(4);
+
+   is.read(reinterpret_cast<char*>(&p->mode_), 1);
+   is.read(reinterpret_cast<char*>(&p->submode_), 1);
+   is.read(&p->precedence_, 1);
+   is.read(&p->classification_, 1);
+   is.read(p->messageOriginator_.data(), 4);
+   is.read(reinterpret_cast<char*>(&p->category_), 1);
+   is.read(reinterpret_cast<char*>(&p->subcategory_), 1);
+   is.read(reinterpret_cast<char*>(&p->userDefined_), 2);
+   is.read(reinterpret_cast<char*>(&p->year_), 1);
+   is.read(reinterpret_cast<char*>(&p->month_), 1);
+   is.read(reinterpret_cast<char*>(&p->torDay_), 1);
+   is.read(reinterpret_cast<char*>(&p->torHour_), 1);
+   is.read(reinterpret_cast<char*>(&p->torMinute_), 1);
+   is.read(reinterpret_cast<char*>(&p->numberOfDestinations_), 1);
+
+   p->userDefined_ = ntohs(p->userDefined_);
+
+   p->messageDestination_.resize(p->numberOfDestinations_);
+
+   for (uint8_t d = 0; d < p->numberOfDestinations_; d++)
+   {
+      p->messageDestination_[d].resize(4);
+      is.read(p->messageDestination_[d].data(), 4);
+   }
+
+   if (is.eof())
+   {
+      BOOST_LOG_TRIVIAL(debug) << logPrefix_ << "Reached end of file";
+      headerValid = false;
+   }
+
+   return headerValid;
+}
+
+} // namespace rpg
+} // namespace wsr88d
+} // namespace scwx

wxdata/wxdata.cmake

@@ -46,7 +46,8 @@ set(SRC_WSR88D_RDA source/scwx/wsr88d/rda/clutter_filter_map.cpp
                    source/scwx/wsr88d/rda/rda_adaptation_data.cpp
                    source/scwx/wsr88d/rda/rda_status_data.cpp
                    source/scwx/wsr88d/rda/volume_coverage_pattern_data.cpp)
-set(HDR_WSR88D_RPG include/scwx/wsr88d/rpg/digital_precipitation_data_array_packet.hpp
+set(HDR_WSR88D_RPG include/scwx/wsr88d/rpg/ccb_header.hpp
+                   include/scwx/wsr88d/rpg/digital_precipitation_data_array_packet.hpp
                    include/scwx/wsr88d/rpg/digital_radial_data_array_packet.hpp
                    include/scwx/wsr88d/rpg/generic_data_packet.hpp
                    include/scwx/wsr88d/rpg/hda_hail_symbol_packet.hpp
@@ -74,7 +75,8 @@ set(HDR_WSR88D_RPG include/scwx/wsr88d/rpg/digital_precipitation_data_array_pack
                    include/scwx/wsr88d/rpg/vector_arrow_data_packet.hpp
                    include/scwx/wsr88d/rpg/wind_barb_data_packet.hpp
                    include/scwx/wsr88d/rpg/wmo_header.hpp)
-set(SRC_WSR88D_RPG source/scwx/wsr88d/rpg/digital_precipitation_data_array_packet.cpp
+set(SRC_WSR88D_RPG source/scwx/wsr88d/rpg/ccb_header.cpp
+                   source/scwx/wsr88d/rpg/digital_precipitation_data_array_packet.cpp
                    source/scwx/wsr88d/rpg/digital_radial_data_array_packet.cpp
                    source/scwx/wsr88d/rpg/generic_data_packet.cpp
                    source/scwx/wsr88d/rpg/hda_hail_symbol_packet.cpp