From 01d24d70b8fdd0e59ba383066f5a97da68e12747 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Tue, 11 Jan 2022 18:37:02 -0600 Subject: [PATCH] Support zlib compressed level 3 files --- test/data | 2 +- test/source/scwx/wsr88d/level3_file.test.cpp | 1 + wxdata/include/scwx/wsr88d/rpg/ccb_header.hpp | 60 +++++ wxdata/source/scwx/wsr88d/level3_file.cpp | 112 +++++++++- wxdata/source/scwx/wsr88d/rpg/ccb_header.cpp | 209 ++++++++++++++++++ wxdata/wxdata.cmake | 6 +- 6 files changed, 377 insertions(+), 13 deletions(-) create mode 100644 wxdata/include/scwx/wsr88d/rpg/ccb_header.hpp create mode 100644 wxdata/source/scwx/wsr88d/rpg/ccb_header.cpp diff --git a/test/data b/test/data index 4e9a2804..5de70ecc 160000 --- a/test/data +++ b/test/data @@ -1 +1 @@ -Subproject commit 4e9a2804d977acebbebdcfb73efa9dec190b28e5 +Subproject commit 5de70eccee20d5d8358acb4ebcc209259658fddd diff --git a/test/source/scwx/wsr88d/level3_file.test.cpp b/test/source/scwx/wsr88d/level3_file.test.cpp index d371195e..74f59b35 100644 --- a/test/source/scwx/wsr88d/level3_file.test.cpp +++ b/test/source/scwx/wsr88d/level3_file.test.cpp @@ -73,6 +73,7 @@ INSTANTIATE_TEST_SUITE_P( std::pair {176, "KLSX_SDUS83_DPRLSX_202112110140"}, std::pair {177, "KLSX_SDUS83_HHCLSX_202112110140"}, std::pair {99, "Level3_LSX_N1U_20211228_0446.nids"}, + std::pair {37, "Level3_STL_NCR_20211211_0200.nids"}, std::pair {180, "Level3_STL_TZ0_20211211_0200.nids"}, std::pair {182, diff --git a/wxdata/include/scwx/wsr88d/rpg/ccb_header.hpp b/wxdata/include/scwx/wsr88d/rpg/ccb_header.hpp new file mode 100644 index 00000000..df53cd92 --- /dev/null +++ b/wxdata/include/scwx/wsr88d/rpg/ccb_header.hpp @@ -0,0 +1,60 @@ +#pragma once + +#include +#include + +namespace scwx +{ +namespace wsr88d +{ +namespace rpg +{ + +class CcbHeaderImpl; + +/** + * @brief The Communication Control Block Header is defined in the Interface + * Control Document (ICD) for the National Weather Service Telecommunications + * Gateway (NWSTG). + * + * + */ +class CcbHeader +{ +public: + explicit CcbHeader(); + ~CcbHeader(); + + CcbHeader(const CcbHeader&) = delete; + CcbHeader& operator=(const CcbHeader&) = delete; + + CcbHeader(CcbHeader&&) noexcept; + CcbHeader& operator=(CcbHeader&&) noexcept; + + uint16_t ff() const; + uint16_t ccb_length() const; + uint8_t mode() const; + uint8_t submode() const; + char precedence() const; + char classification() const; + std::string message_originator() const; + uint8_t category() const; + uint8_t subcategory() const; + uint16_t user_defined() const; + uint8_t year() const; + uint8_t month() const; + uint8_t tor_day() const; + uint8_t tor_hour() const; + uint8_t tor_minute() const; + uint8_t number_of_destinations() const; + std::string message_destination(uint8_t i) const; + + bool Parse(std::istream& is); + +private: + std::unique_ptr p; +}; + +} // namespace rpg +} // namespace wsr88d +} // namespace scwx diff --git a/wxdata/source/scwx/wsr88d/level3_file.cpp b/wxdata/source/scwx/wsr88d/level3_file.cpp index 7302a715..512636dd 100644 --- a/wxdata/source/scwx/wsr88d/level3_file.cpp +++ b/wxdata/source/scwx/wsr88d/level3_file.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -11,6 +12,7 @@ #include #include #include +#include #include namespace scwx @@ -25,6 +27,7 @@ class Level3FileImpl public: explicit Level3FileImpl() : wmoHeader_ {}, + ccbHeader_ {}, messageHeader_ {}, descriptionBlock_ {}, symbologyBlock_ {}, @@ -32,11 +35,15 @@ public: tabularBlock_ {} {}; ~Level3FileImpl() = default; + bool DecompressFile(std::istream& is, std::stringstream& ss); + bool LoadFileData(std::istream& is); bool LoadBlocks(std::istream& is); - rpg::WmoHeader wmoHeader_; - rpg::Level3MessageHeader messageHeader_; - rpg::ProductDescriptionBlock descriptionBlock_; + rpg::WmoHeader wmoHeader_; + std::shared_ptr ccbHeader_; + std::shared_ptr innerHeader_; + rpg::Level3MessageHeader messageHeader_; + rpg::ProductDescriptionBlock descriptionBlock_; std::shared_ptr symbologyBlock_; std::shared_ptr graphicBlock_; @@ -89,22 +96,107 @@ bool Level3File::LoadData(std::istream& is) BOOST_LOG_TRIVIAL(debug) << logPrefix_ << "Category: " << p->wmoHeader_.product_category(); - dataValid = p->messageHeader_.Parse(is); + // If the header is compressed + if (is.peek() == 0x78) + { + std::stringstream ss; + + dataValid = p->DecompressFile(is, ss); + + if (dataValid) + { + dataValid = p->LoadFileData(ss); + } + } + else + { + dataValid = p->LoadFileData(is); + } } + return dataValid; +} + +bool Level3FileImpl::DecompressFile(std::istream& is, std::stringstream& ss) +{ + bool dataValid = true; + + std::streampos dataStart = is.tellg(); + std::streamsize totalBytesCopied = 0; + int totalBytesConsumed = 0; + + while (dataValid && is.peek() == 0x78) + try + { + boost::iostreams::filtering_streambuf in; + boost::iostreams::zlib_decompressor zlibDecompressor; + in.push(zlibDecompressor); + in.push(is); + + std::streamsize bytesCopied = boost::iostreams::copy(in, ss); + int bytesConsumed = zlibDecompressor.filter().total_in(); + + totalBytesCopied += bytesCopied; + totalBytesConsumed += bytesConsumed; + + is.seekg(dataStart + static_cast(totalBytesConsumed), + std::ios_base::beg); + + if (bytesConsumed <= 0) + { + // Not sure this will ever occur, but will prevent an infinite loop + break; + } + } + catch (const boost::iostreams::zlib_error& ex) + { + int error = ex.error(); + + BOOST_LOG_TRIVIAL(warning) + << logPrefix_ << "Error decompressing data: " << ex.what(); + + dataValid = false; + } + + if (dataValid) + { + BOOST_LOG_TRIVIAL(trace) + << logPrefix_ << "Input data consumed = " << totalBytesCopied + << " bytes"; + BOOST_LOG_TRIVIAL(trace) + << logPrefix_ << "Decompressed data size = " << totalBytesConsumed + << " bytes"; + + ccbHeader_ = std::make_shared(); + dataValid = ccbHeader_->Parse(ss); + } + + if (dataValid) + { + innerHeader_ = std::make_shared(); + dataValid = innerHeader_->Parse(ss); + } + + return dataValid; +} + +bool Level3FileImpl::LoadFileData(std::istream& is) +{ + bool dataValid = messageHeader_.Parse(is); + if (dataValid) { BOOST_LOG_TRIVIAL(debug) - << logPrefix_ << "Code: " << p->messageHeader_.message_code(); + << logPrefix_ << "Code: " << messageHeader_.message_code(); - dataValid = p->descriptionBlock_.Parse(is); + dataValid = descriptionBlock_.Parse(is); } if (dataValid) { - if (p->descriptionBlock_.IsCompressionEnabled()) + if (descriptionBlock_.IsCompressionEnabled()) { - size_t messageLength = p->messageHeader_.length_of_message(); + size_t messageLength = messageHeader_.length_of_message(); size_t prefixLength = rpg::Level3MessageHeader::SIZE + rpg::ProductDescriptionBlock::SIZE; size_t recordSize = @@ -123,7 +215,7 @@ bool Level3File::LoadData(std::istream& is) << logPrefix_ << "Decompressed data size = " << bytesCopied << " bytes"; - dataValid = p->LoadBlocks(ss); + dataValid = LoadBlocks(ss); } catch (const boost::iostreams::bzip2_error& ex) { @@ -136,7 +228,7 @@ bool Level3File::LoadData(std::istream& is) } else { - dataValid = p->LoadBlocks(is); + dataValid = LoadBlocks(is); } } diff --git a/wxdata/source/scwx/wsr88d/rpg/ccb_header.cpp b/wxdata/source/scwx/wsr88d/rpg/ccb_header.cpp new file mode 100644 index 00000000..92983a94 --- /dev/null +++ b/wxdata/source/scwx/wsr88d/rpg/ccb_header.cpp @@ -0,0 +1,209 @@ +#include +#include + +#include +#include +#include + +#include + +#ifdef WIN32 +# include +#else +# include +#endif + +namespace scwx +{ +namespace wsr88d +{ +namespace rpg +{ + +static const std::string logPrefix_ = "[scwx::wsr88d::rpg::ccb_header] "; + +class CcbHeaderImpl +{ +public: + explicit CcbHeaderImpl() : + ff_ {0}, + ccbLength_ {0}, + mode_ {0}, + submode_ {0}, + precedence_ {0}, + classification_ {0}, + messageOriginator_ {}, + category_ {0}, + subcategory_ {0}, + userDefined_ {0}, + year_ {0}, + month_ {0}, + torDay_ {0}, + torHour_ {0}, + torMinute_ {0}, + numberOfDestinations_ {0}, + messageDestination_ {} + { + } + ~CcbHeaderImpl() = default; + + uint16_t ff_; + uint16_t ccbLength_; + uint8_t mode_; + uint8_t submode_; + char precedence_; + char classification_; + std::string messageOriginator_; + uint8_t category_; + uint8_t subcategory_; + uint16_t userDefined_; + uint8_t year_; + uint8_t month_; + uint8_t torDay_; + uint8_t torHour_; + uint8_t torMinute_; + uint8_t numberOfDestinations_; + std::vector messageDestination_; +}; + +CcbHeader::CcbHeader() : p(std::make_unique()) {} +CcbHeader::~CcbHeader() = default; + +CcbHeader::CcbHeader(CcbHeader&&) noexcept = default; +CcbHeader& CcbHeader::operator=(CcbHeader&&) noexcept = default; + +uint16_t CcbHeader::ff() const +{ + return p->ff_; +} + +uint16_t CcbHeader::ccb_length() const +{ + return p->ccbLength_; +} + +uint8_t CcbHeader::mode() const +{ + return p->mode_; +} + +uint8_t CcbHeader::submode() const +{ + return p->submode_; +} + +char CcbHeader::precedence() const +{ + return p->precedence_; +} + +char CcbHeader::classification() const +{ + return p->classification_; +} + +std::string CcbHeader::message_originator() const +{ + return p->messageOriginator_; +} + +uint8_t CcbHeader::category() const +{ + return p->category_; +} + +uint8_t CcbHeader::subcategory() const +{ + return p->subcategory_; +} + +uint16_t CcbHeader::user_defined() const +{ + return p->userDefined_; +} + +uint8_t CcbHeader::year() const +{ + return p->year_; +} + +uint8_t CcbHeader::month() const +{ + return p->month_; +} + +uint8_t CcbHeader::tor_day() const +{ + return p->torDay_; +} + +uint8_t CcbHeader::tor_hour() const +{ + return p->torHour_; +} + +uint8_t CcbHeader::tor_minute() const +{ + return p->torMinute_; +} + +uint8_t CcbHeader::number_of_destinations() const +{ + return p->numberOfDestinations_; +} + +std::string CcbHeader::message_destination(uint8_t i) const +{ + return p->messageDestination_[i]; +} + +bool CcbHeader::Parse(std::istream& is) +{ + bool headerValid = true; + + is.read(reinterpret_cast(&p->ccbLength_), 2); + + p->ccbLength_ = ntohs(p->ccbLength_); + + p->ff_ = p->ccbLength_ >> 14; + p->ccbLength_ = p->ccbLength_ & 0x3fff; + + p->messageOriginator_.resize(4); + + is.read(reinterpret_cast(&p->mode_), 1); + is.read(reinterpret_cast(&p->submode_), 1); + is.read(&p->precedence_, 1); + is.read(&p->classification_, 1); + is.read(p->messageOriginator_.data(), 4); + is.read(reinterpret_cast(&p->category_), 1); + is.read(reinterpret_cast(&p->subcategory_), 1); + is.read(reinterpret_cast(&p->userDefined_), 2); + is.read(reinterpret_cast(&p->year_), 1); + is.read(reinterpret_cast(&p->month_), 1); + is.read(reinterpret_cast(&p->torDay_), 1); + is.read(reinterpret_cast(&p->torHour_), 1); + is.read(reinterpret_cast(&p->torMinute_), 1); + is.read(reinterpret_cast(&p->numberOfDestinations_), 1); + + p->userDefined_ = ntohs(p->userDefined_); + + p->messageDestination_.resize(p->numberOfDestinations_); + + for (uint8_t d = 0; d < p->numberOfDestinations_; d++) + { + p->messageDestination_[d].resize(4); + is.read(p->messageDestination_[d].data(), 4); + } + + if (is.eof()) + { + BOOST_LOG_TRIVIAL(debug) << logPrefix_ << "Reached end of file"; + headerValid = false; + } + + return headerValid; +} + +} // namespace rpg +} // namespace wsr88d +} // namespace scwx diff --git a/wxdata/wxdata.cmake b/wxdata/wxdata.cmake index 524fb462..aa1d2521 100644 --- a/wxdata/wxdata.cmake +++ b/wxdata/wxdata.cmake @@ -46,7 +46,8 @@ set(SRC_WSR88D_RDA source/scwx/wsr88d/rda/clutter_filter_map.cpp source/scwx/wsr88d/rda/rda_adaptation_data.cpp source/scwx/wsr88d/rda/rda_status_data.cpp source/scwx/wsr88d/rda/volume_coverage_pattern_data.cpp) -set(HDR_WSR88D_RPG include/scwx/wsr88d/rpg/digital_precipitation_data_array_packet.hpp +set(HDR_WSR88D_RPG include/scwx/wsr88d/rpg/ccb_header.hpp + include/scwx/wsr88d/rpg/digital_precipitation_data_array_packet.hpp include/scwx/wsr88d/rpg/digital_radial_data_array_packet.hpp include/scwx/wsr88d/rpg/generic_data_packet.hpp include/scwx/wsr88d/rpg/hda_hail_symbol_packet.hpp @@ -74,7 +75,8 @@ set(HDR_WSR88D_RPG include/scwx/wsr88d/rpg/digital_precipitation_data_array_pack include/scwx/wsr88d/rpg/vector_arrow_data_packet.hpp include/scwx/wsr88d/rpg/wind_barb_data_packet.hpp include/scwx/wsr88d/rpg/wmo_header.hpp) -set(SRC_WSR88D_RPG source/scwx/wsr88d/rpg/digital_precipitation_data_array_packet.cpp +set(SRC_WSR88D_RPG source/scwx/wsr88d/rpg/ccb_header.cpp + source/scwx/wsr88d/rpg/digital_precipitation_data_array_packet.cpp source/scwx/wsr88d/rpg/digital_radial_data_array_packet.cpp source/scwx/wsr88d/rpg/generic_data_packet.cpp source/scwx/wsr88d/rpg/hda_hail_symbol_packet.cpp