From a32029cb3143395789f05ae54f1038de168212b4 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Thu, 10 Feb 2022 22:10:22 -0600 Subject: [PATCH] Nexrad File Factory --- test/data | 2 +- .../scwx/wsr88d/nexrad_file_factory.test.cpp | 57 ++++++++ test/test.cmake | 3 +- wxdata/include/scwx/wsr88d/ar2v_file.hpp | 4 +- wxdata/include/scwx/wsr88d/level3_file.hpp | 3 +- wxdata/include/scwx/wsr88d/nexrad_file.hpp | 35 +++++ .../scwx/wsr88d/nexrad_file_factory.hpp | 28 ++++ wxdata/source/scwx/wsr88d/ar2v_file.cpp | 66 +++++---- wxdata/source/scwx/wsr88d/nexrad_file.cpp | 22 +++ .../scwx/wsr88d/nexrad_file_factory.cpp | 125 ++++++++++++++++++ wxdata/wxdata.cmake | 8 +- 11 files changed, 322 insertions(+), 31 deletions(-) create mode 100644 test/source/scwx/wsr88d/nexrad_file_factory.test.cpp create mode 100644 wxdata/include/scwx/wsr88d/nexrad_file.hpp create mode 100644 wxdata/include/scwx/wsr88d/nexrad_file_factory.hpp create mode 100644 wxdata/source/scwx/wsr88d/nexrad_file.cpp create mode 100644 wxdata/source/scwx/wsr88d/nexrad_file_factory.cpp diff --git a/test/data b/test/data index a1b78dcb..45539a5b 160000 --- a/test/data +++ b/test/data @@ -1 +1 @@ -Subproject commit a1b78dcb40b85271270afd7dc9d88fe8594471b2 +Subproject commit 45539a5b0ecfc6baaaf7355b6b9542cc31ed9ecd diff --git a/test/source/scwx/wsr88d/nexrad_file_factory.test.cpp b/test/source/scwx/wsr88d/nexrad_file_factory.test.cpp new file mode 100644 index 00000000..5f1b0bbf --- /dev/null +++ b/test/source/scwx/wsr88d/nexrad_file_factory.test.cpp @@ -0,0 +1,57 @@ +#include +#include +#include + +#include + +#include + +namespace scwx +{ +namespace wsr88d +{ + +static const std::string logPrefix_ = + "[scwx::wsr88d::nexrad_file_factory.test] "; + +TEST(NexradFileFactory, Level2V06) +{ + std::string filename = std::string(SCWX_TEST_DATA_DIR) + + "/nexrad/level2/Level2_KLSX_20210527_1757.ar2v"; + + std::shared_ptr file = NexradFileFactory::Create(filename); + std::shared_ptr level2File = + std::dynamic_pointer_cast(file); + + EXPECT_NE(file, nullptr); + EXPECT_NE(level2File, nullptr); +} + +TEST(NexradFileFactory, Level2V06Gzip) +{ + std::string filename = std::string(SCWX_TEST_DATA_DIR) + + "/nexrad/level2/KLSX20130206_175044_V06.gz"; + + std::shared_ptr file = NexradFileFactory::Create(filename); + std::shared_ptr level2File = + std::dynamic_pointer_cast(file); + + EXPECT_NE(file, nullptr); + EXPECT_NE(level2File, nullptr); +} + +TEST(NexradFileFactory, Level3) +{ + std::string filename = std::string(SCWX_TEST_DATA_DIR) + + "/nexrad/level3/KLSX_SDUS23_N2QLSX_202112110250"; + + std::shared_ptr file = NexradFileFactory::Create(filename); + std::shared_ptr level3File = + std::dynamic_pointer_cast(file); + + EXPECT_NE(file, nullptr); + EXPECT_NE(level3File, nullptr); +} + +} // namespace wsr88d +} // namespace scwx diff --git a/test/test.cmake b/test/test.cmake index ae1805ca..a386927e 100644 --- a/test/test.cmake +++ b/test/test.cmake @@ -19,7 +19,8 @@ set(SRC_UTIL_TESTS source/scwx/util/rangebuf.test.cpp source/scwx/util/streams.test.cpp source/scwx/util/vectorbuf.test.cpp) set(SRC_WSR88D_TESTS source/scwx/wsr88d/ar2v_file.test.cpp - source/scwx/wsr88d/level3_file.test.cpp) + source/scwx/wsr88d/level3_file.test.cpp + source/scwx/wsr88d/nexrad_file_factory.test.cpp) add_executable(wxtest ${SRC_MAIN} ${SRC_AWIPS_TESTS} diff --git a/wxdata/include/scwx/wsr88d/ar2v_file.hpp b/wxdata/include/scwx/wsr88d/ar2v_file.hpp index 8ebeec4b..b79372df 100644 --- a/wxdata/include/scwx/wsr88d/ar2v_file.hpp +++ b/wxdata/include/scwx/wsr88d/ar2v_file.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -19,7 +20,7 @@ class Ar2vFileImpl; * the Archive II/User, Document Number 2620010H, published by the WSR-88D Radar * Operations Center. */ -class Ar2vFile +class Ar2vFile : public NexradFile { public: explicit Ar2vFile(); @@ -46,6 +47,7 @@ public: std::chrono::system_clock::time_point time) const; bool LoadFile(const std::string& filename); + bool LoadData(std::istream& is); private: std::unique_ptr p; diff --git a/wxdata/include/scwx/wsr88d/level3_file.hpp b/wxdata/include/scwx/wsr88d/level3_file.hpp index 90ef23ab..54b9ada1 100644 --- a/wxdata/include/scwx/wsr88d/level3_file.hpp +++ b/wxdata/include/scwx/wsr88d/level3_file.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -12,7 +13,7 @@ namespace wsr88d class Level3FileImpl; -class Level3File +class Level3File : public NexradFile { public: explicit Level3File(); diff --git a/wxdata/include/scwx/wsr88d/nexrad_file.hpp b/wxdata/include/scwx/wsr88d/nexrad_file.hpp new file mode 100644 index 00000000..7280f4d6 --- /dev/null +++ b/wxdata/include/scwx/wsr88d/nexrad_file.hpp @@ -0,0 +1,35 @@ +#pragma once + +#include +#include + +namespace scwx +{ +namespace wsr88d +{ + +class NexradFileImpl; + +class NexradFile +{ +protected: + explicit NexradFile(); + + NexradFile(const NexradFile&) = delete; + NexradFile& operator=(const NexradFile&) = delete; + + NexradFile(NexradFile&&) noexcept; + NexradFile& operator=(NexradFile&&) noexcept; + +public: + virtual ~NexradFile(); + + virtual bool LoadFile(const std::string& filename) = 0; + virtual bool LoadData(std::istream& is) = 0; + +private: + std::unique_ptr p; +}; + +} // namespace wsr88d +} // namespace scwx diff --git a/wxdata/include/scwx/wsr88d/nexrad_file_factory.hpp b/wxdata/include/scwx/wsr88d/nexrad_file_factory.hpp new file mode 100644 index 00000000..0f0ab5cb --- /dev/null +++ b/wxdata/include/scwx/wsr88d/nexrad_file_factory.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include + +namespace scwx +{ +namespace wsr88d +{ + +class NexradFileFactory +{ +private: + explicit NexradFileFactory() = delete; + ~NexradFileFactory() = delete; + + NexradFileFactory(const NexradFileFactory&) = delete; + NexradFileFactory& operator=(const NexradFileFactory&) = delete; + + NexradFileFactory(NexradFileFactory&&) noexcept = delete; + NexradFileFactory& operator=(NexradFileFactory&&) noexcept = delete; + +public: + static std::shared_ptr Create(const std::string& filename); + static std::shared_ptr Create(std::istream& is); +}; + +} // namespace wsr88d +} // namespace scwx diff --git a/wxdata/source/scwx/wsr88d/ar2v_file.cpp b/wxdata/source/scwx/wsr88d/ar2v_file.cpp index da16c52e..30b81753 100644 --- a/wxdata/source/scwx/wsr88d/ar2v_file.cpp +++ b/wxdata/source/scwx/wsr88d/ar2v_file.cpp @@ -37,7 +37,7 @@ public: void HandleMessage(std::shared_ptr& message); void IndexFile(); - void LoadLDMRecords(std::ifstream& f); + void LoadLDMRecords(std::istream& is); void ParseLDMRecords(); void ProcessRadarData(std::shared_ptr message); @@ -179,29 +179,40 @@ bool Ar2vFile::LoadFile(const std::string& filename) if (fileValid) { - // Read Volume Header Record - p->tapeFilename_.resize(9, ' '); - p->extensionNumber_.resize(3, ' '); - p->icao_.resize(4, ' '); - - f.read(&p->tapeFilename_[0], 9); - f.read(&p->extensionNumber_[0], 3); - f.read(reinterpret_cast(&p->julianDate_), 4); - f.read(reinterpret_cast(&p->milliseconds_), 4); - f.read(&p->icao_[0], 4); - - p->julianDate_ = ntohl(p->julianDate_); - p->milliseconds_ = ntohl(p->milliseconds_); + fileValid = LoadData(f); } - if (f.eof()) + return fileValid; +} + +bool Ar2vFile::LoadData(std::istream& is) +{ + BOOST_LOG_TRIVIAL(debug) << logPrefix_ << "Loading Data"; + + bool dataValid = true; + + // Read Volume Header Record + p->tapeFilename_.resize(9, ' '); + p->extensionNumber_.resize(3, ' '); + p->icao_.resize(4, ' '); + + is.read(&p->tapeFilename_[0], 9); + is.read(&p->extensionNumber_[0], 3); + is.read(reinterpret_cast(&p->julianDate_), 4); + is.read(reinterpret_cast(&p->milliseconds_), 4); + is.read(&p->icao_[0], 4); + + p->julianDate_ = ntohl(p->julianDate_); + p->milliseconds_ = ntohl(p->milliseconds_); + + if (is.eof()) { BOOST_LOG_TRIVIAL(warning) << logPrefix_ << "Could not read Volume Header Record\n"; - fileValid = false; + dataValid = false; } - if (fileValid) + if (dataValid) { BOOST_LOG_TRIVIAL(debug) << logPrefix_ << "Filename: " << p->tapeFilename_; @@ -212,27 +223,27 @@ bool Ar2vFile::LoadFile(const std::string& filename) << logPrefix_ << "Time: " << p->milliseconds_; BOOST_LOG_TRIVIAL(debug) << logPrefix_ << "ICAO: " << p->icao_; - p->LoadLDMRecords(f); + p->LoadLDMRecords(is); } p->IndexFile(); - return fileValid; + return dataValid; } -void Ar2vFileImpl::LoadLDMRecords(std::ifstream& f) +void Ar2vFileImpl::LoadLDMRecords(std::istream& is) { BOOST_LOG_TRIVIAL(debug) << logPrefix_ << "Loading LDM Records"; numRecords_ = 0; - while (f.peek() != EOF) + while (is.peek() != EOF) { - std::streampos startPosition = f.tellg(); + std::streampos startPosition = is.tellg(); int32_t controlWord = 0; size_t recordSize; - f.read(reinterpret_cast(&controlWord), 4); + is.read(reinterpret_cast(&controlWord), 4); controlWord = ntohl(controlWord); recordSize = std::abs(controlWord); @@ -240,8 +251,13 @@ void Ar2vFileImpl::LoadLDMRecords(std::ifstream& f) BOOST_LOG_TRIVIAL(trace) << logPrefix_ << "LDM Record Found: Size = " << recordSize << " bytes"; + if (recordSize == 0) + { + break; + } + boost::iostreams::filtering_streambuf in; - util::rangebuf r(f.rdbuf(), recordSize); + util::rangebuf r(is.rdbuf(), recordSize); in.push(boost::iostreams::bzip2_decompressor()); in.push(r); @@ -261,7 +277,7 @@ void Ar2vFileImpl::LoadLDMRecords(std::ifstream& f) BOOST_LOG_TRIVIAL(warning) << logPrefix_ << "Error decompressing record " << numRecords_; - f.seekg(startPosition + std::streampos(recordSize)); + is.seekg(startPosition + std::streampos(recordSize)); } ++numRecords_; diff --git a/wxdata/source/scwx/wsr88d/nexrad_file.cpp b/wxdata/source/scwx/wsr88d/nexrad_file.cpp new file mode 100644 index 00000000..819a6250 --- /dev/null +++ b/wxdata/source/scwx/wsr88d/nexrad_file.cpp @@ -0,0 +1,22 @@ +#include + +namespace scwx +{ +namespace wsr88d +{ + +class NexradFileImpl +{ +public: + explicit NexradFileImpl() {} + ~NexradFileImpl() = default; +}; + +NexradFile::NexradFile() : p(std::make_unique()) {} +NexradFile::~NexradFile() = default; + +NexradFile::NexradFile(NexradFile&&) noexcept = default; +NexradFile& NexradFile::operator=(NexradFile&&) noexcept = default; + +} // namespace wsr88d +} // namespace scwx diff --git a/wxdata/source/scwx/wsr88d/nexrad_file_factory.cpp b/wxdata/source/scwx/wsr88d/nexrad_file_factory.cpp new file mode 100644 index 00000000..08aa2b72 --- /dev/null +++ b/wxdata/source/scwx/wsr88d/nexrad_file_factory.cpp @@ -0,0 +1,125 @@ +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +namespace scwx +{ +namespace wsr88d +{ + +static const std::string logPrefix_ = "[scwx::wsr88d::nexrad_file_factory] "; + +std::shared_ptr +NexradFileFactory::Create(const std::string& filename) +{ + BOOST_LOG_TRIVIAL(debug) << logPrefix_ << "Create(" << filename << ")"; + + std::shared_ptr nexradFile = nullptr; + bool fileValid = true; + + std::ifstream f(filename, std::ios_base::in | std::ios_base::binary); + if (!f.good()) + { + BOOST_LOG_TRIVIAL(warning) + << logPrefix_ << "Could not open file for reading: " << filename; + fileValid = false; + } + + if (fileValid) + { + nexradFile = Create(f); + } + + return nexradFile; +} + +std::shared_ptr NexradFileFactory::Create(std::istream& is) +{ + std::shared_ptr message = nullptr; + + std::istream* pis = &is; + std::streampos pisBegin = is.tellg(); + std::stringstream ss; + std::string buffer; + bool dataValid; + + buffer.resize(4); + + is.read(buffer.data(), 4); + dataValid = is.good(); + is.seekg(pisBegin, std::ios_base::beg); + + if (dataValid && buffer.starts_with("\x1f\x8b")) + { + boost::iostreams::filtering_streambuf in; + in.push(boost::iostreams::gzip_decompressor()); + in.push(is); + + try + { + std::streamsize bytesCopied = boost::iostreams::copy(in, ss); + + pis = &ss; + pisBegin = ss.tellg(); + + ss.read(buffer.data(), 4); + dataValid = ss.good(); + ss.seekg(pisBegin, std::ios_base::beg); + + BOOST_LOG_TRIVIAL(trace) + << logPrefix_ << "Decompressed file = " << bytesCopied << " bytes"; + + if (!dataValid) + { + BOOST_LOG_TRIVIAL(warning) + << logPrefix_ << "Error reading decompressed stream"; + } + } + catch (const boost::iostreams::gzip_error& ex) + { + BOOST_LOG_TRIVIAL(warning) + << logPrefix_ << "Error decompressing file: " << ex.what(); + + dataValid = false; + } + } + else if (!dataValid) + { + BOOST_LOG_TRIVIAL(warning) << logPrefix_ << "Error reading file"; + } + + if (dataValid) + { + if (buffer.starts_with("AR2V")) + { + message = std::make_shared(); + } + else + { + message = std::make_shared(); + } + } + + if (message != nullptr) + { + dataValid = message->LoadData(*pis); + + if (!dataValid) + { + message = nullptr; + } + } + + return message; +} + +} // namespace wsr88d +} // namespace scwx diff --git a/wxdata/wxdata.cmake b/wxdata/wxdata.cmake index 20165e0f..1fe871db 100644 --- a/wxdata/wxdata.cmake +++ b/wxdata/wxdata.cmake @@ -41,9 +41,13 @@ set(SRC_UTIL source/scwx/util/rangebuf.cpp source/scwx/util/time.cpp source/scwx/util/vectorbuf.cpp) set(HDR_WSR88D include/scwx/wsr88d/ar2v_file.hpp - include/scwx/wsr88d/level3_file.hpp) + include/scwx/wsr88d/level3_file.hpp + include/scwx/wsr88d/nexrad_file.hpp + include/scwx/wsr88d/nexrad_file_factory.hpp) set(SRC_WSR88D source/scwx/wsr88d/ar2v_file.cpp - source/scwx/wsr88d/level3_file.cpp) + source/scwx/wsr88d/level3_file.cpp + source/scwx/wsr88d/nexrad_file.cpp + source/scwx/wsr88d/nexrad_file_factory.cpp) set(HDR_WSR88D_RDA include/scwx/wsr88d/rda/clutter_filter_map.hpp include/scwx/wsr88d/rda/digital_radar_data.hpp include/scwx/wsr88d/rda/level2_message.hpp