From 87af6479d6bf07189d76bc4e813b6e6b24d207ed Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 1 Feb 2025 00:25:00 -0600 Subject: [PATCH 01/67] Rewrite warnings provider to use HEAD requests instead of directory listing to find recent warnings --- .../scwx/qt/manager/text_event_manager.cpp | 46 ++- .../scwx/provider/warnings_provider.hpp | 4 +- .../scwx/provider/warnings_provider.cpp | 305 ++++++++++-------- 3 files changed, 208 insertions(+), 147 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp index e48ecd48..0a7c66f5 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp @@ -22,8 +22,10 @@ namespace manager static const std::string logPrefix_ = "scwx::qt::manager::text_event_manager"; static const auto logger_ = scwx::util::Logger::Create(logPrefix_); -static const std::string& kDefaultWarningsProviderUrl { - "https://warnings.allisonhouse.com"}; +static constexpr std::chrono::hours kInitialLoadHistoryDuration_ = + std::chrono::days {3}; +static constexpr std::chrono::hours kDefaultLoadHistoryDuration_ = + std::chrono::hours {1}; class TextEventManager::Impl { @@ -42,7 +44,9 @@ public: warningsProviderChangedCallbackUuid_ = generalSettings.warnings_provider().RegisterValueChangedCallback( - [this](const std::string& value) { + [this](const std::string& value) + { + loadHistoryDuration_ = kInitialLoadHistoryDuration_; warningsProvider_ = std::make_shared(value); }); @@ -94,6 +98,8 @@ public: std::shared_mutex textEventMutex_; std::shared_ptr warningsProvider_ {nullptr}; + std::chrono::hours loadHistoryDuration_ {kInitialLoadHistoryDuration_}; + std::chrono::sys_time prevLoadTime_ {}; boost::uuids::uuid warningsProviderChangedCallbackUuid_ {}; }; @@ -254,21 +260,33 @@ void TextEventManager::Impl::Refresh() std::shared_ptr warningsProvider = warningsProvider_; - // Update the file listing from the warnings provider - auto [newFiles, totalFiles] = warningsProvider->ListFiles(); + // Load updated files from the warnings provider + // Start time should default to: + // - 3 days of history for the first load + // - 1 hour of history for subsequent loads + // If the time jumps, we should attempt to load from no later than the + // previous load time + auto loadTime = + std::chrono::floor(std::chrono::system_clock::now()); + auto startTime = loadTime - loadHistoryDuration_; - if (newFiles > 0) + if (prevLoadTime_ != std::chrono::sys_time {}) { - // Load new files - auto updatedFiles = warningsProvider->LoadUpdatedFiles(); + startTime = std::min(startTime, prevLoadTime_); + } - // Handle messages - for (auto& file : updatedFiles) + auto updatedFiles = warningsProvider->LoadUpdatedFiles(startTime); + + // Store the load time and reset the load history duration + prevLoadTime_ = loadTime; + loadHistoryDuration_ = kDefaultLoadHistoryDuration_; + + // Handle messages + for (auto& file : updatedFiles) + { + for (auto& message : file->messages()) { - for (auto& message : file->messages()) - { - HandleMessage(message); - } + HandleMessage(message); } } diff --git a/wxdata/include/scwx/provider/warnings_provider.hpp b/wxdata/include/scwx/provider/warnings_provider.hpp index e519ec5d..140b671f 100644 --- a/wxdata/include/scwx/provider/warnings_provider.hpp +++ b/wxdata/include/scwx/provider/warnings_provider.hpp @@ -22,10 +22,8 @@ public: WarningsProvider(WarningsProvider&&) noexcept; WarningsProvider& operator=(WarningsProvider&&) noexcept; - std::pair - ListFiles(std::chrono::system_clock::time_point newerThan = {}); std::vector> - LoadUpdatedFiles(std::chrono::system_clock::time_point newerThan = {}); + LoadUpdatedFiles(std::chrono::sys_time newerThan = {}); private: class Impl; diff --git a/wxdata/source/scwx/provider/warnings_provider.cpp b/wxdata/source/scwx/provider/warnings_provider.cpp index 8cfe9b77..d506fc5c 100644 --- a/wxdata/source/scwx/provider/warnings_provider.cpp +++ b/wxdata/source/scwx/provider/warnings_provider.cpp @@ -1,9 +1,18 @@ -#include -#include -#include +// Prevent redefinition of __cpp_lib_format +#if defined(_MSC_VER) +# include +#endif -#include -#include +// Enable chrono formatters +#ifndef __cpp_lib_format +# define __cpp_lib_format 202110L +#endif + +#include +#include +#include + +#include #if defined(_MSC_VER) # pragma warning(push, 0) @@ -11,8 +20,6 @@ #define LIBXML_HTML_ENABLED #include -#include -#include #if (__cpp_lib_chrono < 201907L) # include @@ -35,13 +42,17 @@ class WarningsProvider::Impl public: struct FileInfoRecord { - std::chrono::system_clock::time_point startTime_ {}; - std::chrono::system_clock::time_point lastModified_ {}; - size_t size_ {}; - bool updated_ {}; + FileInfoRecord(const std::string& contentLength, + const std::string& lastModified) : + contentLengthStr_ {contentLength}, lastModifiedStr_ {lastModified} + { + } + + std::string contentLengthStr_ {}; + std::string lastModifiedStr_ {}; }; - typedef std::map WarningFileMap; + using WarningFileMap = std::map; explicit Impl(const std::string& baseUrl) : baseUrl_ {baseUrl}, files_ {}, filesMutex_ {} @@ -50,10 +61,13 @@ public: ~Impl() {} + bool UpdateFileRecord(const cpr::Response& response, + const std::string& filename); + std::string baseUrl_; - WarningFileMap files_; - std::shared_mutex filesMutex_; + WarningFileMap files_; + std::mutex filesMutex_; }; WarningsProvider::WarningsProvider(const std::string& baseUrl) : @@ -66,145 +80,176 @@ WarningsProvider::WarningsProvider(WarningsProvider&&) noexcept = default; WarningsProvider& WarningsProvider::operator=(WarningsProvider&&) noexcept = default; -std::pair -WarningsProvider::ListFiles(std::chrono::system_clock::time_point newerThan) +std::vector> +WarningsProvider::LoadUpdatedFiles( + std::chrono::sys_time startTime) { using namespace std::chrono; -#if (__cpp_lib_chrono < 201907L) +#if (__cpp_lib_chrono >= 201907L) + namespace date = std::chrono; + namespace df = std; +#else using namespace date; + namespace df = date; #endif - static constexpr LazyRE2 reWarningsFilename = { - "warnings_[0-9]{8}_[0-9]{2}.txt"}; - static const std::string dateTimeFormat {"warnings_%Y%m%d_%H.txt"}; - - logger_->trace("Listing files"); - - size_t updatedObjects = 0; - size_t totalObjects = 0; - - // Perform a directory listing - auto records = network::DirList(p->baseUrl_); - - // Sort records by filename - std::sort(records.begin(), - records.end(), - [](auto& a, auto& b) { return a.filename_ < b.filename_; }); - - // Filter warning records - auto warningRecords = - records | - std::views::filter( - [](auto& record) - { - return record.type_ == std::filesystem::file_type::regular && - RE2::FullMatch(record.filename_, *reWarningsFilename); - }); - - std::unique_lock lock(p->filesMutex_); - - Impl::WarningFileMap warningFileMap; - - // Store records - for (auto& record : warningRecords) - { - // Determine start time - std::chrono::sys_time startTime; - std::istringstream ssFilename {record.filename_}; - - ssFilename >> parse(dateTimeFormat, startTime); - - // If start time is valid - if (!ssFilename.fail()) - { - // Determine if the record should be marked updated - bool updated = true; - auto it = p->files_.find(record.filename_); - if (it != p->files_.cend()) - { - auto& existingRecord = it->second; - - updated = existingRecord.updated_ || - record.size_ != existingRecord.size_ || - record.mtime_ != existingRecord.lastModified_; - } - - // Update object counts, but only if newer than threshold - if (newerThan < startTime) - { - if (updated) - { - ++updatedObjects; - } - ++totalObjects; - } - - // Store record - warningFileMap.emplace( - std::piecewise_construct, - std::forward_as_tuple(record.filename_), - std::forward_as_tuple( - startTime, record.mtime_, record.size_, updated)); - } - } - - p->files_ = std::move(warningFileMap); - - return std::make_pair(updatedObjects, totalObjects); -} - -std::vector> -WarningsProvider::LoadUpdatedFiles( - std::chrono::system_clock::time_point newerThan) -{ - logger_->debug("Loading updated files"); - + std::vector< + std::pair, false>>> + asyncCallbacks; std::vector> updatedFiles; - std::vector> asyncResponses; + std::chrono::sys_time now = + std::chrono::floor(std::chrono::system_clock::now()); + std::chrono::sys_time currentHour = + (startTime != std::chrono::sys_time {}) ? + startTime : + now - std::chrono::hours {1}; - std::unique_lock lock(p->filesMutex_); + logger_->trace("Querying files newer than: {}", util::TimeString(startTime)); - // For each warning file - for (auto& record : p->files_) + while (currentHour <= now) { - // If file is updated, and time is later than the threshold - if (record.second.updated_ && newerThan < record.second.startTime_) - { - // Retrieve warning file - asyncResponses.emplace_back( - record.first, - cpr::GetAsync(cpr::Url {p->baseUrl_ + "/" + record.first})); + static constexpr std::string_view dateTimeFormat { + "warnings_{:%Y%m%d_%H}.txt"}; + const std::string filename = df::format(dateTimeFormat, currentHour); + const std::string url = p->baseUrl_ + "/" + filename; - // Clear updated flag - record.second.updated_ = false; - } + logger_->trace("HEAD request for file: {}", filename); + + asyncCallbacks.emplace_back( + filename, + cpr::HeadCallback( + [url, filename, this]( + cpr::Response headResponse) -> std::optional + { + if (headResponse.status_code == cpr::status::HTTP_OK) + { + bool updated = + p->UpdateFileRecord(headResponse, url); // TODO: filename + + if (updated) + { + logger_->trace("GET request for file: {}", filename); + return cpr::GetAsync(cpr::Url {url}); + } + } + else if (headResponse.status_code != cpr::status::HTTP_NOT_FOUND) + { + logger_->warn("HEAD request for file failed: {} ({})", + url, + headResponse.status_line); + } + + return std::nullopt; + }, + cpr::Url {url})); + + // Query the next hour + currentHour += 1h; } - lock.unlock(); - - // Wait for warning files to load - for (auto& asyncResponse : asyncResponses) + for (auto& asyncCallback : asyncCallbacks) { - cpr::Response response = asyncResponse.second.get(); - if (response.status_code == cpr::status::HTTP_OK) - { - logger_->debug("Loading file: {}", asyncResponse.first); + auto& filename = asyncCallback.first; + auto& callback = asyncCallback.second; - // Load file - std::shared_ptr textProductFile { - std::make_shared()}; - std::istringstream responseBody {response.text}; - if (textProductFile->LoadData(responseBody)) + if (callback.valid()) + { + // Wait for futures to complete + callback.wait(); + auto asyncResponse = callback.get(); + + if (asyncResponse.has_value()) { - updatedFiles.push_back(textProductFile); + auto response = asyncResponse.value().get(); + + if (response.status_code == cpr::status::HTTP_OK) + { + logger_->debug("Loading file: {}", filename); + + // Load file + std::shared_ptr textProductFile { + std::make_shared()}; + std::istringstream responseBody {response.text}; + if (textProductFile->LoadData(responseBody)) + { + updatedFiles.push_back(textProductFile); + } + } + else + { + logger_->warn("Could not load file: {} ({})", + filename, + response.status_line); + } } } + else + { + logger_->error("Invalid future state"); + } } return updatedFiles; } +bool WarningsProvider::Impl::UpdateFileRecord(const cpr::Response& response, + const std::string& filename) +{ + bool updated = false; + + auto contentLengthIt = response.header.find("Content-Length"); + auto lastModifiedIt = response.header.find("Last-Modified"); + + std::string contentLength {}; + std::string lastModified {}; + + if (contentLengthIt != response.header.cend()) + { + contentLength = contentLengthIt->second; + } + if (lastModifiedIt != response.header.cend()) + { + lastModified = lastModifiedIt->second; + } + + std::unique_lock lock(filesMutex_); + + auto it = files_.find(filename); + if (it != files_.cend()) + { + auto& existingRecord = it->second; + + // If the size or last modified changes, request an update + + if (!contentLength.empty() && + contentLength != existingRecord.contentLengthStr_) + { + // Size changed + existingRecord.contentLengthStr_ = contentLengthIt->second; + updated = true; + } + else if (!lastModified.empty() && + lastModified != existingRecord.lastModifiedStr_) + { + // Last modified changed + existingRecord.lastModifiedStr_ = lastModifiedIt->second; + updated = true; + } + } + else + { + // File not found + files_.emplace(std::piecewise_construct, + std::forward_as_tuple(filename), + std::forward_as_tuple(contentLength, lastModified)); + updated = true; + } + + return updated; +} + } // namespace provider } // namespace scwx From 549f7ece6156e9021d12cc27dcd583cb3b3aad55 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 1 Feb 2025 01:24:59 -0600 Subject: [PATCH 02/67] Fixing warnings provider test --- .../scwx/provider/warnings_provider.test.cpp | 44 ++++--------------- 1 file changed, 9 insertions(+), 35 deletions(-) diff --git a/test/source/scwx/provider/warnings_provider.test.cpp b/test/source/scwx/provider/warnings_provider.test.cpp index 78ef9b95..de315b4b 100644 --- a/test/source/scwx/provider/warnings_provider.test.cpp +++ b/test/source/scwx/provider/warnings_provider.test.cpp @@ -13,53 +13,27 @@ static const std::string& kAlternateUrl {"https://warnings.cod.edu"}; class WarningsProviderTest : public testing::TestWithParam { }; -TEST_P(WarningsProviderTest, ListFiles) -{ - WarningsProvider provider(GetParam()); - - auto [newObjects, totalObjects] = provider.ListFiles(); - - // No objects, skip test - if (totalObjects == 0) - { - GTEST_SKIP(); - } - - EXPECT_GT(newObjects, 0); - EXPECT_GT(totalObjects, 0); - EXPECT_EQ(newObjects, totalObjects); -} TEST_P(WarningsProviderTest, LoadUpdatedFiles) { WarningsProvider provider(GetParam()); - auto [newObjects, totalObjects] = provider.ListFiles(); - auto updatedFiles = provider.LoadUpdatedFiles(); + std::chrono::sys_time now = + std::chrono::floor(std::chrono::system_clock::now()); + std::chrono::sys_time startTime = + now - std::chrono::days {3}; + + auto updatedFiles = provider.LoadUpdatedFiles(startTime); // No objects, skip test - if (totalObjects == 0) + if (updatedFiles.empty()) { GTEST_SKIP(); } - EXPECT_GT(newObjects, 0); - EXPECT_GT(totalObjects, 0); - EXPECT_EQ(newObjects, totalObjects); - EXPECT_EQ(updatedFiles.size(), newObjects); + EXPECT_GT(updatedFiles.size(), 0); - auto [newObjects2, totalObjects2] = provider.ListFiles(); - auto updatedFiles2 = provider.LoadUpdatedFiles(); - - // There should be no more than 2 updated warnings files since the last query - // (assumption that the previous newest file was updated, and a new file was - // created on the hour) - EXPECT_LE(newObjects2, 2); - EXPECT_EQ(updatedFiles2.size(), newObjects2); - - // The total number of objects may have changed, since the oldest file could - // have dropped off the list - EXPECT_GT(totalObjects2, 0); + auto updatedFiles2 = provider.LoadUpdatedFiles(); } INSTANTIATE_TEST_SUITE_P(WarningsProvider, From a8da035566fe87af0a526dcf00a48ac11957d63e Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 1 Feb 2025 11:33:30 -0600 Subject: [PATCH 03/67] Warnings provider clang-tidy fixes --- .../scwx/provider/warnings_provider.hpp | 7 ++--- .../scwx/provider/warnings_provider.cpp | 26 +++++++++---------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/wxdata/include/scwx/provider/warnings_provider.hpp b/wxdata/include/scwx/provider/warnings_provider.hpp index 140b671f..0d14d258 100644 --- a/wxdata/include/scwx/provider/warnings_provider.hpp +++ b/wxdata/include/scwx/provider/warnings_provider.hpp @@ -2,9 +2,7 @@ #include -namespace scwx -{ -namespace provider +namespace scwx::provider { /** @@ -30,5 +28,4 @@ private: std::unique_ptr p; }; -} // namespace provider -} // namespace scwx +} // namespace scwx::provider diff --git a/wxdata/source/scwx/provider/warnings_provider.cpp b/wxdata/source/scwx/provider/warnings_provider.cpp index d506fc5c..a6b1a37d 100644 --- a/wxdata/source/scwx/provider/warnings_provider.cpp +++ b/wxdata/source/scwx/provider/warnings_provider.cpp @@ -29,9 +29,7 @@ # pragma warning(pop) #endif -namespace scwx -{ -namespace provider +namespace scwx::provider { static const std::string logPrefix_ = "scwx::provider::warnings_provider"; @@ -42,9 +40,9 @@ class WarningsProvider::Impl public: struct FileInfoRecord { - FileInfoRecord(const std::string& contentLength, - const std::string& lastModified) : - contentLengthStr_ {contentLength}, lastModifiedStr_ {lastModified} + FileInfoRecord(std::string contentLength, std::string lastModified) : + contentLengthStr_ {std::move(contentLength)}, + lastModifiedStr_ {std::move(lastModified)} { } @@ -54,12 +52,16 @@ public: using WarningFileMap = std::map; - explicit Impl(const std::string& baseUrl) : - baseUrl_ {baseUrl}, files_ {}, filesMutex_ {} + explicit Impl(std::string baseUrl) : + baseUrl_ {std::move(baseUrl)}, files_ {}, filesMutex_ {} { } - ~Impl() {} + ~Impl() = default; + Impl(const Impl&) = delete; + Impl& operator=(const Impl&) = delete; + Impl(const Impl&&) = delete; + Impl& operator=(const Impl&&) = delete; bool UpdateFileRecord(const cpr::Response& response, const std::string& filename); @@ -87,8 +89,7 @@ WarningsProvider::LoadUpdatedFiles( using namespace std::chrono; #if (__cpp_lib_chrono >= 201907L) - namespace date = std::chrono; - namespace df = std; + namespace df = std; #else using namespace date; namespace df = date; @@ -251,5 +252,4 @@ bool WarningsProvider::Impl::UpdateFileRecord(const cpr::Response& response, return updated; } -} // namespace provider -} // namespace scwx +} // namespace scwx::provider From d34cd6847103024ddb06e732a5ba3abbef37b3ca Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 1 Feb 2025 11:33:43 -0600 Subject: [PATCH 04/67] Warnings provider gcc fixes --- wxdata/source/scwx/provider/warnings_provider.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/wxdata/source/scwx/provider/warnings_provider.cpp b/wxdata/source/scwx/provider/warnings_provider.cpp index a6b1a37d..560cb307 100644 --- a/wxdata/source/scwx/provider/warnings_provider.cpp +++ b/wxdata/source/scwx/provider/warnings_provider.cpp @@ -90,9 +90,14 @@ WarningsProvider::LoadUpdatedFiles( #if (__cpp_lib_chrono >= 201907L) namespace df = std; + + static constexpr std::string_view kDateTimeFormat { + "warnings_{:%Y%m%d_%H}.txt"}; #else using namespace date; namespace df = date; + +# define kDateTimeFormat "warnings_%Y%m%d_%H.txt" #endif std::vector< @@ -112,9 +117,7 @@ WarningsProvider::LoadUpdatedFiles( while (currentHour <= now) { - static constexpr std::string_view dateTimeFormat { - "warnings_{:%Y%m%d_%H}.txt"}; - const std::string filename = df::format(dateTimeFormat, currentHour); + const std::string filename = df::format(kDateTimeFormat, currentHour); const std::string url = p->baseUrl_ + "/" + filename; logger_->trace("HEAD request for file: {}", filename); From 895e760fee524d40896ab6bc9ecdee2112c0595b Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 1 Feb 2025 15:50:28 -0600 Subject: [PATCH 05/67] Create IemWarningsProvider class for archive warnings --- .../scwx/provider/iem_warnings_provider.hpp | 28 ++++++++++++++++++ .../scwx/provider/iem_warnings_provider.cpp | 29 +++++++++++++++++++ wxdata/wxdata.cmake | 2 ++ 3 files changed, 59 insertions(+) create mode 100644 wxdata/include/scwx/provider/iem_warnings_provider.hpp create mode 100644 wxdata/source/scwx/provider/iem_warnings_provider.cpp diff --git a/wxdata/include/scwx/provider/iem_warnings_provider.hpp b/wxdata/include/scwx/provider/iem_warnings_provider.hpp new file mode 100644 index 00000000..64c5e96f --- /dev/null +++ b/wxdata/include/scwx/provider/iem_warnings_provider.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include + +namespace scwx::provider +{ + +/** + * @brief Warnings Provider + */ +class IemWarningsProvider +{ +public: + explicit IemWarningsProvider(); + ~IemWarningsProvider(); + + IemWarningsProvider(const IemWarningsProvider&) = delete; + IemWarningsProvider& operator=(const IemWarningsProvider&) = delete; + + IemWarningsProvider(IemWarningsProvider&&) noexcept; + IemWarningsProvider& operator=(IemWarningsProvider&&) noexcept; + +private: + class Impl; + std::unique_ptr p; +}; + +} // namespace scwx::provider diff --git a/wxdata/source/scwx/provider/iem_warnings_provider.cpp b/wxdata/source/scwx/provider/iem_warnings_provider.cpp new file mode 100644 index 00000000..7c780816 --- /dev/null +++ b/wxdata/source/scwx/provider/iem_warnings_provider.cpp @@ -0,0 +1,29 @@ +#include +#include + +namespace scwx::provider +{ + +static const std::string logPrefix_ = "scwx::provider::iem_warnings_provider"; +static const auto logger_ = util::Logger::Create(logPrefix_); + +class IemWarningsProvider::Impl +{ +public: + explicit Impl() = default; + ~Impl() = default; + Impl(const Impl&) = delete; + Impl& operator=(const Impl&) = delete; + Impl(const Impl&&) = delete; + Impl& operator=(const Impl&&) = delete; +}; + +IemWarningsProvider::IemWarningsProvider() : p(std::make_unique()) {} +IemWarningsProvider::~IemWarningsProvider() = default; + +IemWarningsProvider::IemWarningsProvider(IemWarningsProvider&&) noexcept = + default; +IemWarningsProvider& +IemWarningsProvider::operator=(IemWarningsProvider&&) noexcept = default; + +} // namespace scwx::provider diff --git a/wxdata/wxdata.cmake b/wxdata/wxdata.cmake index 94b0e3a7..77e65c58 100644 --- a/wxdata/wxdata.cmake +++ b/wxdata/wxdata.cmake @@ -61,12 +61,14 @@ set(SRC_NETWORK source/scwx/network/cpr.cpp set(HDR_PROVIDER include/scwx/provider/aws_level2_data_provider.hpp include/scwx/provider/aws_level3_data_provider.hpp include/scwx/provider/aws_nexrad_data_provider.hpp + include/scwx/provider/iem_warnings_provider.hpp include/scwx/provider/nexrad_data_provider.hpp include/scwx/provider/nexrad_data_provider_factory.hpp include/scwx/provider/warnings_provider.hpp) set(SRC_PROVIDER source/scwx/provider/aws_level2_data_provider.cpp source/scwx/provider/aws_level3_data_provider.cpp source/scwx/provider/aws_nexrad_data_provider.cpp + source/scwx/provider/iem_warnings_provider.cpp source/scwx/provider/nexrad_data_provider.cpp source/scwx/provider/nexrad_data_provider_factory.cpp source/scwx/provider/warnings_provider.cpp) From 9f33189c18f5b277f752b0ca75dd85e52fe6c24c Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 1 Feb 2025 18:21:41 -0600 Subject: [PATCH 06/67] Refactor json utility to wxdata, add ReadJsonString function --- scwx-qt/source/scwx/qt/config/radar_site.cpp | 2 +- .../source/scwx/qt/manager/marker_manager.cpp | 26 +-- .../scwx/qt/manager/placefile_manager.cpp | 6 +- .../scwx/qt/manager/settings_manager.cpp | 2 +- .../source/scwx/qt/manager/update_manager.cpp | 28 +-- scwx-qt/source/scwx/qt/model/layer_model.cpp | 2 +- .../source/scwx/qt/model/radar_site_model.cpp | 6 +- scwx-qt/source/scwx/qt/util/json.cpp | 178 +--------------- scwx-qt/source/scwx/qt/util/json.hpp | 7 +- wxdata/include/scwx/util/json.hpp | 15 ++ wxdata/source/scwx/util/json.cpp | 199 ++++++++++++++++++ wxdata/wxdata.cmake | 2 + 12 files changed, 245 insertions(+), 228 deletions(-) create mode 100644 wxdata/include/scwx/util/json.hpp create mode 100644 wxdata/source/scwx/util/json.cpp diff --git a/scwx-qt/source/scwx/qt/config/radar_site.cpp b/scwx-qt/source/scwx/qt/config/radar_site.cpp index 5c1dba2e..69815636 100644 --- a/scwx-qt/source/scwx/qt/config/radar_site.cpp +++ b/scwx-qt/source/scwx/qt/config/radar_site.cpp @@ -245,7 +245,7 @@ size_t RadarSite::ReadConfig(const std::string& path) bool dataValid = true; size_t sitesAdded = 0; - boost::json::value j = util::json::ReadJsonFile(path); + boost::json::value j = util::json::ReadJsonQFile(path); dataValid = j.is_array(); diff --git a/scwx-qt/source/scwx/qt/manager/marker_manager.cpp b/scwx-qt/source/scwx/qt/manager/marker_manager.cpp index 952dea44..ea21b211 100644 --- a/scwx-qt/source/scwx/qt/manager/marker_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/marker_manager.cpp @@ -1,10 +1,10 @@ #include #include #include -#include #include #include #include +#include #include #include @@ -62,7 +62,7 @@ public: bool markerFileRead_ {false}; - void InitalizeIds(); + void InitalizeIds(); types::MarkerId NewId(); types::MarkerId lastId_ {0}; }; @@ -70,15 +70,9 @@ public: class MarkerManager::Impl::MarkerRecord { public: - MarkerRecord(const types::MarkerInfo& info) : - markerInfo_ {info} - { - } + MarkerRecord(const types::MarkerInfo& info) : markerInfo_ {info} {} - const types::MarkerInfo& toMarkerInfo() - { - return markerInfo_; - } + const types::MarkerInfo& toMarkerInfo() { return markerInfo_; } types::MarkerInfo markerInfo_; @@ -175,7 +169,7 @@ void MarkerManager::Impl::ReadMarkerSettings() // Determine if marker settings exists if (std::filesystem::exists(markerSettingsPath_)) { - markerJson = util::json::ReadJsonFile(markerSettingsPath_); + markerJson = scwx::util::json::ReadJsonFile(markerSettingsPath_); } if (markerJson != nullptr && markerJson.is_array()) @@ -224,8 +218,8 @@ void MarkerManager::Impl::WriteMarkerSettings() logger_->info("Saving location marker settings"); const std::shared_lock lock(markerRecordLock_); - auto markerJson = boost::json::value_from(markerRecords_); - util::json::WriteJsonFile(markerSettingsPath_, markerJson); + auto markerJson = boost::json::value_from(markerRecords_); + scwx::util::json::WriteJsonFile(markerSettingsPath_, markerJson); } std::shared_ptr @@ -357,10 +351,11 @@ types::MarkerId MarkerManager::add_marker(const types::MarkerInfo& marker) types::MarkerId id; { const std::unique_lock lock(p->markerRecordLock_); - id = p->NewId(); + id = p->NewId(); size_t index = p->markerRecords_.size(); p->idToIndex_.emplace(id, index); - p->markerRecords_.emplace_back(std::make_shared(marker)); + p->markerRecords_.emplace_back( + std::make_shared(marker)); p->markerRecords_[index]->markerInfo_.id = id; add_icon(marker.iconName); @@ -499,7 +494,6 @@ void MarkerManager::set_marker_settings_path(const std::string& path) p->markerSettingsPath_ = path; } - std::shared_ptr MarkerManager::Instance() { static std::weak_ptr markerManagerReference_ {}; diff --git a/scwx-qt/source/scwx/qt/manager/placefile_manager.cpp b/scwx-qt/source/scwx/qt/manager/placefile_manager.cpp index a6158773..d85f9e40 100644 --- a/scwx-qt/source/scwx/qt/manager/placefile_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/placefile_manager.cpp @@ -2,10 +2,10 @@ #include #include #include -#include #include #include #include +#include #include #include @@ -385,7 +385,7 @@ void PlacefileManager::Impl::ReadPlacefileSettings() // Determine if placefile settings exists if (std::filesystem::exists(placefileSettingsPath_)) { - placefileJson = util::json::ReadJsonFile(placefileSettingsPath_); + placefileJson = scwx::util::json::ReadJsonFile(placefileSettingsPath_); } // If placefile settings was successfully read @@ -428,7 +428,7 @@ void PlacefileManager::Impl::WritePlacefileSettings() std::shared_lock lock {placefileRecordLock_}; auto placefileJson = boost::json::value_from(placefileRecords_); - util::json::WriteJsonFile(placefileSettingsPath_, placefileJson); + scwx::util::json::WriteJsonFile(placefileSettingsPath_, placefileJson); } void PlacefileManager::SetRadarSite( diff --git a/scwx-qt/source/scwx/qt/manager/settings_manager.cpp b/scwx-qt/source/scwx/qt/manager/settings_manager.cpp index 5b2e9cbb..a47428bb 100644 --- a/scwx-qt/source/scwx/qt/manager/settings_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/settings_manager.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include diff --git a/scwx-qt/source/scwx/qt/manager/update_manager.cpp b/scwx-qt/source/scwx/qt/manager/update_manager.cpp index d21068cb..5910fcaf 100644 --- a/scwx-qt/source/scwx/qt/manager/update_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/update_manager.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -29,8 +30,7 @@ public: ~Impl() {} - static std::string GetVersionString(const std::string& releaseName); - static boost::json::value ParseResponseText(const std::string& s); + static std::string GetVersionString(const std::string& releaseName); size_t PopulateReleases(); size_t AddReleases(const boost::json::value& json); @@ -70,28 +70,6 @@ UpdateManager::Impl::GetVersionString(const std::string& releaseName) return versionString; } -boost::json::value UpdateManager::Impl::ParseResponseText(const std::string& s) -{ - boost::json::stream_parser p; - boost::system::error_code ec; - - p.write(s, ec); - if (ec) - { - logger_->warn("{}", ec.message()); - return nullptr; - } - - p.finish(ec); - if (ec) - { - logger_->warn("{}", ec.message()); - return nullptr; - } - - return p.release(); -} - bool UpdateManager::CheckForUpdates(const std::string& currentVersion) { std::unique_lock lock(p->updateMutex_); @@ -148,7 +126,7 @@ size_t UpdateManager::Impl::PopulateReleases() // Successful REST API query if (r.status_code == 200) { - boost::json::value json = Impl::ParseResponseText(r.text); + boost::json::value json = util::json::ReadJsonString(r.text); if (json == nullptr) { logger_->warn("Response not JSON: {}", r.header["content-type"]); diff --git a/scwx-qt/source/scwx/qt/model/layer_model.cpp b/scwx-qt/source/scwx/qt/model/layer_model.cpp index 2f1b8a9d..6be8eb9d 100644 --- a/scwx-qt/source/scwx/qt/model/layer_model.cpp +++ b/scwx-qt/source/scwx/qt/model/layer_model.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include #include diff --git a/scwx-qt/source/scwx/qt/model/radar_site_model.cpp b/scwx-qt/source/scwx/qt/model/radar_site_model.cpp index e1a593d7..f131a1cd 100644 --- a/scwx-qt/source/scwx/qt/model/radar_site_model.cpp +++ b/scwx-qt/source/scwx/qt/model/radar_site_model.cpp @@ -4,8 +4,8 @@ #include #include #include -#include #include +#include #include #include @@ -117,7 +117,7 @@ void RadarSiteModelImpl::ReadPresets() // Determine if presets exists if (std::filesystem::exists(presetsPath_)) { - presetsJson = util::json::ReadJsonFile(presetsPath_); + presetsJson = scwx::util::json::ReadJsonFile(presetsPath_); } // If presets was successfully read @@ -160,7 +160,7 @@ void RadarSiteModelImpl::WritePresets() logger_->info("Saving presets"); auto presetsJson = boost::json::value_from(presets_); - util::json::WriteJsonFile(presetsPath_, presetsJson); + scwx::util::json::WriteJsonFile(presetsPath_, presetsJson); } int RadarSiteModel::rowCount(const QModelIndex& parent) const diff --git a/scwx-qt/source/scwx/qt/util/json.cpp b/scwx-qt/source/scwx/qt/util/json.cpp index 7bf0d23a..d508a224 100644 --- a/scwx-qt/source/scwx/qt/util/json.cpp +++ b/scwx-qt/source/scwx/qt/util/json.cpp @@ -1,41 +1,19 @@ #include +#include #include -#include - -#include -#include #include #include -namespace scwx -{ -namespace qt -{ -namespace util -{ -namespace json +namespace scwx::qt::util::json { static const std::string logPrefix_ = "scwx::qt::util::json"; static const auto logger_ = scwx::util::Logger::Create(logPrefix_); -/* Adapted from: - * https://www.boost.org/doc/libs/1_77_0/libs/json/doc/html/json/examples.html#json.examples.pretty - * - * Copyright (c) 2019, 2020 Vinnie Falco - * Copyright (c) 2020 Krystian Stasiowski - * Distributed under the Boost Software License, Version 1.0. (See - * http://www.boost.org/LICENSE_1_0.txt) - */ -static void PrettyPrintJson(std::ostream& os, - boost::json::value const& jv, - std::string* indent = nullptr); - static boost::json::value ReadJsonFile(QFile& file); -static boost::json::value ReadJsonStream(std::istream& is); -boost::json::value ReadJsonFile(const std::string& path) +boost::json::value ReadJsonQFile(const std::string& path) { boost::json::value json; @@ -46,8 +24,7 @@ boost::json::value ReadJsonFile(const std::string& path) } else { - std::ifstream ifs {path}; - json = ReadJsonStream(ifs); + json = ::scwx::util::json::ReadJsonFile(path); } return json; @@ -65,7 +42,7 @@ static boost::json::value ReadJsonFile(QFile& file) std::string jsonSource = jsonStream.readAll().toStdString(); std::istringstream is {jsonSource}; - json = ReadJsonStream(is); + json = ::scwx::util::json::ReadJsonStream(is); file.close(); } @@ -78,147 +55,4 @@ static boost::json::value ReadJsonFile(QFile& file) return json; } -static boost::json::value ReadJsonStream(std::istream& is) -{ - std::string line; - - boost::json::stream_parser p; - boost::system::error_code ec; - - while (std::getline(is, line)) - { - p.write(line, ec); - if (ec) - { - logger_->warn("{}", ec.message()); - return nullptr; - } - } - - p.finish(ec); - if (ec) - { - logger_->warn("{}", ec.message()); - return nullptr; - } - - return p.release(); -} - -void WriteJsonFile(const std::string& path, - const boost::json::value& json, - bool prettyPrint) -{ - std::ofstream ofs {path}; - - if (!ofs.is_open()) - { - logger_->warn("Cannot write JSON file: \"{}\"", path); - } - else - { - if (prettyPrint) - { - PrettyPrintJson(ofs, json); - } - else - { - ofs << json; - } - ofs.close(); - } -} - -static void PrettyPrintJson(std::ostream& os, - boost::json::value const& jv, - std::string* indent) -{ - std::string indent_; - if (!indent) - indent = &indent_; - switch (jv.kind()) - { - case boost::json::kind::object: - { - os << "{\n"; - indent->append(4, ' '); - auto const& obj = jv.get_object(); - if (!obj.empty()) - { - auto it = obj.begin(); - for (;;) - { - os << *indent << boost::json::serialize(it->key()) << " : "; - PrettyPrintJson(os, it->value(), indent); - if (++it == obj.end()) - break; - os << ",\n"; - } - } - os << "\n"; - indent->resize(indent->size() - 4); - os << *indent << "}"; - break; - } - - case boost::json::kind::array: - { - os << "[\n"; - indent->append(4, ' '); - auto const& arr = jv.get_array(); - if (!arr.empty()) - { - auto it = arr.begin(); - for (;;) - { - os << *indent; - PrettyPrintJson(os, *it, indent); - if (++it == arr.end()) - break; - os << ",\n"; - } - } - os << "\n"; - indent->resize(indent->size() - 4); - os << *indent << "]"; - break; - } - - case boost::json::kind::string: - { - os << boost::json::serialize(jv.get_string()); - break; - } - - case boost::json::kind::uint64: - os << jv.get_uint64(); - break; - - case boost::json::kind::int64: - os << jv.get_int64(); - break; - - case boost::json::kind::double_: - os << jv.get_double(); - break; - - case boost::json::kind::bool_: - if (jv.get_bool()) - os << "true"; - else - os << "false"; - break; - - case boost::json::kind::null: - os << "null"; - break; - } - - if (indent->empty()) - os << "\n"; -} - -} // namespace json -} // namespace util -} // namespace qt -} // namespace scwx +} // namespace scwx::qt::util::json diff --git a/scwx-qt/source/scwx/qt/util/json.hpp b/scwx-qt/source/scwx/qt/util/json.hpp index bbf497f4..9dd09810 100644 --- a/scwx-qt/source/scwx/qt/util/json.hpp +++ b/scwx-qt/source/scwx/qt/util/json.hpp @@ -1,7 +1,5 @@ #pragma once -#include - #include namespace scwx @@ -13,10 +11,7 @@ namespace util namespace json { -boost::json::value ReadJsonFile(const std::string& path); -void WriteJsonFile(const std::string& path, - const boost::json::value& json, - bool prettyPrint = true); +boost::json::value ReadJsonQFile(const std::string& path); } // namespace json } // namespace util diff --git a/wxdata/include/scwx/util/json.hpp b/wxdata/include/scwx/util/json.hpp new file mode 100644 index 00000000..ab836edc --- /dev/null +++ b/wxdata/include/scwx/util/json.hpp @@ -0,0 +1,15 @@ +#pragma once + +#include + +namespace scwx::util::json +{ + +boost::json::value ReadJsonFile(const std::string& path); +boost::json::value ReadJsonStream(std::istream& is); +boost::json::value ReadJsonString(std::string_view sv); +void WriteJsonFile(const std::string& path, + const boost::json::value& json, + bool prettyPrint = true); + +} // namespace scwx::util::json diff --git a/wxdata/source/scwx/util/json.cpp b/wxdata/source/scwx/util/json.cpp new file mode 100644 index 00000000..b8f51507 --- /dev/null +++ b/wxdata/source/scwx/util/json.cpp @@ -0,0 +1,199 @@ +#include +#include + +#include + +#include +#include + +namespace scwx::util::json +{ + +static const std::string logPrefix_ = "scwx::util::json"; +static const auto logger_ = scwx::util::Logger::Create(logPrefix_); + +/* Adapted from: + * https://www.boost.org/doc/libs/1_77_0/libs/json/doc/html/json/examples.html#json.examples.pretty + * + * Copyright (c) 2019, 2020 Vinnie Falco + * Copyright (c) 2020 Krystian Stasiowski + * Distributed under the Boost Software License, Version 1.0. (See + * http://www.boost.org/LICENSE_1_0.txt) + */ +static void PrettyPrintJson(std::ostream& os, + boost::json::value const& jv, + std::string* indent = nullptr); + +boost::json::value ReadJsonFile(const std::string& path) +{ + boost::json::value json; + + std::ifstream ifs {path}; + json = ReadJsonStream(ifs); + + return json; +} + +boost::json::value ReadJsonStream(std::istream& is) +{ + std::string line; + + boost::json::stream_parser p; + boost::system::error_code ec; + + while (std::getline(is, line)) + { + p.write(line, ec); + if (ec) + { + logger_->warn("{}", ec.message()); + return nullptr; + } + } + + p.finish(ec); + if (ec) + { + logger_->warn("{}", ec.message()); + return nullptr; + } + + return p.release(); +} + +boost::json::value ReadJsonString(std::string_view sv) +{ + boost::json::stream_parser p; + boost::system::error_code ec; + + p.write(sv, ec); + if (ec) + { + logger_->warn("{}", ec.message()); + return nullptr; + } + + p.finish(ec); + if (ec) + { + logger_->warn("{}", ec.message()); + return nullptr; + } + + return p.release(); +} + +void WriteJsonFile(const std::string& path, + const boost::json::value& json, + bool prettyPrint) +{ + std::ofstream ofs {path}; + + if (!ofs.is_open()) + { + logger_->warn("Cannot write JSON file: \"{}\"", path); + } + else + { + if (prettyPrint) + { + PrettyPrintJson(ofs, json); + } + else + { + ofs << json; + } + ofs.close(); + } +} + +static void PrettyPrintJson(std::ostream& os, + boost::json::value const& jv, + std::string* indent) +{ + std::string indent_; + if (!indent) + indent = &indent_; + switch (jv.kind()) + { + case boost::json::kind::object: + { + os << "{\n"; + indent->append(4, ' '); + auto const& obj = jv.get_object(); + if (!obj.empty()) + { + auto it = obj.begin(); + for (;;) + { + os << *indent << boost::json::serialize(it->key()) << " : "; + PrettyPrintJson(os, it->value(), indent); + if (++it == obj.end()) + break; + os << ",\n"; + } + } + os << "\n"; + indent->resize(indent->size() - 4); + os << *indent << "}"; + break; + } + + case boost::json::kind::array: + { + os << "[\n"; + indent->append(4, ' '); + auto const& arr = jv.get_array(); + if (!arr.empty()) + { + auto it = arr.begin(); + for (;;) + { + os << *indent; + PrettyPrintJson(os, *it, indent); + if (++it == arr.end()) + break; + os << ",\n"; + } + } + os << "\n"; + indent->resize(indent->size() - 4); + os << *indent << "]"; + break; + } + + case boost::json::kind::string: + { + os << boost::json::serialize(jv.get_string()); + break; + } + + case boost::json::kind::uint64: + os << jv.get_uint64(); + break; + + case boost::json::kind::int64: + os << jv.get_int64(); + break; + + case boost::json::kind::double_: + os << jv.get_double(); + break; + + case boost::json::kind::bool_: + if (jv.get_bool()) + os << "true"; + else + os << "false"; + break; + + case boost::json::kind::null: + os << "null"; + break; + } + + if (indent->empty()) + os << "\n"; +} + +} // namespace scwx::util::json diff --git a/wxdata/wxdata.cmake b/wxdata/wxdata.cmake index 77e65c58..4b08ad19 100644 --- a/wxdata/wxdata.cmake +++ b/wxdata/wxdata.cmake @@ -78,6 +78,7 @@ set(HDR_UTIL include/scwx/util/digest.hpp include/scwx/util/float.hpp include/scwx/util/hash.hpp include/scwx/util/iterator.hpp + include/scwx/util/json.hpp include/scwx/util/logger.hpp include/scwx/util/map.hpp include/scwx/util/rangebuf.hpp @@ -90,6 +91,7 @@ set(SRC_UTIL source/scwx/util/digest.cpp source/scwx/util/environment.cpp source/scwx/util/float.cpp source/scwx/util/hash.cpp + source/scwx/util/json.cpp source/scwx/util/logger.cpp source/scwx/util/rangebuf.cpp source/scwx/util/streams.cpp From cd7435a4d5ddfcd7ec01001ade22c9110c2b4cd7 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 1 Feb 2025 22:20:33 -0600 Subject: [PATCH 07/67] Add IEM types supporting AFOS list --- wxdata/include/scwx/types/iem_types.hpp | 73 +++++++++++++++++ wxdata/source/scwx/types/iem_types.cpp | 103 ++++++++++++++++++++++++ wxdata/wxdata.cmake | 6 ++ 3 files changed, 182 insertions(+) create mode 100644 wxdata/include/scwx/types/iem_types.hpp create mode 100644 wxdata/source/scwx/types/iem_types.cpp diff --git a/wxdata/include/scwx/types/iem_types.hpp b/wxdata/include/scwx/types/iem_types.hpp new file mode 100644 index 00000000..ee461c36 --- /dev/null +++ b/wxdata/include/scwx/types/iem_types.hpp @@ -0,0 +1,73 @@ +#pragma once + +#include +#include + +#include + +namespace scwx::types::iem +{ + +/** + * @brief AFOS Entry object + * + * + */ +struct AfosEntry +{ + std::int64_t index_ {}; + std::string entered_ {}; + std::string pil_ {}; + std::string productId_ {}; + std::string cccc_ {}; + std::int64_t count_ {}; + std::string link_ {}; + std::string textLink_ {}; +}; + +/** + * @brief AFOS List object + * + * + */ +struct AfosList +{ + std::vector data_ {}; +}; + +/** + * @brief Bad Request (400) object + */ +struct BadRequest +{ + std::string detail_ {}; +}; + +/** + * @brief Validation Error (422) object + */ +struct ValidationError +{ + struct Detail + { + std::string type_ {}; + std::vector> loc_ {}; + std::string msg_ {}; + std::string input_ {}; + struct Context + { + std::string error_ {}; + } ctx_; + }; + + std::vector detail_ {}; +}; + +AfosList tag_invoke(boost::json::value_to_tag, + const boost::json::value& jv); +BadRequest tag_invoke(boost::json::value_to_tag, + const boost::json::value& jv); +ValidationError tag_invoke(boost::json::value_to_tag, + const boost::json::value& jv); + +} // namespace scwx::types::iem diff --git a/wxdata/source/scwx/types/iem_types.cpp b/wxdata/source/scwx/types/iem_types.cpp new file mode 100644 index 00000000..c1921ede --- /dev/null +++ b/wxdata/source/scwx/types/iem_types.cpp @@ -0,0 +1,103 @@ +#include + +#include + +namespace scwx::types::iem +{ + +AfosEntry tag_invoke(boost::json::value_to_tag, + const boost::json::value& jv) +{ + auto& jo = jv.as_object(); + + AfosEntry entry {}; + + // Required parameters + entry.index_ = jo.at("index").as_int64(); + entry.entered_ = jo.at("entered").as_string(); + entry.pil_ = jo.at("pil").as_string(); + entry.productId_ = jo.at("product_id").as_string(); + entry.cccc_ = jo.at("cccc").as_string(); + entry.count_ = jo.at("count").as_int64(); + entry.link_ = jo.at("link").as_string(); + entry.textLink_ = jo.at("text_link").as_string(); + + return entry; +} + +AfosList tag_invoke(boost::json::value_to_tag, + const boost::json::value& jv) +{ + auto& jo = jv.as_object(); + + AfosList list {}; + + // Required parameters + list.data_ = boost::json::value_to>(jo.at("data")); + + return list; +} + +BadRequest tag_invoke(boost::json::value_to_tag, + const boost::json::value& jv) +{ + auto& jo = jv.as_object(); + + BadRequest badRequest {}; + + // Required parameters + badRequest.detail_ = jo.at("detail").as_string(); + + return badRequest; +} + +ValidationError::Detail::Context +tag_invoke(boost::json::value_to_tag, + const boost::json::value& jv) +{ + auto& jo = jv.as_object(); + + ValidationError::Detail::Context ctx {}; + + // Required parameters + ctx.error_ = jo.at("error").as_string(); + + return ctx; +} + +ValidationError::Detail +tag_invoke(boost::json::value_to_tag, + const boost::json::value& jv) +{ + auto& jo = jv.as_object(); + + ValidationError::Detail detail {}; + + // Required parameters + detail.type_ = jo.at("type").as_string(); + detail.loc_ = boost::json::value_to< + std::vector>>(jo.at("loc")); + detail.msg_ = jo.at("msg").as_string(); + detail.input_ = jo.at("input").as_string(); + + detail.ctx_ = + boost::json::value_to(jo.at("ctx")); + + return detail; +} + +ValidationError tag_invoke(boost::json::value_to_tag, + const boost::json::value& jv) +{ + auto& jo = jv.as_object(); + + ValidationError error {}; + + // Required parameters + error.detail_ = boost::json::value_to>( + jo.at("detail")); + + return error; +} + +} // namespace scwx::types::iem diff --git a/wxdata/wxdata.cmake b/wxdata/wxdata.cmake index 4b08ad19..92de3bf0 100644 --- a/wxdata/wxdata.cmake +++ b/wxdata/wxdata.cmake @@ -72,6 +72,8 @@ set(SRC_PROVIDER source/scwx/provider/aws_level2_data_provider.cpp source/scwx/provider/nexrad_data_provider.cpp source/scwx/provider/nexrad_data_provider_factory.cpp source/scwx/provider/warnings_provider.cpp) +set(HDR_TYPES include/scwx/types/iem_types.hpp) +set(SRC_TYPES source/scwx/types/iem_types.cpp) set(HDR_UTIL include/scwx/util/digest.hpp include/scwx/util/enum.hpp include/scwx/util/environment.hpp @@ -228,6 +230,8 @@ add_library(wxdata OBJECT ${HDR_AWIPS} ${SRC_NETWORK} ${HDR_PROVIDER} ${SRC_PROVIDER} + ${HDR_TYPES} + ${SRC_TYPES} ${HDR_UTIL} ${SRC_UTIL} ${HDR_WSR88D} @@ -248,6 +252,8 @@ source_group("Header Files\\network" FILES ${HDR_NETWORK}) source_group("Source Files\\network" FILES ${SRC_NETWORK}) source_group("Header Files\\provider" FILES ${HDR_PROVIDER}) source_group("Source Files\\provider" FILES ${SRC_PROVIDER}) +source_group("Header Files\\types" FILES ${HDR_TYPES}) +source_group("Source Files\\types" FILES ${SRC_TYPES}) source_group("Header Files\\util" FILES ${HDR_UTIL}) source_group("Source Files\\util" FILES ${SRC_UTIL}) source_group("Header Files\\wsr88d" FILES ${HDR_WSR88D}) From 59a8fdbf56463957e5886feb8e78bfbc483a43ed Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 1 Feb 2025 22:32:39 -0600 Subject: [PATCH 08/67] List NWS text products metadata --- .../scwx/provider/iem_warnings_provider.hpp | 6 + .../scwx/provider/iem_warnings_provider.cpp | 123 ++++++++++++++++++ 2 files changed, 129 insertions(+) diff --git a/wxdata/include/scwx/provider/iem_warnings_provider.hpp b/wxdata/include/scwx/provider/iem_warnings_provider.hpp index 64c5e96f..2061afa4 100644 --- a/wxdata/include/scwx/provider/iem_warnings_provider.hpp +++ b/wxdata/include/scwx/provider/iem_warnings_provider.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include namespace scwx::provider { @@ -20,6 +21,11 @@ public: IemWarningsProvider(IemWarningsProvider&&) noexcept; IemWarningsProvider& operator=(IemWarningsProvider&&) noexcept; + std::vector + ListTextProducts(std::chrono::sys_time date, + std::optional cccc = {}, + std::optional pil = {}); + private: class Impl; std::unique_ptr p; diff --git a/wxdata/source/scwx/provider/iem_warnings_provider.cpp b/wxdata/source/scwx/provider/iem_warnings_provider.cpp index 7c780816..b40955ca 100644 --- a/wxdata/source/scwx/provider/iem_warnings_provider.cpp +++ b/wxdata/source/scwx/provider/iem_warnings_provider.cpp @@ -1,12 +1,23 @@ #include +#include +#include +#include #include +#include +#include + namespace scwx::provider { static const std::string logPrefix_ = "scwx::provider::iem_warnings_provider"; static const auto logger_ = util::Logger::Create(logPrefix_); +static const std::string kBaseUrl_ = "https://mesonet.agron.iastate.edu/api/1"; + +static const std::string kListNwsTextProductsEndpoint_ = "/nws/afos/list.json"; +static const std::string kNwsTextProductEndpoint_ = "/nwstext/"; + class IemWarningsProvider::Impl { public: @@ -16,6 +27,11 @@ public: Impl& operator=(const Impl&) = delete; Impl(const Impl&&) = delete; Impl& operator=(const Impl&&) = delete; + + std::vector + ListTextProducts(std::chrono::sys_time date, + std::optional cccc = {}, + std::optional pil = {}); }; IemWarningsProvider::IemWarningsProvider() : p(std::make_unique()) {} @@ -26,4 +42,111 @@ IemWarningsProvider::IemWarningsProvider(IemWarningsProvider&&) noexcept = IemWarningsProvider& IemWarningsProvider::operator=(IemWarningsProvider&&) noexcept = default; +std::vector IemWarningsProvider::ListTextProducts( + std::chrono::sys_time date, + std::optional cccc, + std::optional pil) +{ + return p->ListTextProducts(date, cccc, pil); +} + +std::vector IemWarningsProvider::Impl::ListTextProducts( + std::chrono::sys_time date, + std::optional cccc, + std::optional pil) +{ + using namespace std::chrono; + +#if (__cpp_lib_chrono >= 201907L) + namespace df = std; + + static constexpr std::string_view kDateFormat {"{:%Y-%m-%d}"}; +#else + using namespace date; + namespace df = date; + +# define kDateFormat "%Y-%m-%d" +#endif + + auto parameters = cpr::Parameters {{"date", df::format(kDateFormat, date)}}; + + // WMO Source Code + if (cccc.has_value()) + { + parameters.Add({"cccc", std::string {cccc.value()}}); + } + + // AFOS / AWIPS ID / 3-6 length identifier + if (pil.has_value()) + { + parameters.Add({"pil", std::string {pil.value()}}); + } + + auto response = + cpr::Get(cpr::Url {kBaseUrl_ + kListNwsTextProductsEndpoint_}, + network::cpr::GetHeader(), + parameters); + boost::json::value json = util::json::ReadJsonString(response.text); + + std::vector textProducts {}; + + if (response.status_code == cpr::status::HTTP_OK) + { + try + { + // Get AFOS list from response + auto entries = boost::json::value_to(json); + + for (auto& entry : entries.data_) + { + textProducts.push_back(entry.productId_); + } + + logger_->trace("Found {} products", entries.data_.size()); + } + catch (const std::exception& ex) + { + // Unexpected bad response + logger_->warn("Error parsing JSON: {}", ex.what()); + } + } + else if (response.status_code == cpr::status::HTTP_BAD_REQUEST && + json != nullptr) + { + try + { + // Log bad request details + auto badRequest = boost::json::value_to(json); + logger_->warn("ListTextProducts bad request: {}", badRequest.detail_); + } + catch (const std::exception& ex) + { + // Unexpected bad response + logger_->warn("Error parsing bad response: {}", ex.what()); + } + } + else if (response.status_code == cpr::status::HTTP_UNPROCESSABLE_ENTITY && + json != nullptr) + { + try + { + // Log validation error details + auto error = boost::json::value_to(json); + logger_->warn("ListTextProducts validation error: {}", + error.detail_.at(0).msg_); + } + catch (const std::exception& ex) + { + // Unexpected bad response + logger_->warn("Error parsing validation error: {}", ex.what()); + } + } + else + { + logger_->warn("Could not list text products: {}", response.status_line); + } + + return textProducts; +} + } // namespace scwx::provider From f06191f290c20587502ed7c2db5f9768bf073b85 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 1 Feb 2025 22:33:24 -0600 Subject: [PATCH 09/67] Add IEM warnings provider test --- .../provider/iem_warnings_provider.test.cpp | 34 +++++++++++++++++++ test/test.cmake | 1 + 2 files changed, 35 insertions(+) create mode 100644 test/source/scwx/provider/iem_warnings_provider.test.cpp diff --git a/test/source/scwx/provider/iem_warnings_provider.test.cpp b/test/source/scwx/provider/iem_warnings_provider.test.cpp new file mode 100644 index 00000000..4f66e185 --- /dev/null +++ b/test/source/scwx/provider/iem_warnings_provider.test.cpp @@ -0,0 +1,34 @@ +#include + +#include + +namespace scwx +{ +namespace provider +{ + +TEST(IemWarningsProviderTest, LoadUpdatedFiles) +{ + using namespace std::chrono; + using sys_days = time_point; + + IemWarningsProvider provider {}; + + auto date = sys_days {2023y / March / 25d}; + + auto torProducts = provider.ListTextProducts(date, {}, "TOR"); + + EXPECT_EQ(torProducts.size(), 35); + + if (torProducts.size() >= 1) + { + EXPECT_EQ(torProducts.at(0), "202303250016-KMEG-WFUS54-TORMEG"); + } + if (torProducts.size() >= 35) + { + EXPECT_EQ(torProducts.at(34), "202303252015-KFFC-WFUS52-TORFFC"); + } +} + +} // namespace provider +} // namespace scwx diff --git a/test/test.cmake b/test/test.cmake index 3ec6ef19..17141e0c 100644 --- a/test/test.cmake +++ b/test/test.cmake @@ -19,6 +19,7 @@ set(SRC_GR_TESTS source/scwx/gr/placefile.test.cpp) set(SRC_NETWORK_TESTS source/scwx/network/dir_list.test.cpp) set(SRC_PROVIDER_TESTS source/scwx/provider/aws_level2_data_provider.test.cpp source/scwx/provider/aws_level3_data_provider.test.cpp + source/scwx/provider/iem_warnings_provider.test.cpp source/scwx/provider/warnings_provider.test.cpp) set(SRC_QT_CONFIG_TESTS source/scwx/qt/config/county_database.test.cpp source/scwx/qt/config/radar_site.test.cpp) From 2720ad6a38fd9b13ebecc02e09cd20f62cda7907 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 1 Feb 2025 23:40:06 -0600 Subject: [PATCH 10/67] Add IEM load text product API functionality --- .../provider/iem_warnings_provider.test.cpp | 15 ++++- .../scwx/provider/iem_warnings_provider.hpp | 7 ++- wxdata/source/scwx/awips/wmo_header.cpp | 12 +++- .../scwx/provider/iem_warnings_provider.cpp | 60 +++++++++++++++---- wxdata/source/scwx/types/iem_types.cpp | 16 +++-- 5 files changed, 90 insertions(+), 20 deletions(-) diff --git a/test/source/scwx/provider/iem_warnings_provider.test.cpp b/test/source/scwx/provider/iem_warnings_provider.test.cpp index 4f66e185..6b6809e5 100644 --- a/test/source/scwx/provider/iem_warnings_provider.test.cpp +++ b/test/source/scwx/provider/iem_warnings_provider.test.cpp @@ -7,7 +7,7 @@ namespace scwx namespace provider { -TEST(IemWarningsProviderTest, LoadUpdatedFiles) +TEST(IemWarningsProviderTest, ListTextProducts) { using namespace std::chrono; using sys_days = time_point; @@ -30,5 +30,18 @@ TEST(IemWarningsProviderTest, LoadUpdatedFiles) } } +TEST(IemWarningsProviderTest, LoadTextProducts) +{ + static const std::vector productIds { + "202303250016-KMEG-WFUS54-TORMEG", // + "202303252015-KFFC-WFUS52-TORFFC"}; + + IemWarningsProvider provider {}; + + auto textProducts = provider.LoadTextProducts(productIds); + + EXPECT_EQ(textProducts.size(), 2); +} + } // namespace provider } // namespace scwx diff --git a/wxdata/include/scwx/provider/iem_warnings_provider.hpp b/wxdata/include/scwx/provider/iem_warnings_provider.hpp index 2061afa4..35c6bbad 100644 --- a/wxdata/include/scwx/provider/iem_warnings_provider.hpp +++ b/wxdata/include/scwx/provider/iem_warnings_provider.hpp @@ -1,5 +1,7 @@ #pragma once +#include + #include #include @@ -21,11 +23,14 @@ public: IemWarningsProvider(IemWarningsProvider&&) noexcept; IemWarningsProvider& operator=(IemWarningsProvider&&) noexcept; - std::vector + static std::vector ListTextProducts(std::chrono::sys_time date, std::optional cccc = {}, std::optional pil = {}); + static std::vector> + LoadTextProducts(const std::vector& textProducts); + private: class Impl; std::unique_ptr p; diff --git a/wxdata/source/scwx/awips/wmo_header.cpp b/wxdata/source/scwx/awips/wmo_header.cpp index eb4501e7..4d502604 100644 --- a/wxdata/source/scwx/awips/wmo_header.cpp +++ b/wxdata/source/scwx/awips/wmo_header.cpp @@ -132,9 +132,19 @@ bool WmoHeader::Parse(std::istream& is) { util::getline(is, sohLine); util::getline(is, sequenceLine); + util::getline(is, wmoLine); + } + else + { + // The next line could be the WMO line or the sequence line + util::getline(is, wmoLine); + if (wmoLine.length() < 18) + { + sequenceLine.swap(wmoLine); + util::getline(is, wmoLine); + } } - util::getline(is, wmoLine); util::getline(is, awipsLine); if (is.eof()) diff --git a/wxdata/source/scwx/provider/iem_warnings_provider.cpp b/wxdata/source/scwx/provider/iem_warnings_provider.cpp index b40955ca..47c29547 100644 --- a/wxdata/source/scwx/provider/iem_warnings_provider.cpp +++ b/wxdata/source/scwx/provider/iem_warnings_provider.cpp @@ -27,11 +27,6 @@ public: Impl& operator=(const Impl&) = delete; Impl(const Impl&&) = delete; Impl& operator=(const Impl&&) = delete; - - std::vector - ListTextProducts(std::chrono::sys_time date, - std::optional cccc = {}, - std::optional pil = {}); }; IemWarningsProvider::IemWarningsProvider() : p(std::make_unique()) {} @@ -46,14 +41,6 @@ std::vector IemWarningsProvider::ListTextProducts( std::chrono::sys_time date, std::optional cccc, std::optional pil) -{ - return p->ListTextProducts(date, cccc, pil); -} - -std::vector IemWarningsProvider::Impl::ListTextProducts( - std::chrono::sys_time date, - std::optional cccc, - std::optional pil) { using namespace std::chrono; @@ -149,4 +136,51 @@ std::vector IemWarningsProvider::Impl::ListTextProducts( return textProducts; } +std::vector> +IemWarningsProvider::LoadTextProducts( + const std::vector& textProducts) +{ + auto parameters = cpr::Parameters {{"nolimit", "true"}}; + + std::vector> + asyncResponses {}; + + for (auto& productId : textProducts) + { + asyncResponses.emplace_back( + productId, + cpr::GetAsync( + cpr::Url {kBaseUrl_ + kNwsTextProductEndpoint_ + productId}, + network::cpr::GetHeader(), + parameters)); + } + + std::vector> textProductFiles; + + for (auto& asyncResponse : asyncResponses) + { + auto response = asyncResponse.second.get(); + + if (response.status_code == cpr::status::HTTP_OK) + { + // Load file + std::shared_ptr textProductFile { + std::make_shared()}; + std::istringstream responseBody {response.text}; + if (textProductFile->LoadData(responseBody)) + { + textProductFiles.push_back(textProductFile); + } + } + else + { + logger_->warn("Could not load text product: {} ({})", + asyncResponse.first, + response.status_line); + } + } + + return textProductFiles; +} + } // namespace scwx::provider diff --git a/wxdata/source/scwx/types/iem_types.cpp b/wxdata/source/scwx/types/iem_types.cpp index c1921ede..ed3884d3 100644 --- a/wxdata/source/scwx/types/iem_types.cpp +++ b/wxdata/source/scwx/types/iem_types.cpp @@ -77,11 +77,19 @@ tag_invoke(boost::json::value_to_tag, detail.type_ = jo.at("type").as_string(); detail.loc_ = boost::json::value_to< std::vector>>(jo.at("loc")); - detail.msg_ = jo.at("msg").as_string(); - detail.input_ = jo.at("input").as_string(); + detail.msg_ = jo.at("msg").as_string(); - detail.ctx_ = - boost::json::value_to(jo.at("ctx")); + // Optional parameters + if (jo.contains("input")) + { + detail.input_ = jo.at("input").as_string(); + } + + if (jo.contains("ctx")) + { + detail.ctx_ = + boost::json::value_to(jo.at("ctx")); + } return detail; } From e6cfef06a73286ea435d2a04a3c923fb5f739e14 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sun, 2 Feb 2025 01:34:35 -0600 Subject: [PATCH 11/67] Text product message fixes to support IEM --- .../scwx/awips/text_product_message.cpp | 44 ++++++++++--------- wxdata/source/scwx/util/streams.cpp | 18 +++++--- 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/wxdata/source/scwx/awips/text_product_message.cpp b/wxdata/source/scwx/awips/text_product_message.cpp index 54ce7e25..3b571d83 100644 --- a/wxdata/source/scwx/awips/text_product_message.cpp +++ b/wxdata/source/scwx/awips/text_product_message.cpp @@ -11,9 +11,7 @@ #include #include -namespace scwx -{ -namespace awips +namespace scwx::awips { static const std::string logPrefix_ = "scwx::awips::text_product_message"; @@ -49,6 +47,11 @@ public: { } ~TextProductMessageImpl() = default; + + TextProductMessageImpl(const TextProductMessageImpl&) = delete; + TextProductMessageImpl& operator=(const TextProductMessageImpl&) = delete; + TextProductMessageImpl(const TextProductMessageImpl&&) = delete; + TextProductMessageImpl& operator=(const TextProductMessageImpl&&) = delete; std::string messageContent_; std::shared_ptr wmoHeader_; @@ -232,7 +235,7 @@ bool TextProductMessage::Parse(std::istream& is) if (i == 0) { - if (is.peek() != '\r') + if (is.peek() != '\r' && is.peek() != '\n') { segment->header_ = TryParseSegmentHeader(is); } @@ -318,8 +321,8 @@ bool TextProductMessage::Parse(std::istream& is) return dataValid; } -void ParseCodedInformation(std::shared_ptr segment, - const std::string& wfo) +void ParseCodedInformation(const std::shared_ptr& segment, + const std::string& wfo) { typedef std::vector::const_iterator StringIterator; @@ -352,8 +355,8 @@ void ParseCodedInformation(std::shared_ptr segment, codedLocationEnd = it; } - else if (codedMotionBegin == productContent.cend() && - it->starts_with("TIME...MOT...LOC")) + if (codedMotionBegin == productContent.cend() && + it->starts_with("TIME...MOT...LOC")) { codedMotionBegin = it; } @@ -366,8 +369,7 @@ void ParseCodedInformation(std::shared_ptr segment, codedMotionEnd = it; } - else if (!segment->observed_ && - it->find("...OBSERVED") != std::string::npos) + if (!segment->observed_ && it->find("...OBSERVED") != std::string::npos) { segment->observed_ = true; } @@ -378,6 +380,8 @@ void ParseCodedInformation(std::shared_ptr segment, segment->tornadoPossible_ = true; } + // Assignment of an iterator permitted + // NOLINTBEGIN(bugprone-assignment-in-if-condition) else if (segment->threatCategory_ == ibw::ThreatCategory::Base && (threatTagIt = std::find_if(kThreatCategoryTags.cbegin(), kThreatCategoryTags.cend(), @@ -385,6 +389,7 @@ void ParseCodedInformation(std::shared_ptr segment, return it->starts_with(tag); })) != kThreatCategoryTags.cend() && it->length() > threatTagIt->length()) + // NOLINTEND(bugprone-assignment-in-if-condition) { const std::string threatCategoryName = it->substr(threatTagIt->length()); @@ -458,7 +463,7 @@ void SkipBlankLines(std::istream& is) { std::string line; - while (is.peek() == '\r') + while (is.peek() == '\r' || is.peek() == '\n') { util::getline(is, line); } @@ -513,7 +518,7 @@ std::vector TryParseMndHeader(std::istream& is) std::string line; std::streampos isBegin = is.tellg(); - while (!is.eof() && is.peek() != '\r') + while (!is.eof() && is.peek() != '\r' && is.peek() != '\n') { util::getline(is, line); mndHeader.push_back(line); @@ -546,7 +551,7 @@ std::vector TryParseOverviewBlock(std::istream& is) if (is.peek() == '.') { - while (!is.eof() && is.peek() != '\r') + while (!is.eof() && is.peek() != '\r' && is.peek() != '\n') { util::getline(is, line); overviewBlock.push_back(line); @@ -576,7 +581,7 @@ std::optional TryParseSegmentHeader(std::istream& is) header->ugcString_.push_back(line); // If UGC is multi-line, continue parsing - while (!is.eof() && is.peek() != '\r' && + while (!is.eof() && is.peek() != '\r' && is.peek() != '\n' && !RE2::PartialMatch(line, *reUgcExpiration)) { util::getline(is, line); @@ -595,7 +600,7 @@ std::optional TryParseSegmentHeader(std::istream& is) header->vtecString_.push_back(std::move(*vtec)); } - while (!is.eof() && is.peek() != '\r') + while (!is.eof() && is.peek() != '\r' && is.peek() != '\n') { util::getline(is, line); if (!RE2::PartialMatch(line, *reDateTimeString)) @@ -640,10 +645,8 @@ std::optional TryParseVtecString(std::istream& is) if (RE2::PartialMatch(line, *rePVtecString)) { - bool vtecValid; - - vtec = Vtec(); - vtecValid = vtec->pVtec_.Parse(line); + vtec = Vtec(); + bool vtecValid = vtec->pVtec_.Parse(line); isBegin = is.tellg(); @@ -687,5 +690,4 @@ std::shared_ptr TextProductMessage::Create(std::istream& is) return message; } -} // namespace awips -} // namespace scwx +} // namespace scwx::awips diff --git a/wxdata/source/scwx/util/streams.cpp b/wxdata/source/scwx/util/streams.cpp index 9e094f9b..6374ed35 100644 --- a/wxdata/source/scwx/util/streams.cpp +++ b/wxdata/source/scwx/util/streams.cpp @@ -1,8 +1,7 @@ #include +#include -namespace scwx -{ -namespace util +namespace scwx::util { std::istream& getline(std::istream& is, std::string& t) @@ -17,7 +16,8 @@ std::istream& getline(std::istream& is, std::string& t) int c = sb->sbumpc(); switch (c) { - case '\n': return is; + case '\n': + return is; case '\r': while (sb->sgetc() == '\r') @@ -30,6 +30,10 @@ std::istream& getline(std::istream& is, std::string& t) } return is; + case common::Characters::ETX: + sb->sungetc(); + return is; + case std::streambuf::traits_type::eof(): if (t.empty()) { @@ -37,10 +41,10 @@ std::istream& getline(std::istream& is, std::string& t) } return is; - default: t += static_cast(c); + default: + t += static_cast(c); } } } -} // namespace util -} // namespace scwx +} // namespace scwx::util From 8da440ea1f3efa1630727a4b4ddcdcf1cd9c42f6 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sun, 2 Feb 2025 01:35:19 -0600 Subject: [PATCH 12/67] General linter cleanup --- wxdata/source/scwx/awips/wmo_header.cpp | 17 +++++++++++------ .../scwx/provider/iem_warnings_provider.cpp | 5 ++++- wxdata/source/scwx/util/json.cpp | 2 ++ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/wxdata/source/scwx/awips/wmo_header.cpp b/wxdata/source/scwx/awips/wmo_header.cpp index 4d502604..a701e476 100644 --- a/wxdata/source/scwx/awips/wmo_header.cpp +++ b/wxdata/source/scwx/awips/wmo_header.cpp @@ -12,14 +12,14 @@ # include #endif -namespace scwx -{ -namespace awips +namespace scwx::awips { static const std::string logPrefix_ = "scwx::awips::wmo_header"; static const auto logger_ = util::Logger::Create(logPrefix_); +static constexpr std::size_t kWmoHeaderMinLineLength_ = 18; + class WmoHeaderImpl { public: @@ -37,6 +37,11 @@ public: } ~WmoHeaderImpl() = default; + WmoHeaderImpl(const WmoHeaderImpl&) = delete; + WmoHeaderImpl& operator=(const WmoHeaderImpl&) = delete; + WmoHeaderImpl(const WmoHeaderImpl&&) = delete; + WmoHeaderImpl& operator=(const WmoHeaderImpl&&) = delete; + bool operator==(const WmoHeaderImpl& o) const; std::string sequenceNumber_; @@ -138,8 +143,9 @@ bool WmoHeader::Parse(std::istream& is) { // The next line could be the WMO line or the sequence line util::getline(is, wmoLine); - if (wmoLine.length() < 18) + if (wmoLine.length() < kWmoHeaderMinLineLength_) { + // This is likely the sequence line instead sequenceLine.swap(wmoLine); util::getline(is, wmoLine); } @@ -249,5 +255,4 @@ bool WmoHeader::Parse(std::istream& is) return headerValid; } -} // namespace awips -} // namespace scwx +} // namespace scwx::awips diff --git a/wxdata/source/scwx/provider/iem_warnings_provider.cpp b/wxdata/source/scwx/provider/iem_warnings_provider.cpp index 47c29547..4bbd339a 100644 --- a/wxdata/source/scwx/provider/iem_warnings_provider.cpp +++ b/wxdata/source/scwx/provider/iem_warnings_provider.cpp @@ -144,13 +144,16 @@ IemWarningsProvider::LoadTextProducts( std::vector> asyncResponses {}; + asyncResponses.reserve(textProducts.size()); + + const std::string endpointUrl = kBaseUrl_ + kNwsTextProductEndpoint_; for (auto& productId : textProducts) { asyncResponses.emplace_back( productId, cpr::GetAsync( - cpr::Url {kBaseUrl_ + kNwsTextProductEndpoint_ + productId}, + cpr::Url {endpointUrl + productId}, network::cpr::GetHeader(), parameters)); } diff --git a/wxdata/source/scwx/util/json.cpp b/wxdata/source/scwx/util/json.cpp index b8f51507..d5873758 100644 --- a/wxdata/source/scwx/util/json.cpp +++ b/wxdata/source/scwx/util/json.cpp @@ -107,6 +107,8 @@ void WriteJsonFile(const std::string& path, } } +// Allow recursion within the pretty print function +// NOLINTNEXTLINE(misc-no-recursion) static void PrettyPrintJson(std::ostream& os, boost::json::value const& jv, std::string* indent) From cf87cc9bf0ff3871138310fb822d4e84a44ba1f3 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sun, 2 Feb 2025 01:36:12 -0600 Subject: [PATCH 13/67] Updating test for IEM provider --- .../provider/iem_warnings_provider.test.cpp | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/test/source/scwx/provider/iem_warnings_provider.test.cpp b/test/source/scwx/provider/iem_warnings_provider.test.cpp index 6b6809e5..471ac9d0 100644 --- a/test/source/scwx/provider/iem_warnings_provider.test.cpp +++ b/test/source/scwx/provider/iem_warnings_provider.test.cpp @@ -33,14 +33,28 @@ TEST(IemWarningsProviderTest, ListTextProducts) TEST(IemWarningsProviderTest, LoadTextProducts) { static const std::vector productIds { - "202303250016-KMEG-WFUS54-TORMEG", // - "202303252015-KFFC-WFUS52-TORFFC"}; + "202303250016-KMEG-WFUS54-TORMEG", + "202303252015-KFFC-WFUS52-TORFFC", + "202303311942-KLZK-WWUS54-SVSLZK"}; IemWarningsProvider provider {}; auto textProducts = provider.LoadTextProducts(productIds); - EXPECT_EQ(textProducts.size(), 2); + EXPECT_EQ(textProducts.size(), 3); + + if (textProducts.size() >= 1) + { + EXPECT_EQ(textProducts.at(0)->message_count(), 1); + } + if (textProducts.size() >= 2) + { + EXPECT_EQ(textProducts.at(1)->message_count(), 1); + } + if (textProducts.size() >= 3) + { + EXPECT_EQ(textProducts.at(2)->message_count(), 2); + } } } // namespace provider From f9e79b3e407e9ceee279b7b5ab32ebfa251f974c Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sun, 2 Feb 2025 01:42:45 -0600 Subject: [PATCH 14/67] Rename IEM warnings provider to IEM API provider --- ...der.test.cpp => iem_api_provider.test.cpp} | 10 +++++----- test/test.cmake | 2 +- ...ings_provider.hpp => iem_api_provider.hpp} | 14 ++++++------- ...ings_provider.cpp => iem_api_provider.cpp} | 20 +++++++++---------- wxdata/wxdata.cmake | 4 ++-- 5 files changed, 25 insertions(+), 25 deletions(-) rename test/source/scwx/provider/{iem_warnings_provider.test.cpp => iem_api_provider.test.cpp} (84%) rename wxdata/include/scwx/provider/{iem_warnings_provider.hpp => iem_api_provider.hpp} (64%) rename wxdata/source/scwx/provider/{iem_warnings_provider.cpp => iem_api_provider.cpp} (89%) diff --git a/test/source/scwx/provider/iem_warnings_provider.test.cpp b/test/source/scwx/provider/iem_api_provider.test.cpp similarity index 84% rename from test/source/scwx/provider/iem_warnings_provider.test.cpp rename to test/source/scwx/provider/iem_api_provider.test.cpp index 471ac9d0..68cb59ae 100644 --- a/test/source/scwx/provider/iem_warnings_provider.test.cpp +++ b/test/source/scwx/provider/iem_api_provider.test.cpp @@ -1,4 +1,4 @@ -#include +#include #include @@ -7,12 +7,12 @@ namespace scwx namespace provider { -TEST(IemWarningsProviderTest, ListTextProducts) +TEST(IemApiProviderTest, ListTextProducts) { using namespace std::chrono; using sys_days = time_point; - IemWarningsProvider provider {}; + IemApiProvider provider {}; auto date = sys_days {2023y / March / 25d}; @@ -30,14 +30,14 @@ TEST(IemWarningsProviderTest, ListTextProducts) } } -TEST(IemWarningsProviderTest, LoadTextProducts) +TEST(IemApiProviderTest, LoadTextProducts) { static const std::vector productIds { "202303250016-KMEG-WFUS54-TORMEG", "202303252015-KFFC-WFUS52-TORFFC", "202303311942-KLZK-WWUS54-SVSLZK"}; - IemWarningsProvider provider {}; + IemApiProvider provider {}; auto textProducts = provider.LoadTextProducts(productIds); diff --git a/test/test.cmake b/test/test.cmake index 17141e0c..64b6c69e 100644 --- a/test/test.cmake +++ b/test/test.cmake @@ -19,7 +19,7 @@ set(SRC_GR_TESTS source/scwx/gr/placefile.test.cpp) set(SRC_NETWORK_TESTS source/scwx/network/dir_list.test.cpp) set(SRC_PROVIDER_TESTS source/scwx/provider/aws_level2_data_provider.test.cpp source/scwx/provider/aws_level3_data_provider.test.cpp - source/scwx/provider/iem_warnings_provider.test.cpp + source/scwx/provider/iem_api_provider.test.cpp source/scwx/provider/warnings_provider.test.cpp) set(SRC_QT_CONFIG_TESTS source/scwx/qt/config/county_database.test.cpp source/scwx/qt/config/radar_site.test.cpp) diff --git a/wxdata/include/scwx/provider/iem_warnings_provider.hpp b/wxdata/include/scwx/provider/iem_api_provider.hpp similarity index 64% rename from wxdata/include/scwx/provider/iem_warnings_provider.hpp rename to wxdata/include/scwx/provider/iem_api_provider.hpp index 35c6bbad..33a82bcf 100644 --- a/wxdata/include/scwx/provider/iem_warnings_provider.hpp +++ b/wxdata/include/scwx/provider/iem_api_provider.hpp @@ -11,17 +11,17 @@ namespace scwx::provider /** * @brief Warnings Provider */ -class IemWarningsProvider +class IemApiProvider { public: - explicit IemWarningsProvider(); - ~IemWarningsProvider(); + explicit IemApiProvider(); + ~IemApiProvider(); - IemWarningsProvider(const IemWarningsProvider&) = delete; - IemWarningsProvider& operator=(const IemWarningsProvider&) = delete; + IemApiProvider(const IemApiProvider&) = delete; + IemApiProvider& operator=(const IemApiProvider&) = delete; - IemWarningsProvider(IemWarningsProvider&&) noexcept; - IemWarningsProvider& operator=(IemWarningsProvider&&) noexcept; + IemApiProvider(IemApiProvider&&) noexcept; + IemApiProvider& operator=(IemApiProvider&&) noexcept; static std::vector ListTextProducts(std::chrono::sys_time date, diff --git a/wxdata/source/scwx/provider/iem_warnings_provider.cpp b/wxdata/source/scwx/provider/iem_api_provider.cpp similarity index 89% rename from wxdata/source/scwx/provider/iem_warnings_provider.cpp rename to wxdata/source/scwx/provider/iem_api_provider.cpp index 4bbd339a..96d1489e 100644 --- a/wxdata/source/scwx/provider/iem_warnings_provider.cpp +++ b/wxdata/source/scwx/provider/iem_api_provider.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -10,7 +10,7 @@ namespace scwx::provider { -static const std::string logPrefix_ = "scwx::provider::iem_warnings_provider"; +static const std::string logPrefix_ = "scwx::provider::iem_api_provider"; static const auto logger_ = util::Logger::Create(logPrefix_); static const std::string kBaseUrl_ = "https://mesonet.agron.iastate.edu/api/1"; @@ -18,7 +18,7 @@ static const std::string kBaseUrl_ = "https://mesonet.agron.iastate.edu/api/1"; static const std::string kListNwsTextProductsEndpoint_ = "/nws/afos/list.json"; static const std::string kNwsTextProductEndpoint_ = "/nwstext/"; -class IemWarningsProvider::Impl +class IemApiProvider::Impl { public: explicit Impl() = default; @@ -29,15 +29,15 @@ public: Impl& operator=(const Impl&&) = delete; }; -IemWarningsProvider::IemWarningsProvider() : p(std::make_unique()) {} -IemWarningsProvider::~IemWarningsProvider() = default; +IemApiProvider::IemApiProvider() : p(std::make_unique()) {} +IemApiProvider::~IemApiProvider() = default; -IemWarningsProvider::IemWarningsProvider(IemWarningsProvider&&) noexcept = +IemApiProvider::IemApiProvider(IemApiProvider&&) noexcept = default; -IemWarningsProvider& -IemWarningsProvider::operator=(IemWarningsProvider&&) noexcept = default; +IemApiProvider& +IemApiProvider::operator=(IemApiProvider&&) noexcept = default; -std::vector IemWarningsProvider::ListTextProducts( +std::vector IemApiProvider::ListTextProducts( std::chrono::sys_time date, std::optional cccc, std::optional pil) @@ -137,7 +137,7 @@ std::vector IemWarningsProvider::ListTextProducts( } std::vector> -IemWarningsProvider::LoadTextProducts( +IemApiProvider::LoadTextProducts( const std::vector& textProducts) { auto parameters = cpr::Parameters {{"nolimit", "true"}}; diff --git a/wxdata/wxdata.cmake b/wxdata/wxdata.cmake index 92de3bf0..468f124b 100644 --- a/wxdata/wxdata.cmake +++ b/wxdata/wxdata.cmake @@ -61,14 +61,14 @@ set(SRC_NETWORK source/scwx/network/cpr.cpp set(HDR_PROVIDER include/scwx/provider/aws_level2_data_provider.hpp include/scwx/provider/aws_level3_data_provider.hpp include/scwx/provider/aws_nexrad_data_provider.hpp - include/scwx/provider/iem_warnings_provider.hpp + include/scwx/provider/iem_api_provider.hpp include/scwx/provider/nexrad_data_provider.hpp include/scwx/provider/nexrad_data_provider_factory.hpp include/scwx/provider/warnings_provider.hpp) set(SRC_PROVIDER source/scwx/provider/aws_level2_data_provider.cpp source/scwx/provider/aws_level3_data_provider.cpp source/scwx/provider/aws_nexrad_data_provider.cpp - source/scwx/provider/iem_warnings_provider.cpp + source/scwx/provider/iem_api_provider.cpp source/scwx/provider/nexrad_data_provider.cpp source/scwx/provider/nexrad_data_provider_factory.cpp source/scwx/provider/warnings_provider.cpp) From 2eb65defbc60c529ce722a9a17d4aa740f1e8bf8 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sun, 2 Feb 2025 01:43:36 -0600 Subject: [PATCH 15/67] Fix broken text product message function signature --- wxdata/source/scwx/awips/text_product_message.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wxdata/source/scwx/awips/text_product_message.cpp b/wxdata/source/scwx/awips/text_product_message.cpp index 3b571d83..465b665e 100644 --- a/wxdata/source/scwx/awips/text_product_message.cpp +++ b/wxdata/source/scwx/awips/text_product_message.cpp @@ -25,8 +25,8 @@ static const auto logger_ = scwx::util::Logger::Create(logPrefix_); // Look for hhmm (xM|UTC) to key the date/time string static constexpr LazyRE2 reDateTimeString = {"^[0-9]{3,4} ([AP]M|UTC)"}; -static void ParseCodedInformation(std::shared_ptr segment, - const std::string& wfo); +static void ParseCodedInformation(const std::shared_ptr& segment, + const std::string& wfo); static std::vector ParseProductContent(std::istream& is); static void SkipBlankLines(std::istream& is); static bool TryParseEndOfProduct(std::istream& is); @@ -47,7 +47,7 @@ public: { } ~TextProductMessageImpl() = default; - + TextProductMessageImpl(const TextProductMessageImpl&) = delete; TextProductMessageImpl& operator=(const TextProductMessageImpl&) = delete; TextProductMessageImpl(const TextProductMessageImpl&&) = delete; From c00016cb699b7a07fa1dab0bdce33e32f4759b95 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Mon, 17 Feb 2025 23:36:02 -0600 Subject: [PATCH 16/67] Warning file record should use filename, not URL --- wxdata/source/scwx/provider/warnings_provider.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/wxdata/source/scwx/provider/warnings_provider.cpp b/wxdata/source/scwx/provider/warnings_provider.cpp index 560cb307..6ef7e237 100644 --- a/wxdata/source/scwx/provider/warnings_provider.cpp +++ b/wxdata/source/scwx/provider/warnings_provider.cpp @@ -130,8 +130,7 @@ WarningsProvider::LoadUpdatedFiles( { if (headResponse.status_code == cpr::status::HTTP_OK) { - bool updated = - p->UpdateFileRecord(headResponse, url); // TODO: filename + bool updated = p->UpdateFileRecord(headResponse, filename); if (updated) { From 7e9895e0025fafa97b061714605682eceea981e3 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Mon, 17 Feb 2025 23:39:52 -0600 Subject: [PATCH 17/67] Adding robust date calculation to WMO header --- wxdata/include/scwx/awips/wmo_header.hpp | 36 ++++- .../scwx/awips/text_product_message.cpp | 64 +------- wxdata/source/scwx/awips/wmo_header.cpp | 150 ++++++++++++++++-- 3 files changed, 178 insertions(+), 72 deletions(-) diff --git a/wxdata/include/scwx/awips/wmo_header.hpp b/wxdata/include/scwx/awips/wmo_header.hpp index f3487b6d..f0aed0de 100644 --- a/wxdata/include/scwx/awips/wmo_header.hpp +++ b/wxdata/include/scwx/awips/wmo_header.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -27,7 +28,7 @@ public: explicit WmoHeader(); ~WmoHeader(); - WmoHeader(const WmoHeader&) = delete; + WmoHeader(const WmoHeader&) = delete; WmoHeader& operator=(const WmoHeader&) = delete; WmoHeader(WmoHeader&&) noexcept; @@ -45,8 +46,41 @@ public: std::string product_category() const; std::string product_designator() const; + /** + * @brief Get the WMO date/time + * + * Gets the WMO date/time. Uses the optional date hint provided via + * SetDateHint(std::chrono::year_month). If the date hint has not been + * provided, the endTimeHint parameter is required. + * + * @param [in] endTimeHint The optional end time bounds to provide. This is + * ignored if a date hint has been provided to determine an absolute date. + */ + std::chrono::sys_time GetDateTime( + std::optional endTimeHint = + std::nullopt); + + /** + * @brief Parse a WMO header + * + * @param [in] is The input stream to parse + */ bool Parse(std::istream& is); + /** + * @brief Provide a date hint for the WMO parser + * + * The WMO header contains a date/time in the format DDMMSS. The year and + * month must be derived using another source. The date hint provides the + * additional context required to determine the absolute product time. + * + * This function will update any absolute date/time already calculated, or + * affect the calculation of a subsequent absolute date/time. + * + * @param [in] dateHint The date hint to provide the WMO header parser + */ + void SetDateHint(std::chrono::year_month dateHint); + private: std::unique_ptr p; }; diff --git a/wxdata/source/scwx/awips/text_product_message.cpp b/wxdata/source/scwx/awips/text_product_message.cpp index 465b665e..9674b325 100644 --- a/wxdata/source/scwx/awips/text_product_message.cpp +++ b/wxdata/source/scwx/awips/text_product_message.cpp @@ -119,71 +119,11 @@ std::chrono::system_clock::time_point Segment::event_begin() const // If event begin is 000000T0000Z if (eventBegin == std::chrono::system_clock::time_point {}) { - using namespace std::chrono; - // Determine event end from P-VTEC string - system_clock::time_point eventEnd = + std::chrono::system_clock::time_point eventEnd = header_->vtecString_[0].pVtec_.event_end(); - auto endDays = floor(eventEnd); - year_month_day endDate {endDays}; - - // Determine WMO date/time - std::string wmoDateTime = wmoHeader_->date_time(); - - bool wmoDateTimeValid = false; - unsigned int dayOfMonth = 0; - unsigned long beginHour = 0; - unsigned long beginMinute = 0; - - try - { - // WMO date time is in the format DDHHMM - dayOfMonth = - static_cast(std::stoul(wmoDateTime.substr(0, 2))); - beginHour = std::stoul(wmoDateTime.substr(2, 2)); - beginMinute = std::stoul(wmoDateTime.substr(4, 2)); - wmoDateTimeValid = true; - } - catch (const std::exception&) - { - logger_->warn("Malformed WMO date/time: {}", wmoDateTime); - } - - if (wmoDateTimeValid) - { - // Combine end date year and month with WMO date time - eventBegin = - sys_days {endDate.year() / endDate.month() / day {dayOfMonth}} + - hours {beginHour} + minutes {beginMinute}; - - // If the begin date is after the end date, assume the start time - // was the previous month (give a 1 day grace period for expiring - // events in the past) - if (eventBegin > eventEnd + 24h) - { - // If the current end month is January - if (endDate.month() == January) - { - // The begin month must be December of last year - eventBegin = - sys_days { - year {static_cast((endDate.year() - 1y).count())} / - December / day {dayOfMonth}} + - hours {beginHour} + minutes {beginMinute}; - } - else - { - // Back up one month - eventBegin = - sys_days {endDate.year() / - month {static_cast( - (endDate.month() - month {1}).count())} / - day {dayOfMonth}} + - hours {beginHour} + minutes {beginMinute}; - } - } - } + eventBegin = wmoHeader_->GetDateTime(eventEnd); } } diff --git a/wxdata/source/scwx/awips/wmo_header.cpp b/wxdata/source/scwx/awips/wmo_header.cpp index a701e476..27d169e4 100644 --- a/wxdata/source/scwx/awips/wmo_header.cpp +++ b/wxdata/source/scwx/awips/wmo_header.cpp @@ -42,17 +42,26 @@ public: WmoHeaderImpl(const WmoHeaderImpl&&) = delete; WmoHeaderImpl& operator=(const WmoHeaderImpl&&) = delete; + void CalculateAbsoluteDateTime(); + bool ParseDateTime(unsigned int& dayOfMonth, + unsigned long& hour, + unsigned long& minute); + bool operator==(const WmoHeaderImpl& o) const; - std::string sequenceNumber_; - std::string dataType_; - std::string geographicDesignator_; - std::string bulletinId_; - std::string icao_; - std::string dateTime_; - std::string bbbIndicator_; - std::string productCategory_; - std::string productDesignator_; + std::string sequenceNumber_ {}; + std::string dataType_ {}; + std::string geographicDesignator_ {}; + std::string bulletinId_ {}; + std::string icao_ {}; + std::string dateTime_ {}; + std::string bbbIndicator_ {}; + std::string productCategory_ {}; + std::string productDesignator_ {}; + + std::optional dateHint_ {}; + std::optional> + absoluteDateTime_ {}; }; WmoHeader::WmoHeader() : p(std::make_unique()) {} @@ -124,6 +133,71 @@ std::string WmoHeader::product_designator() const return p->productDesignator_; } +std::chrono::sys_time WmoHeader::GetDateTime( + std::optional endTimeHint) +{ + std::chrono::sys_time wmoDateTime {}; + + if (p->absoluteDateTime_.has_value()) + { + wmoDateTime = p->absoluteDateTime_.value(); + } + else if (endTimeHint.has_value()) + { + bool dateTimeValid = false; + unsigned int dayOfMonth = 0; + unsigned long hour = 0; + unsigned long minute = 0; + + dateTimeValid = p->ParseDateTime(dayOfMonth, hour, minute); + + if (dateTimeValid) + { + using namespace std::chrono; + + auto endDays = floor(endTimeHint.value()); + year_month_day endDate {endDays}; + + // Combine end date year and month with WMO date time + wmoDateTime = + sys_days {endDate.year() / endDate.month() / day {dayOfMonth}} + + hours {hour} + minutes {minute}; + + // If the begin date is after the end date, assume the start time + // was the previous month (give a 1 day grace period for expiring + // events in the past) + if (wmoDateTime > endTimeHint.value() + 24h) + { + // If the current end month is January + if (endDate.month() == January) + { + year_month x = year {2024} / December; + sys_days y; + + // The begin month must be December of last year + wmoDateTime = + sys_days { + year {static_cast((endDate.year() - 1y).count())} / + December / day {dayOfMonth}} + + hours {hour} + minutes {minute}; + } + else + { + // Back up one month + wmoDateTime = + sys_days {endDate.year() / + month {static_cast( + (endDate.month() - month {1}).count())} / + day {dayOfMonth}} + + hours {hour} + minutes {minute}; + } + } + } + } + + return wmoDateTime; +} + bool WmoHeader::Parse(std::istream& is) { bool headerValid = true; @@ -224,6 +298,8 @@ bool WmoHeader::Parse(std::istream& is) p->icao_ = wmoTokenList[1]; p->dateTime_ = wmoTokenList[2]; + p->CalculateAbsoluteDateTime(); + if (wmoTokenList.size() == 4) { p->bbbIndicator_ = wmoTokenList[3]; @@ -255,4 +331,60 @@ bool WmoHeader::Parse(std::istream& is) return headerValid; } +void WmoHeader::SetDateHint(std::chrono::year_month dateHint) +{ + p->dateHint_ = dateHint; + p->CalculateAbsoluteDateTime(); +} + +bool WmoHeaderImpl::ParseDateTime(unsigned int& dayOfMonth, + unsigned long& hour, + unsigned long& minute) +{ + bool dateTimeValid = false; + + try + { + // WMO date time is in the format DDHHMM + dayOfMonth = + static_cast(std::stoul(dateTime_.substr(0, 2))); + hour = std::stoul(dateTime_.substr(2, 2)); + minute = std::stoul(dateTime_.substr(4, 2)); + dateTimeValid = true; + } + catch (const std::exception&) + { + logger_->warn("Malformed WMO date/time: {}", dateTime_); + } + + return dateTimeValid; +} + +void WmoHeaderImpl::CalculateAbsoluteDateTime() +{ + bool dateTimeValid = false; + + if (dateHint_.has_value() && !dateTime_.empty()) + { + unsigned int dayOfMonth = 0; + unsigned long hour = 0; + unsigned long minute = 0; + + dateTimeValid = ParseDateTime(dayOfMonth, hour, minute); + + if (dateTimeValid) + { + using namespace std::chrono; + absoluteDateTime_ = sys_days {dateHint_->year() / dateHint_->month() / + day {dayOfMonth}} + + hours {hour} + minutes {minute}; + } + } + + if (!dateTimeValid) + { + absoluteDateTime_.reset(); + } +} + } // namespace scwx::awips From 3998f0fe3686fc0e6657d12d6ec859f9639440ce Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Tue, 18 Feb 2025 12:07:35 -0600 Subject: [PATCH 18/67] Use unique .clang-tidy for test, ignoring checks for magic numbers --- .clang-tidy | 4 ++-- .github/workflows/clang-tidy-review.yml | 2 +- test/.clang-tidy | 16 ++++++++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 test/.clang-tidy diff --git a/.clang-tidy b/.clang-tidy index 3c98e81d..dbf9fbd7 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -6,10 +6,10 @@ Checks: - 'misc-*' - 'modernize-*' - 'performance-*' + - '-bugprone-easily-swappable-parameters' - '-cppcoreguidelines-pro-type-reinterpret-cast' - '-misc-include-cleaner' - '-misc-non-private-member-variables-in-classes' - - '-modernize-use-trailing-return-type' - - '-bugprone-easily-swappable-parameters' - '-modernize-return-braced-init-list' + - '-modernize-use-trailing-return-type' FormatStyle: 'file' diff --git a/.github/workflows/clang-tidy-review.yml b/.github/workflows/clang-tidy-review.yml index c37236d6..a7ec09ff 100644 --- a/.github/workflows/clang-tidy-review.yml +++ b/.github/workflows/clang-tidy-review.yml @@ -126,7 +126,7 @@ jobs: --build_dir='../build' \ --base_dir='${{ github.workspace }}/source' \ --clang_tidy_checks='' \ - --config_file='.clang-tidy' \ + --config_file='' \ --include='*.[ch],*.[ch]xx,*.[chi]pp,*.[ch]++,*.cc,*.hh' \ --exclude='' \ --apt-packages='' \ diff --git a/test/.clang-tidy b/test/.clang-tidy new file mode 100644 index 00000000..d5079a03 --- /dev/null +++ b/test/.clang-tidy @@ -0,0 +1,16 @@ +Checks: + - '-*' + - 'bugprone-*' + - 'clang-analyzer-*' + - 'cppcoreguidelines-*' + - 'misc-*' + - 'modernize-*' + - 'performance-*' + - '-bugprone-easily-swappable-parameters' + - '-cppcoreguidelines-avoid-magic-numbers' + - '-cppcoreguidelines-pro-type-reinterpret-cast' + - '-misc-include-cleaner' + - '-misc-non-private-member-variables-in-classes' + - '-modernize-return-braced-init-list' + - '-modernize-use-trailing-return-type' +FormatStyle: 'file' From 8646c3da6d5403538b5a53f26993074df4c11ce7 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Tue, 18 Feb 2025 12:08:57 -0600 Subject: [PATCH 19/67] Add WMO header test --- test/source/scwx/awips/wmo_header.test.cpp | 130 +++++++++++++++++++++ test/test.cmake | 3 +- 2 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 test/source/scwx/awips/wmo_header.test.cpp diff --git a/test/source/scwx/awips/wmo_header.test.cpp b/test/source/scwx/awips/wmo_header.test.cpp new file mode 100644 index 00000000..bdc4406f --- /dev/null +++ b/test/source/scwx/awips/wmo_header.test.cpp @@ -0,0 +1,130 @@ +#include + +#include + +namespace scwx::awips +{ + +static const std::string logPrefix_ = "scwx::awips::wmo_header.test"; + +static const std::string kWmoHeaderSample_ { + "887\n" + "WFUS54 KOUN 280044\n" + "TOROUN"}; + +TEST(WmoHeader, WmoFields) +{ + std::stringstream ss {kWmoHeaderSample_}; + WmoHeader header; + bool valid = header.Parse(ss); + + EXPECT_EQ(valid, true); + EXPECT_EQ(header.sequence_number(), "887"); + EXPECT_EQ(header.data_type(), "WF"); + EXPECT_EQ(header.geographic_designator(), "US"); + EXPECT_EQ(header.bulletin_id(), "54"); + EXPECT_EQ(header.icao(), "KOUN"); + EXPECT_EQ(header.date_time(), "280044"); + EXPECT_EQ(header.bbb_indicator(), ""); + EXPECT_EQ(header.product_category(), "TOR"); + EXPECT_EQ(header.product_designator(), "OUN"); + EXPECT_EQ(header.GetDateTime(), + std::chrono::sys_time {}); +} + +TEST(WmoHeader, DateHintBeforeParse) +{ + using namespace std::chrono; + + std::stringstream ss {kWmoHeaderSample_}; + WmoHeader header; + + header.SetDateHint(2022y / October); + bool valid = header.Parse(ss); + + EXPECT_EQ(valid, true); + EXPECT_EQ(header.GetDateTime(), + sys_days {2022y / October / 28d} + 0h + 44min); +} + +TEST(WmoHeader, DateHintAfterParse) +{ + using namespace std::chrono; + + std::stringstream ss {kWmoHeaderSample_}; + WmoHeader header; + + bool valid = header.Parse(ss); + header.SetDateHint(2022y / October); + + EXPECT_EQ(valid, true); + EXPECT_EQ(header.GetDateTime(), + sys_days {2022y / October / 28d} + 0h + 44min); +} + +TEST(WmoHeader, EndTimeHintSameMonth) +{ + using namespace std::chrono; + + std::stringstream ss {kWmoHeaderSample_}; + WmoHeader header; + + bool valid = header.Parse(ss); + + auto endTimeHint = sys_days {2022y / October / 29d} + 0h + 0min + 0s; + + EXPECT_EQ(valid, true); + EXPECT_EQ(header.GetDateTime(endTimeHint), + sys_days {2022y / October / 28d} + 0h + 44min); +} + +TEST(WmoHeader, EndTimeHintPreviousMonth) +{ + using namespace std::chrono; + + std::stringstream ss {kWmoHeaderSample_}; + WmoHeader header; + + bool valid = header.Parse(ss); + + auto endTimeHint = sys_days {2022y / October / 27d} + 0h + 0min + 0s; + + EXPECT_EQ(valid, true); + EXPECT_EQ(header.GetDateTime(endTimeHint), + sys_days {2022y / September / 28d} + 0h + 44min); +} + +TEST(WmoHeader, EndTimeHintPreviousYear) +{ + using namespace std::chrono; + + std::stringstream ss {kWmoHeaderSample_}; + WmoHeader header; + + bool valid = header.Parse(ss); + + auto endTimeHint = sys_days {2022y / January / 27d} + 0h + 0min + 0s; + + EXPECT_EQ(valid, true); + EXPECT_EQ(header.GetDateTime(endTimeHint), + sys_days {2021y / December / 28d} + 0h + 44min); +} + +TEST(WmoHeader, EndTimeHintIgnored) +{ + using namespace std::chrono; + + std::stringstream ss {kWmoHeaderSample_}; + WmoHeader header; + + header.SetDateHint(2022y / October); + bool valid = header.Parse(ss); + + auto endTimeHint = sys_days {2020y / January / 1d} + 0h + 0min + 0s; + + EXPECT_EQ(valid, true); + EXPECT_EQ(header.GetDateTime(endTimeHint), + sys_days {2022y / October / 28d} + 0h + 44min); +} + +} // namespace scwx::awips diff --git a/test/test.cmake b/test/test.cmake index 64b6c69e..0ae26b53 100644 --- a/test/test.cmake +++ b/test/test.cmake @@ -12,7 +12,8 @@ set(SRC_AWIPS_TESTS source/scwx/awips/coded_location.test.cpp source/scwx/awips/coded_time_motion_location.test.cpp source/scwx/awips/pvtec.test.cpp source/scwx/awips/text_product_file.test.cpp - source/scwx/awips/ugc.test.cpp) + source/scwx/awips/ugc.test.cpp + source/scwx/awips/wmo_header.test.cpp) set(SRC_COMMON_TESTS source/scwx/common/color_table.test.cpp source/scwx/common/products.test.cpp) set(SRC_GR_TESTS source/scwx/gr/placefile.test.cpp) From b60318c393caf65972129e76f5594080cc57eba1 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Tue, 18 Feb 2025 12:16:45 -0600 Subject: [PATCH 20/67] WMO header clang-tidy fixes --- wxdata/include/scwx/awips/wmo_header.hpp | 7 ++----- wxdata/source/scwx/awips/wmo_header.cpp | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/wxdata/include/scwx/awips/wmo_header.hpp b/wxdata/include/scwx/awips/wmo_header.hpp index f0aed0de..6ca63709 100644 --- a/wxdata/include/scwx/awips/wmo_header.hpp +++ b/wxdata/include/scwx/awips/wmo_header.hpp @@ -4,9 +4,7 @@ #include #include -namespace scwx -{ -namespace awips +namespace scwx::awips { class WmoHeaderImpl; @@ -85,5 +83,4 @@ private: std::unique_ptr p; }; -} // namespace awips -} // namespace scwx +} // namespace scwx::awips diff --git a/wxdata/source/scwx/awips/wmo_header.cpp b/wxdata/source/scwx/awips/wmo_header.cpp index 27d169e4..ddaf06b2 100644 --- a/wxdata/source/scwx/awips/wmo_header.cpp +++ b/wxdata/source/scwx/awips/wmo_header.cpp @@ -18,7 +18,11 @@ namespace scwx::awips static const std::string logPrefix_ = "scwx::awips::wmo_header"; static const auto logger_ = util::Logger::Create(logPrefix_); -static constexpr std::size_t kWmoHeaderMinLineLength_ = 18; +static constexpr std::size_t kWmoHeaderMinLineLength_ = 18; +static constexpr std::size_t kWmoIdentifierLength_ = 6; +static constexpr std::size_t kIcaoLength_ = 4; +static constexpr std::size_t kDateTimeLength_ = 6; +static constexpr std::size_t kAwipsIdentifierLineLength_ = 6; class WmoHeaderImpl { @@ -166,14 +170,12 @@ std::chrono::sys_time WmoHeader::GetDateTime( // If the begin date is after the end date, assume the start time // was the previous month (give a 1 day grace period for expiring // events in the past) + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) if (wmoDateTime > endTimeHint.value() + 24h) { // If the current end month is January if (endDate.month() == January) { - year_month x = year {2024} / December; - sys_days y; - // The begin month must be December of last year wmoDateTime = sys_days { @@ -269,17 +271,17 @@ bool WmoHeader::Parse(std::istream& is) logger_->warn("Invalid number of WMO tokens"); headerValid = false; } - else if (wmoTokenList[0].size() != 6) + else if (wmoTokenList[0].size() != kWmoIdentifierLength_) { logger_->warn("WMO identifier malformed"); headerValid = false; } - else if (wmoTokenList[1].size() != 4) + else if (wmoTokenList[1].size() != kIcaoLength_) { logger_->warn("ICAO malformed"); headerValid = false; } - else if (wmoTokenList[2].size() != 6) + else if (wmoTokenList[2].size() != kDateTimeLength_) { logger_->warn("Date/time malformed"); headerValid = false; @@ -316,7 +318,7 @@ bool WmoHeader::Parse(std::istream& is) if (headerValid) { - if (awipsLine.size() != 6) + if (awipsLine.size() != kAwipsIdentifierLineLength_) { logger_->warn("AWIPS Identifier Line bad size"); headerValid = false; From 163b7039646a0afdfe33b715e449fe1ea6b6a9db Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Tue, 18 Feb 2025 12:28:30 -0600 Subject: [PATCH 21/67] Use constexpr instead of #define where possible in time.cpp --- wxdata/source/scwx/util/time.cpp | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/wxdata/source/scwx/util/time.cpp b/wxdata/source/scwx/util/time.cpp index 7a706224..20c6121d 100644 --- a/wxdata/source/scwx/util/time.cpp +++ b/wxdata/source/scwx/util/time.cpp @@ -60,15 +60,19 @@ std::string TimeString(std::chrono::system_clock::time_point time, using namespace std::chrono; #if (__cpp_lib_chrono >= 201907L) -# define FORMAT_STRING_24_HOUR "{:%Y-%m-%d %H:%M:%S %Z}" -# define FORMAT_STRING_12_HOUR "{:%Y-%m-%d %I:%M:%S %p %Z}" namespace date = std::chrono; namespace df = std; + + static constexpr std::string_view kFormatString24Hour = + "{:%Y-%m-%d %H:%M:%S %Z}"; + static constexpr std::string_view kFormatString12Hour = + "{:%Y-%m-%d %I:%M:%S %p %Z}"; #else -# define FORMAT_STRING_24_HOUR "%Y-%m-%d %H:%M:%S %Z" -# define FORMAT_STRING_12_HOUR "%Y-%m-%d %I:%M:%S %p %Z" using namespace date; namespace df = date; + +# define kFormatString24Hour "%Y-%m-%d %H:%M:%S %Z" +# define kFormatString12Hour "%Y-%m-%d %I:%M:%S %p %Z" #endif auto timeInSeconds = time_point_cast(time); @@ -84,11 +88,11 @@ std::string TimeString(std::chrono::system_clock::time_point time, if (clockFormat == ClockFormat::_24Hour) { - os << df::format(FORMAT_STRING_24_HOUR, zt); + os << df::format(kFormatString24Hour, zt); } else { - os << df::format(FORMAT_STRING_12_HOUR, zt); + os << df::format(kFormatString12Hour, zt); } } catch (const std::exception& ex) @@ -110,11 +114,11 @@ std::string TimeString(std::chrono::system_clock::time_point time, { if (clockFormat == ClockFormat::_24Hour) { - os << df::format(FORMAT_STRING_24_HOUR, timeInSeconds); + os << df::format(kFormatString24Hour, timeInSeconds); } else { - os << df::format(FORMAT_STRING_12_HOUR, timeInSeconds); + os << df::format(kFormatString12Hour, timeInSeconds); } } } From d63c2a3ef9a3633eeeaaea4a903084d808788817 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Tue, 18 Feb 2025 14:09:22 -0600 Subject: [PATCH 22/67] Insert text product messages in chronological order --- .../scwx/qt/manager/text_event_manager.cpp | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp index 0a7c66f5..f0fbaa6c 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp @@ -80,7 +80,8 @@ public: threadPool_.join(); } - void HandleMessage(std::shared_ptr message); + void + HandleMessage(const std::shared_ptr& message); void RefreshAsync(); void Refresh(); @@ -171,7 +172,7 @@ void TextEventManager::LoadFile(const std::string& filename) } void TextEventManager::Impl::HandleMessage( - std::shared_ptr message) + const std::shared_ptr& message) { auto segments = message->segments(); @@ -220,8 +221,23 @@ void TextEventManager::Impl::HandleMessage( // If there was a matching event, and this message has not been stored // (WMO header equivalence check), add the updated message to the existing // event - messageIndex = it->second.size(); - it->second.push_back(message); + + // Determine the chronological sequence of the message. Note, if there + // were no time hints given to the WMO header, this will place the message + // at the end of the vector. + auto insertionPoint = std::upper_bound( + it->second.begin(), + it->second.end(), + message, + [](const std::shared_ptr& a, + const std::shared_ptr& b) { + return a->wmo_header()->GetDateTime() < + b->wmo_header()->GetDateTime(); + }); + + // Insert the message in chronological order + messageIndex = std::distance(it->second.begin(), insertionPoint); + it->second.insert(insertionPoint, message); updated = true; }; From 46972e87694f9ac5badfa5dd6ff29c705ca0ccbd Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Tue, 18 Feb 2025 23:24:27 -0600 Subject: [PATCH 23/67] Formatting iem_api_provider.cpp --- .../source/scwx/provider/iem_api_provider.cpp | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/wxdata/source/scwx/provider/iem_api_provider.cpp b/wxdata/source/scwx/provider/iem_api_provider.cpp index 96d1489e..1d5781f0 100644 --- a/wxdata/source/scwx/provider/iem_api_provider.cpp +++ b/wxdata/source/scwx/provider/iem_api_provider.cpp @@ -32,15 +32,13 @@ public: IemApiProvider::IemApiProvider() : p(std::make_unique()) {} IemApiProvider::~IemApiProvider() = default; -IemApiProvider::IemApiProvider(IemApiProvider&&) noexcept = - default; -IemApiProvider& -IemApiProvider::operator=(IemApiProvider&&) noexcept = default; +IemApiProvider::IemApiProvider(IemApiProvider&&) noexcept = default; +IemApiProvider& IemApiProvider::operator=(IemApiProvider&&) noexcept = default; -std::vector IemApiProvider::ListTextProducts( - std::chrono::sys_time date, - std::optional cccc, - std::optional pil) +std::vector +IemApiProvider::ListTextProducts(std::chrono::sys_time date, + std::optional cccc, + std::optional pil) { using namespace std::chrono; @@ -137,25 +135,23 @@ std::vector IemApiProvider::ListTextProducts( } std::vector> -IemApiProvider::LoadTextProducts( - const std::vector& textProducts) +IemApiProvider::LoadTextProducts(const std::vector& textProducts) { auto parameters = cpr::Parameters {{"nolimit", "true"}}; std::vector> asyncResponses {}; asyncResponses.reserve(textProducts.size()); - + const std::string endpointUrl = kBaseUrl_ + kNwsTextProductEndpoint_; for (auto& productId : textProducts) { asyncResponses.emplace_back( productId, - cpr::GetAsync( - cpr::Url {endpointUrl + productId}, - network::cpr::GetHeader(), - parameters)); + cpr::GetAsync(cpr::Url {endpointUrl + productId}, + network::cpr::GetHeader(), + parameters)); } std::vector> textProductFiles; From a6ba312f6b461397447fe50ab2b48772ef6ba0ef Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Tue, 18 Feb 2025 23:33:58 -0600 Subject: [PATCH 24/67] Provide year/month hint to WMO header parser based on filename --- .../include/scwx/awips/text_product_file.hpp | 2 +- .../source/scwx/awips/text_product_file.cpp | 29 +++++++++++++++++-- .../source/scwx/provider/iem_api_provider.cpp | 3 +- .../scwx/provider/warnings_provider.cpp | 2 +- 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/wxdata/include/scwx/awips/text_product_file.hpp b/wxdata/include/scwx/awips/text_product_file.hpp index 478a93b4..80eda9c8 100644 --- a/wxdata/include/scwx/awips/text_product_file.hpp +++ b/wxdata/include/scwx/awips/text_product_file.hpp @@ -29,7 +29,7 @@ public: std::shared_ptr message(size_t i) const; bool LoadFile(const std::string& filename); - bool LoadData(std::istream& is); + bool LoadData(std::string_view filename, std::istream& is); private: std::unique_ptr p; diff --git a/wxdata/source/scwx/awips/text_product_file.cpp b/wxdata/source/scwx/awips/text_product_file.cpp index 3edc7b2d..cd23516c 100644 --- a/wxdata/source/scwx/awips/text_product_file.cpp +++ b/wxdata/source/scwx/awips/text_product_file.cpp @@ -3,6 +3,8 @@ #include +#include + namespace scwx { namespace awips @@ -59,16 +61,34 @@ bool TextProductFile::LoadFile(const std::string& filename) if (fileValid) { - fileValid = LoadData(f); + fileValid = LoadData(filename, f); } return fileValid; } -bool TextProductFile::LoadData(std::istream& is) +bool TextProductFile::LoadData(std::string_view filename, std::istream& is) { + static constexpr LazyRE2 kDateTimePattern_ = { + R"(((?:19|20)\d{2}))" // Year (YYYY) + R"((0[1-9]|1[0-2]))" // Month (MM) + R"((0[1-9]|[12]\d|3[01]))" // Day (DD) + R"(_?)" // Optional separator (not captured) + R"(([01]\d|2[0-3]))" // Hour (HH) + }; + logger_->trace("Loading Data"); + // Attempt to parse the date from the filename + std::optional yearMonth; + int year {}; + unsigned int month {}; + + if (RE2::PartialMatch(filename, *kDateTimePattern_, &year, &month)) + { + yearMonth = std::chrono::year {year} / std::chrono::month {month}; + } + while (!is.eof()) { std::shared_ptr message = @@ -88,6 +108,11 @@ bool TextProductFile::LoadData(std::istream& is) if (!duplicate) { + if (yearMonth.has_value()) + { + message->wmo_header()->SetDateHint(yearMonth.value()); + } + p->messages_.push_back(message); } } diff --git a/wxdata/source/scwx/provider/iem_api_provider.cpp b/wxdata/source/scwx/provider/iem_api_provider.cpp index 1d5781f0..66cdc5d6 100644 --- a/wxdata/source/scwx/provider/iem_api_provider.cpp +++ b/wxdata/source/scwx/provider/iem_api_provider.cpp @@ -163,10 +163,11 @@ IemApiProvider::LoadTextProducts(const std::vector& textProducts) if (response.status_code == cpr::status::HTTP_OK) { // Load file + auto productId = asyncResponse.first; std::shared_ptr textProductFile { std::make_shared()}; std::istringstream responseBody {response.text}; - if (textProductFile->LoadData(responseBody)) + if (textProductFile->LoadData(productId, responseBody)) { textProductFiles.push_back(textProductFile); } diff --git a/wxdata/source/scwx/provider/warnings_provider.cpp b/wxdata/source/scwx/provider/warnings_provider.cpp index 6ef7e237..f0d6b8dd 100644 --- a/wxdata/source/scwx/provider/warnings_provider.cpp +++ b/wxdata/source/scwx/provider/warnings_provider.cpp @@ -176,7 +176,7 @@ WarningsProvider::LoadUpdatedFiles( std::shared_ptr textProductFile { std::make_shared()}; std::istringstream responseBody {response.text}; - if (textProductFile->LoadData(responseBody)) + if (textProductFile->LoadData(filename, responseBody)) { updatedFiles.push_back(textProductFile); } From d3d98234593a86796a5d06cbfc6508ba3f10a6df Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Tue, 18 Feb 2025 23:34:48 -0600 Subject: [PATCH 25/67] More clang-tidy fixes --- .../include/scwx/awips/text_product_file.hpp | 17 +++++++------ wxdata/include/scwx/awips/wmo_header.hpp | 22 ++++++++--------- .../source/scwx/awips/text_product_file.cpp | 24 +++++++++++-------- 3 files changed, 33 insertions(+), 30 deletions(-) diff --git a/wxdata/include/scwx/awips/text_product_file.hpp b/wxdata/include/scwx/awips/text_product_file.hpp index 80eda9c8..ce164f20 100644 --- a/wxdata/include/scwx/awips/text_product_file.hpp +++ b/wxdata/include/scwx/awips/text_product_file.hpp @@ -5,9 +5,7 @@ #include #include -namespace scwx -{ -namespace awips +namespace scwx::awips { class TextProductFileImpl; @@ -24,16 +22,17 @@ public: TextProductFile(TextProductFile&&) noexcept; TextProductFile& operator=(TextProductFile&&) noexcept; - size_t message_count() const; - std::vector> messages() const; - std::shared_ptr message(size_t i) const; + [[nodiscard]] std::size_t message_count() const; + [[nodiscard]] std::vector> + messages() const; + [[nodiscard]] std::shared_ptr message(size_t i) const; bool LoadFile(const std::string& filename); bool LoadData(std::string_view filename, std::istream& is); private: - std::unique_ptr p; + class Impl; + std::unique_ptr p; }; -} // namespace awips -} // namespace scwx +} // namespace scwx::awips diff --git a/wxdata/include/scwx/awips/wmo_header.hpp b/wxdata/include/scwx/awips/wmo_header.hpp index 6ca63709..f3e24faf 100644 --- a/wxdata/include/scwx/awips/wmo_header.hpp +++ b/wxdata/include/scwx/awips/wmo_header.hpp @@ -34,15 +34,15 @@ public: bool operator==(const WmoHeader& o) const; - std::string sequence_number() const; - std::string data_type() const; - std::string geographic_designator() const; - std::string bulletin_id() const; - std::string icao() const; - std::string date_time() const; - std::string bbb_indicator() const; - std::string product_category() const; - std::string product_designator() const; + [[nodiscard]] std::string sequence_number() const; + [[nodiscard]] std::string data_type() const; + [[nodiscard]] std::string geographic_designator() const; + [[nodiscard]] std::string bulletin_id() const; + [[nodiscard]] std::string icao() const; + [[nodiscard]] std::string date_time() const; + [[nodiscard]] std::string bbb_indicator() const; + [[nodiscard]] std::string product_category() const; + [[nodiscard]] std::string product_designator() const; /** * @brief Get the WMO date/time @@ -54,7 +54,7 @@ public: * @param [in] endTimeHint The optional end time bounds to provide. This is * ignored if a date hint has been provided to determine an absolute date. */ - std::chrono::sys_time GetDateTime( + [[nodiscard]] std::chrono::sys_time GetDateTime( std::optional endTimeHint = std::nullopt); @@ -68,7 +68,7 @@ public: /** * @brief Provide a date hint for the WMO parser * - * The WMO header contains a date/time in the format DDMMSS. The year and + * The WMO header contains a date/time in the format DDHHMM. The year and * month must be derived using another source. The date hint provides the * additional context required to determine the absolute product time. * diff --git a/wxdata/source/scwx/awips/text_product_file.cpp b/wxdata/source/scwx/awips/text_product_file.cpp index cd23516c..9b7dcdb0 100644 --- a/wxdata/source/scwx/awips/text_product_file.cpp +++ b/wxdata/source/scwx/awips/text_product_file.cpp @@ -5,24 +5,29 @@ #include -namespace scwx -{ -namespace awips +namespace scwx::awips { static const std::string logPrefix_ = "scwx::awips::text_product_file"; static const auto logger_ = util::Logger::Create(logPrefix_); -class TextProductFileImpl +class TextProductFile::Impl { public: - explicit TextProductFileImpl() : messages_ {} {}; - ~TextProductFileImpl() = default; + explicit Impl() : messages_ {} {}; + ~Impl() = default; + + Impl(const Impl&) = delete; + Impl& operator=(const Impl&) = delete; + + Impl(Impl&&) = delete; + Impl& operator=(Impl&&) = delete; std::vector> messages_; }; -TextProductFile::TextProductFile() : p(std::make_unique()) +TextProductFile::TextProductFile() : + p(std::make_unique()) { } TextProductFile::~TextProductFile() = default; @@ -97,7 +102,7 @@ bool TextProductFile::LoadData(std::string_view filename, std::istream& is) if (message != nullptr) { - for (auto m : p->messages_) + for (const auto& m : p->messages_) { if (*m->wmo_header().get() == *message->wmo_header().get()) { @@ -125,5 +130,4 @@ bool TextProductFile::LoadData(std::string_view filename, std::istream& is) return !p->messages_.empty(); } -} // namespace awips -} // namespace scwx +} // namespace scwx::awips From d00b2fe06327b3111cd21b17c14ddee0dd863a59 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Tue, 18 Feb 2025 23:55:14 -0600 Subject: [PATCH 26/67] Use const std::string& instead of std::string_view with RE2 to avoid abseil issues --- wxdata/include/scwx/awips/text_product_file.hpp | 2 +- wxdata/source/scwx/awips/text_product_file.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/wxdata/include/scwx/awips/text_product_file.hpp b/wxdata/include/scwx/awips/text_product_file.hpp index ce164f20..b0d1c965 100644 --- a/wxdata/include/scwx/awips/text_product_file.hpp +++ b/wxdata/include/scwx/awips/text_product_file.hpp @@ -28,7 +28,7 @@ public: [[nodiscard]] std::shared_ptr message(size_t i) const; bool LoadFile(const std::string& filename); - bool LoadData(std::string_view filename, std::istream& is); + bool LoadData(const std::string& filename, std::istream& is); private: class Impl; diff --git a/wxdata/source/scwx/awips/text_product_file.cpp b/wxdata/source/scwx/awips/text_product_file.cpp index 9b7dcdb0..96d5503f 100644 --- a/wxdata/source/scwx/awips/text_product_file.cpp +++ b/wxdata/source/scwx/awips/text_product_file.cpp @@ -72,7 +72,7 @@ bool TextProductFile::LoadFile(const std::string& filename) return fileValid; } -bool TextProductFile::LoadData(std::string_view filename, std::istream& is) +bool TextProductFile::LoadData(const std::string& filename, std::istream& is) { static constexpr LazyRE2 kDateTimePattern_ = { R"(((?:19|20)\d{2}))" // Year (YYYY) From 3f83c8e4a9ee0764dfb8bb27e083a3f72c19bac1 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 5 Apr 2025 07:13:49 -0500 Subject: [PATCH 27/67] IEM API provider should use std::string instead of std::string_view for abseil compatibility --- wxdata/source/scwx/provider/iem_api_provider.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wxdata/source/scwx/provider/iem_api_provider.cpp b/wxdata/source/scwx/provider/iem_api_provider.cpp index 66cdc5d6..817fbc2f 100644 --- a/wxdata/source/scwx/provider/iem_api_provider.cpp +++ b/wxdata/source/scwx/provider/iem_api_provider.cpp @@ -139,7 +139,7 @@ IemApiProvider::LoadTextProducts(const std::vector& textProducts) { auto parameters = cpr::Parameters {{"nolimit", "true"}}; - std::vector> + std::vector> asyncResponses {}; asyncResponses.reserve(textProducts.size()); @@ -163,7 +163,7 @@ IemApiProvider::LoadTextProducts(const std::vector& textProducts) if (response.status_code == cpr::status::HTTP_OK) { // Load file - auto productId = asyncResponse.first; + auto& productId = asyncResponse.first; std::shared_ptr textProductFile { std::make_shared()}; std::istringstream responseBody {response.text}; From 16507adbe9ee4e3ae6a0be922d10ebd98b03c782 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 5 Apr 2025 08:14:18 -0500 Subject: [PATCH 28/67] Alert layer should handle alerts by UUID if messages are received out of sequence --- .../scwx/qt/manager/text_event_manager.cpp | 2 +- .../scwx/qt/manager/text_event_manager.hpp | 5 ++- scwx-qt/source/scwx/qt/map/alert_layer.cpp | 35 ++++++++++++++++--- .../scwx/awips/text_product_message.hpp | 3 ++ .../scwx/awips/text_product_message.cpp | 8 +++++ 5 files changed, 46 insertions(+), 7 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp index f0fbaa6c..d76212fd 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp @@ -245,7 +245,7 @@ void TextEventManager::Impl::HandleMessage( if (updated) { - Q_EMIT self_->AlertUpdated(key, messageIndex); + Q_EMIT self_->AlertUpdated(key, messageIndex, message->uuid()); } } diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.hpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.hpp index f97ca223..30748781 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.hpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.hpp @@ -6,6 +6,7 @@ #include #include +#include #include namespace scwx @@ -32,7 +33,9 @@ public: static std::shared_ptr Instance(); signals: - void AlertUpdated(const types::TextEventKey& key, size_t messageIndex); + void AlertUpdated(const types::TextEventKey& key, + std::size_t messageIndex, + boost::uuids::uuid uuid); private: class Impl; diff --git a/scwx-qt/source/scwx/qt/map/alert_layer.cpp b/scwx-qt/source/scwx/qt/map/alert_layer.cpp index 7c5c9db2..77a2332f 100644 --- a/scwx-qt/source/scwx/qt/map/alert_layer.cpp +++ b/scwx-qt/source/scwx/qt/map/alert_layer.cpp @@ -73,8 +73,10 @@ public: connect(textEventManager_.get(), &manager::TextEventManager::AlertUpdated, this, - [this](const types::TextEventKey& key, std::size_t messageIndex) - { HandleAlert(key, messageIndex); }); + [this](const types::TextEventKey& key, + std::size_t messageIndex, + boost::uuids::uuid uuid) + { HandleAlert(key, messageIndex, uuid); }); } ~AlertLayerHandler() { @@ -95,7 +97,9 @@ public: types::TextEventHash> segmentsByKey_ {}; - void HandleAlert(const types::TextEventKey& key, size_t messageIndex); + void HandleAlert(const types::TextEventKey& key, + size_t messageIndex, + boost::uuids::uuid uuid); static AlertLayerHandler& Instance(); @@ -322,7 +326,8 @@ bool IsAlertActive(const std::shared_ptr& segment) } void AlertLayerHandler::HandleAlert(const types::TextEventKey& key, - size_t messageIndex) + size_t messageIndex, + boost::uuids::uuid uuid) { logger_->trace("HandleAlert: {}", key.ToString()); @@ -330,7 +335,27 @@ void AlertLayerHandler::HandleAlert(const types::TextEventKey& key, AlertTypeHash>> alertsUpdated {}; - auto message = textEventManager_->message_list(key).at(messageIndex); + const auto& messageList = textEventManager_->message_list(key); + auto message = messageList.at(messageIndex); + + if (message->uuid() != uuid) + { + // Find message by UUID instead of index, as the message index could have + // changed between the signal being emitted and the handler being called + auto it = std::find_if(messageList.cbegin(), + messageList.cend(), + [&uuid](const auto& message) + { return uuid == message->uuid(); }); + + if (it == messageList.cend()) + { + logger_->warn( + "Could not find alert uuid: {} ({})", key.ToString(), messageIndex); + return; + } + + message = *it; + } // Determine start time for first segment std::chrono::system_clock::time_point segmentBegin {}; diff --git a/wxdata/include/scwx/awips/text_product_message.hpp b/wxdata/include/scwx/awips/text_product_message.hpp index b043494f..6830f91a 100644 --- a/wxdata/include/scwx/awips/text_product_message.hpp +++ b/wxdata/include/scwx/awips/text_product_message.hpp @@ -13,6 +13,8 @@ #include #include +#include + namespace scwx { namespace awips @@ -94,6 +96,7 @@ public: TextProductMessage(TextProductMessage&&) noexcept; TextProductMessage& operator=(TextProductMessage&&) noexcept; + boost::uuids::uuid uuid() const; std::string message_content() const; std::shared_ptr wmo_header() const; std::vector mnd_header() const; diff --git a/wxdata/source/scwx/awips/text_product_message.cpp b/wxdata/source/scwx/awips/text_product_message.cpp index 9674b325..f8716b2b 100644 --- a/wxdata/source/scwx/awips/text_product_message.cpp +++ b/wxdata/source/scwx/awips/text_product_message.cpp @@ -9,6 +9,7 @@ #include #include +#include #include namespace scwx::awips @@ -53,6 +54,8 @@ public: TextProductMessageImpl(const TextProductMessageImpl&&) = delete; TextProductMessageImpl& operator=(const TextProductMessageImpl&&) = delete; + boost::uuids::uuid uuid_ {boost::uuids::random_generator()()}; + std::string messageContent_; std::shared_ptr wmoHeader_; std::vector mndHeader_; @@ -70,6 +73,11 @@ TextProductMessage::TextProductMessage(TextProductMessage&&) noexcept = default; TextProductMessage& TextProductMessage::operator=(TextProductMessage&&) noexcept = default; +boost::uuids::uuid TextProductMessage::uuid() const +{ + return p->uuid_; +} + std::string TextProductMessage::message_content() const { return p->messageContent_; From 1e1422a3dd8ff7172afdb174c808674162d5e6c0 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 5 Apr 2025 08:19:49 -0500 Subject: [PATCH 29/67] Only handle the most recent message for each event in the alert model --- scwx-qt/source/scwx/qt/model/alert_model.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scwx-qt/source/scwx/qt/model/alert_model.cpp b/scwx-qt/source/scwx/qt/model/alert_model.cpp index fed7cc17..516dbfae 100644 --- a/scwx-qt/source/scwx/qt/model/alert_model.cpp +++ b/scwx-qt/source/scwx/qt/model/alert_model.cpp @@ -335,8 +335,15 @@ void AlertModel::HandleAlert(const types::TextEventKey& alertKey, double distanceInMeters; - // Get the most recent segment for the event auto alertMessages = p->textEventManager_->message_list(alertKey); + + // Skip alert if this is not the most recent message + if (messageIndex + 1 < alertMessages.size()) + { + return; + } + + // Get the most recent segment for the event std::shared_ptr alertSegment = alertMessages[messageIndex]->segments().back(); From e4fc13aa929cbe8a42918c0f8d3d4b2432fe0583 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 5 Apr 2025 08:37:51 -0500 Subject: [PATCH 30/67] Add an option for enabling/disabling the release console at build time --- scwx-qt/scwx-qt.cmake | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scwx-qt/scwx-qt.cmake b/scwx-qt/scwx-qt.cmake index 89b31011..1628607a 100644 --- a/scwx-qt/scwx-qt.cmake +++ b/scwx-qt/scwx-qt.cmake @@ -11,6 +11,8 @@ set(CMAKE_AUTORCC ON) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) +OPTION(SCWX_DISABLE_CONSOLE "Disables the Windows console in release mode" ON) + find_package(Boost) find_package(Fontconfig) find_package(geographiclib) @@ -615,7 +617,9 @@ set_target_properties(scwx-qt_update_radar_sites PROPERTIES FOLDER generate) if (WIN32) set(APP_ICON_RESOURCE_WINDOWS ${RESOURCE_OUTPUT}) qt_add_executable(supercell-wx ${EXECUTABLE_SOURCES} ${APP_ICON_RESOURCE_WINDOWS}) - set_target_properties(supercell-wx PROPERTIES WIN32_EXECUTABLE $,TRUE,FALSE>) + if (SCWX_DISABLE_CONSOLE) + set_target_properties(supercell-wx PROPERTIES WIN32_EXECUTABLE $,TRUE,FALSE>) + endif() else() qt_add_executable(supercell-wx ${EXECUTABLE_SOURCES}) endif() From cc54e4d834f264955bdc558476b0783a0fdca40b Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 5 Apr 2025 20:48:36 -0500 Subject: [PATCH 31/67] Load archived warnings when making a timeline selection --- scwx-qt/source/scwx/qt/main/main_window.cpp | 1 + .../scwx/qt/manager/text_event_manager.cpp | 55 ++++++++++++++++++- .../scwx/qt/manager/text_event_manager.hpp | 2 + 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/scwx-qt/source/scwx/qt/main/main_window.cpp b/scwx-qt/source/scwx/qt/main/main_window.cpp index 71f4d00c..d213b4fd 100644 --- a/scwx-qt/source/scwx/qt/main/main_window.cpp +++ b/scwx-qt/source/scwx/qt/main/main_window.cpp @@ -1018,6 +1018,7 @@ void MainWindowImpl::ConnectAnimationSignals() for (auto map : maps_) { map->SelectTime(dateTime); + textEventManager_->SelectTime(dateTime); QMetaObject::invokeMethod( map, static_cast(&QWidget::update)); } diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp index d76212fd..dffdb6bf 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -27,6 +28,9 @@ static constexpr std::chrono::hours kInitialLoadHistoryDuration_ = static constexpr std::chrono::hours kDefaultLoadHistoryDuration_ = std::chrono::hours {1}; +static const std::array kPils_ = { + "TOR", "SVR", "SVS", "FFW", "FFS"}; + class TextEventManager::Impl { public: @@ -82,10 +86,17 @@ public: void HandleMessage(const std::shared_ptr& message); + void LoadArchive(std::chrono::sys_days date, const std::string& pil); + void LoadArchives(std::chrono::sys_days date); void RefreshAsync(); void Refresh(); - boost::asio::thread_pool threadPool_ {1u}; + // Thread pool sized for: + // - Live Refresh (1x) + // - Archive Loading (15x) + // - 3 day window (3x) + // - TOR, SVR, SVS, FFW, FFS (5x) + boost::asio::thread_pool threadPool_ {16u}; TextEventManager* self_; @@ -98,6 +109,8 @@ public: textEventMap_; std::shared_mutex textEventMutex_; + std::unique_ptr iemApiProvider_ { + std::make_unique()}; std::shared_ptr warningsProvider_ {nullptr}; std::chrono::hours loadHistoryDuration_ {kInitialLoadHistoryDuration_}; std::chrono::sys_time prevLoadTime_ {}; @@ -171,6 +184,20 @@ void TextEventManager::LoadFile(const std::string& filename) }); } +void TextEventManager::SelectTime( + std::chrono::system_clock::time_point dateTime) +{ + const auto today = std::chrono::floor(dateTime); + const auto yesterday = today - std::chrono::days {1}; + const auto tomorrow = today + std::chrono::days {1}; + const auto dates = {yesterday, today, tomorrow}; + + for (auto& date : dates) + { + p->LoadArchives(date); + } +} + void TextEventManager::Impl::HandleMessage( const std::shared_ptr& message) { @@ -249,6 +276,32 @@ void TextEventManager::Impl::HandleMessage( } } +void TextEventManager::Impl::LoadArchive(std::chrono::sys_days date, + const std::string& pil) +{ + const auto& productIds = iemApiProvider_->ListTextProducts(date, {}, pil); + const auto& products = iemApiProvider_->LoadTextProducts(productIds); + + for (auto& product : products) + { + const auto& messages = product->messages(); + + for (auto& message : messages) + { + HandleMessage(message); + } + } +} + +void TextEventManager::Impl::LoadArchives(std::chrono::sys_days date) +{ + for (auto& pil : kPils_) + { + boost::asio::post(threadPool_, + [this, date, &pil]() { LoadArchive(date, pil); }); + } +} + void TextEventManager::Impl::RefreshAsync() { boost::asio::post(threadPool_, diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.hpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.hpp index 30748781..312dece0 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.hpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.hpp @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -29,6 +30,7 @@ public: message_list(const types::TextEventKey& key) const; void LoadFile(const std::string& filename); + void SelectTime(std::chrono::system_clock::time_point dateTime); static std::shared_ptr Instance(); From 33cfd4bc0eedd69bd9cb3dbba0e073658209b8d1 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sun, 6 Apr 2025 00:11:09 -0500 Subject: [PATCH 32/67] Don't reload archive text products that have already been loaded --- .../scwx/qt/manager/text_event_manager.cpp | 74 +++++++++++++++++-- .../scwx/provider/iem_api_provider.test.cpp | 11 +-- .../scwx/provider/iem_api_provider.hpp | 4 +- .../source/scwx/provider/iem_api_provider.cpp | 13 +++- 4 files changed, 87 insertions(+), 15 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp index dffdb6bf..45cf24a6 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp @@ -5,7 +5,10 @@ #include #include #include +#include +#include +#include #include #include @@ -90,6 +93,7 @@ public: void LoadArchives(std::chrono::sys_days date); void RefreshAsync(); void Refresh(); + void UpdateArchiveDates(std::chrono::sys_days date); // Thread pool sized for: // - Live Refresh (1x) @@ -112,9 +116,18 @@ public: std::unique_ptr iemApiProvider_ { std::make_unique()}; std::shared_ptr warningsProvider_ {nullptr}; + std::chrono::hours loadHistoryDuration_ {kInitialLoadHistoryDuration_}; std::chrono::sys_time prevLoadTime_ {}; + std::mutex archiveMutex_ {}; + std::list archiveDates_ {}; + std::map< + std::chrono::sys_days, + std::unordered_map>>> + archiveMap_; + boost::uuids::uuid warningsProviderChangedCallbackUuid_ {}; }; @@ -187,10 +200,12 @@ void TextEventManager::LoadFile(const std::string& filename) void TextEventManager::SelectTime( std::chrono::system_clock::time_point dateTime) { + logger_->trace("Select Time: {}", util::TimeString(dateTime)); + const auto today = std::chrono::floor(dateTime); const auto yesterday = today - std::chrono::days {1}; const auto tomorrow = today + std::chrono::days {1}; - const auto dates = {yesterday, today, tomorrow}; + const auto dates = {today, yesterday, tomorrow}; for (auto& date : dates) { @@ -279,22 +294,56 @@ void TextEventManager::Impl::HandleMessage( void TextEventManager::Impl::LoadArchive(std::chrono::sys_days date, const std::string& pil) { - const auto& productIds = iemApiProvider_->ListTextProducts(date, {}, pil); - const auto& products = iemApiProvider_->LoadTextProducts(productIds); - - for (auto& product : products) + std::unique_lock lock {archiveMutex_}; + auto& dateArchive = archiveMap_[date]; + if (dateArchive.contains(pil)) { - const auto& messages = product->messages(); + // Don't reload data that has already been loaded + return; + } + lock.unlock(); - for (auto& message : messages) + logger_->debug("Load Archive: {}, {}", util::TimeString(date), pil); + + // Query for products + const auto& productIds = iemApiProvider_->ListTextProducts(date, {}, pil); + + if (productIds.has_value()) + { + logger_->debug("Loading {} {} products", productIds.value().size(), pil); + + // Load listed products + auto products = iemApiProvider_->LoadTextProducts(productIds.value()); + + for (auto& product : products) { - HandleMessage(message); + const auto& messages = product->messages(); + + for (auto& message : messages) + { + HandleMessage(message); + } } + + lock.lock(); + + // Ensure the archive map still contains the date, and has not been pruned + if (archiveMap_.contains(date)) + { + // Store the products associated with the PIL in the archive + dateArchive.try_emplace(pil, std::move(products)); + } + + lock.unlock(); } } void TextEventManager::Impl::LoadArchives(std::chrono::sys_days date) { + logger_->trace("Load Archives: {}", util::TimeString(date)); + + UpdateArchiveDates(date); + for (auto& pil : kPils_) { boost::asio::post(threadPool_, @@ -380,6 +429,15 @@ void TextEventManager::Impl::Refresh() }); } +void TextEventManager::Impl::UpdateArchiveDates(std::chrono::sys_days date) +{ + std::unique_lock lock {archiveMutex_}; + + // Remove any existing occurrences of day, and add to the back of the list + archiveDates_.remove(date); + archiveDates_.push_back(date); +} + std::shared_ptr TextEventManager::Instance() { static std::weak_ptr textEventManagerReference_ {}; diff --git a/test/source/scwx/provider/iem_api_provider.test.cpp b/test/source/scwx/provider/iem_api_provider.test.cpp index 68cb59ae..854f0d60 100644 --- a/test/source/scwx/provider/iem_api_provider.test.cpp +++ b/test/source/scwx/provider/iem_api_provider.test.cpp @@ -18,15 +18,16 @@ TEST(IemApiProviderTest, ListTextProducts) auto torProducts = provider.ListTextProducts(date, {}, "TOR"); - EXPECT_EQ(torProducts.size(), 35); + ASSERT_EQ(torProducts.has_value(), true); + EXPECT_EQ(torProducts.value().size(), 35); - if (torProducts.size() >= 1) + if (torProducts.value().size() >= 1) { - EXPECT_EQ(torProducts.at(0), "202303250016-KMEG-WFUS54-TORMEG"); + EXPECT_EQ(torProducts.value().at(0), "202303250016-KMEG-WFUS54-TORMEG"); } - if (torProducts.size() >= 35) + if (torProducts.value().size() >= 35) { - EXPECT_EQ(torProducts.at(34), "202303252015-KFFC-WFUS52-TORFFC"); + EXPECT_EQ(torProducts.value().at(34), "202303252015-KFFC-WFUS52-TORFFC"); } } diff --git a/wxdata/include/scwx/provider/iem_api_provider.hpp b/wxdata/include/scwx/provider/iem_api_provider.hpp index 33a82bcf..a0183cd4 100644 --- a/wxdata/include/scwx/provider/iem_api_provider.hpp +++ b/wxdata/include/scwx/provider/iem_api_provider.hpp @@ -5,6 +5,8 @@ #include #include +#include + namespace scwx::provider { @@ -23,7 +25,7 @@ public: IemApiProvider(IemApiProvider&&) noexcept; IemApiProvider& operator=(IemApiProvider&&) noexcept; - static std::vector + static boost::outcome_v2::result> ListTextProducts(std::chrono::sys_time date, std::optional cccc = {}, std::optional pil = {}); diff --git a/wxdata/source/scwx/provider/iem_api_provider.cpp b/wxdata/source/scwx/provider/iem_api_provider.cpp index 817fbc2f..d5752fcb 100644 --- a/wxdata/source/scwx/provider/iem_api_provider.cpp +++ b/wxdata/source/scwx/provider/iem_api_provider.cpp @@ -35,7 +35,7 @@ IemApiProvider::~IemApiProvider() = default; IemApiProvider::IemApiProvider(IemApiProvider&&) noexcept = default; IemApiProvider& IemApiProvider::operator=(IemApiProvider&&) noexcept = default; -std::vector +boost::outcome_v2::result> IemApiProvider::ListTextProducts(std::chrono::sys_time date, std::optional cccc, std::optional pil) @@ -93,6 +93,8 @@ IemApiProvider::ListTextProducts(std::chrono::sys_time date, { // Unexpected bad response logger_->warn("Error parsing JSON: {}", ex.what()); + return boost::system::errc::make_error_code( + boost::system::errc::bad_message); } } else if (response.status_code == cpr::status::HTTP_BAD_REQUEST && @@ -109,6 +111,9 @@ IemApiProvider::ListTextProducts(std::chrono::sys_time date, // Unexpected bad response logger_->warn("Error parsing bad response: {}", ex.what()); } + + return boost::system::errc::make_error_code( + boost::system::errc::invalid_argument); } else if (response.status_code == cpr::status::HTTP_UNPROCESSABLE_ENTITY && json != nullptr) @@ -125,10 +130,16 @@ IemApiProvider::ListTextProducts(std::chrono::sys_time date, // Unexpected bad response logger_->warn("Error parsing validation error: {}", ex.what()); } + + return boost::system::errc::make_error_code( + boost::system::errc::no_message_available); } else { logger_->warn("Could not list text products: {}", response.status_line); + + return boost::system::errc::make_error_code( + boost::system::errc::no_message); } return textProducts; From 02ec27dd2ff0995c290c0a5850b2ab9299434797 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sun, 6 Apr 2025 00:16:05 -0500 Subject: [PATCH 33/67] Ignore default date/time selections for archive warnings --- scwx-qt/source/scwx/qt/manager/text_event_manager.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp index 45cf24a6..b3d0fd3a 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp @@ -200,6 +200,12 @@ void TextEventManager::LoadFile(const std::string& filename) void TextEventManager::SelectTime( std::chrono::system_clock::time_point dateTime) { + if (dateTime == std::chrono::system_clock::time_point {}) + { + // Ignore a default date/time selection + return; + } + logger_->trace("Select Time: {}", util::TimeString(dateTime)); const auto today = std::chrono::floor(dateTime); From 53ade7fc53031777625ae6e3d39b8cf95a91acb1 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sun, 6 Apr 2025 00:33:51 -0500 Subject: [PATCH 34/67] Don't load archived text products for days that have full coverage of live warning data --- .../source/scwx/qt/manager/text_event_manager.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp index b3d0fd3a..bac8b0e7 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp @@ -119,6 +119,7 @@ public: std::chrono::hours loadHistoryDuration_ {kInitialLoadHistoryDuration_}; std::chrono::sys_time prevLoadTime_ {}; + std::chrono::sys_days archiveLimit_ {}; std::mutex archiveMutex_ {}; std::list archiveDates_ {}; @@ -215,7 +216,11 @@ void TextEventManager::SelectTime( for (auto& date : dates) { - p->LoadArchives(date); + if (p->archiveLimit_ == std::chrono::sys_days {} || + date < p->archiveLimit_) + { + p->LoadArchives(date); + } } } @@ -399,6 +404,11 @@ void TextEventManager::Impl::Refresh() startTime = std::min(startTime, prevLoadTime_); } + if (archiveLimit_ == std::chrono::sys_days {}) + { + archiveLimit_ = std::chrono::ceil(startTime); + } + auto updatedFiles = warningsProvider->LoadUpdatedFiles(startTime); // Store the load time and reset the load history duration From 05ff080d789ef717b0b2c6ffd167cc8110978214 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sun, 6 Apr 2025 15:18:36 -0500 Subject: [PATCH 35/67] Allow a 1 character bulletin ID in the WMO header --- wxdata/source/scwx/awips/wmo_header.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/wxdata/source/scwx/awips/wmo_header.cpp b/wxdata/source/scwx/awips/wmo_header.cpp index ddaf06b2..d7da0818 100644 --- a/wxdata/source/scwx/awips/wmo_header.cpp +++ b/wxdata/source/scwx/awips/wmo_header.cpp @@ -19,7 +19,8 @@ static const std::string logPrefix_ = "scwx::awips::wmo_header"; static const auto logger_ = util::Logger::Create(logPrefix_); static constexpr std::size_t kWmoHeaderMinLineLength_ = 18; -static constexpr std::size_t kWmoIdentifierLength_ = 6; +static constexpr std::size_t kWmoIdentifierLengthMin_ = 5; +static constexpr std::size_t kWmoIdentifierLengthMax_ = 6; static constexpr std::size_t kIcaoLength_ = 4; static constexpr std::size_t kDateTimeLength_ = 6; static constexpr std::size_t kAwipsIdentifierLineLength_ = 6; @@ -271,7 +272,8 @@ bool WmoHeader::Parse(std::istream& is) logger_->warn("Invalid number of WMO tokens"); headerValid = false; } - else if (wmoTokenList[0].size() != kWmoIdentifierLength_) + else if (wmoTokenList[0].size() < kWmoIdentifierLengthMin_ || + wmoTokenList[0].size() > kWmoIdentifierLengthMax_) { logger_->warn("WMO identifier malformed"); headerValid = false; @@ -296,9 +298,9 @@ bool WmoHeader::Parse(std::istream& is) { p->dataType_ = wmoTokenList[0].substr(0, 2); p->geographicDesignator_ = wmoTokenList[0].substr(2, 2); - p->bulletinId_ = wmoTokenList[0].substr(4, 2); - p->icao_ = wmoTokenList[1]; - p->dateTime_ = wmoTokenList[2]; + p->bulletinId_ = wmoTokenList[0].substr(4, wmoTokenList[0].size() - 4); + p->icao_ = wmoTokenList[1]; + p->dateTime_ = wmoTokenList[2]; p->CalculateAbsoluteDateTime(); From b117d2088a8fd077439df221f99ca06bbe7e16ea Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sun, 6 Apr 2025 15:35:00 -0500 Subject: [PATCH 36/67] Add missing date includes to IEM API provider --- wxdata/source/scwx/provider/iem_api_provider.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wxdata/source/scwx/provider/iem_api_provider.cpp b/wxdata/source/scwx/provider/iem_api_provider.cpp index d5752fcb..f2704e54 100644 --- a/wxdata/source/scwx/provider/iem_api_provider.cpp +++ b/wxdata/source/scwx/provider/iem_api_provider.cpp @@ -7,6 +7,10 @@ #include #include +#if (__cpp_lib_chrono < 201907L) +# include +#endif + namespace scwx::provider { From 1bdfdcafad0380ae15a8f523193ce75fe4b991e2 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sun, 6 Apr 2025 23:05:18 -0500 Subject: [PATCH 37/67] Missing AWIPS Identifier Line in WMO header should not be treated as an error --- wxdata/source/scwx/awips/wmo_header.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/wxdata/source/scwx/awips/wmo_header.cpp b/wxdata/source/scwx/awips/wmo_header.cpp index d7da0818..fad1231b 100644 --- a/wxdata/source/scwx/awips/wmo_header.cpp +++ b/wxdata/source/scwx/awips/wmo_header.cpp @@ -228,6 +228,7 @@ bool WmoHeader::Parse(std::istream& is) } } + auto awipsLinePos = is.tellg(); util::getline(is, awipsLine); if (is.eof()) @@ -322,8 +323,12 @@ bool WmoHeader::Parse(std::istream& is) { if (awipsLine.size() != kAwipsIdentifierLineLength_) { - logger_->warn("AWIPS Identifier Line bad size"); - headerValid = false; + // Older products may be missing an AWIPS Identifier Line + logger_->trace("AWIPS Identifier Line bad size"); + + is.seekg(awipsLinePos); + p->productCategory_ = ""; + p->productDesignator_ = ""; } else { From 33c73ef0e22333e42d7dcedae180c0348238c52d Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 12 Apr 2025 22:40:36 -0500 Subject: [PATCH 38/67] Add range-v3 dependency --- ACKNOWLEDGEMENTS.md | 1 + conanfile.py | 1 + wxdata/wxdata.cmake | 2 ++ 3 files changed, 4 insertions(+) diff --git a/ACKNOWLEDGEMENTS.md b/ACKNOWLEDGEMENTS.md index 4aec61d2..6f7cd4ef 100644 --- a/ACKNOWLEDGEMENTS.md +++ b/ACKNOWLEDGEMENTS.md @@ -36,6 +36,7 @@ Supercell Wx uses code from the following dependencies: | [OpenSSL](https://www.openssl.org/) | [OpenSSL License](https://spdx.org/licenses/OpenSSL.html) | | [Qt](https://www.qt.io/) | [GNU Lesser General Public License v3.0 only](https://spdx.org/licenses/LGPL-3.0-only.html) | Qt Core, Qt GUI, Qt Multimedia, Qt Network, Qt OpenGL, Qt Positioning, Qt Serial Port, Qt SQL, Qt SVG, Qt Widgets
Additional Licenses: https://doc.qt.io/qt-6/licenses-used-in-qt.html | | [qt6ct](https://github.com/trialuser02/qt6ct) | [BSD 2-Clause "Simplified" License](https://spdx.org/licenses/BSD-2-Clause.html) | +| [range-v3](https://github.com/ericniebler/range-v3) | [Boost Software License 1.0](https://spdx.org/licenses/BSL-1.0.html)
[MIT License](https://spdx.org/licenses/MIT.html)
[Stepanov and McJones, "Elements of Programming" license](https://github.com/ericniebler/range-v3/tree/0.12.0?tab=License-1-ov-file)
[SGI C++ Standard Template Library license](https://github.com/ericniebler/range-v3/tree/0.12.0?tab=License-1-ov-file) | | [re2](https://github.com/google/re2) | [BSD 3-Clause "New" or "Revised" License](https://spdx.org/licenses/BSD-3-Clause.html) | | [spdlog](https://github.com/gabime/spdlog) | [MIT License](https://spdx.org/licenses/MIT.html) | | [SQLite](https://www.sqlite.org/) | Public Domain | diff --git a/conanfile.py b/conanfile.py index e68c9f39..ea9c6ab9 100644 --- a/conanfile.py +++ b/conanfile.py @@ -18,6 +18,7 @@ class SupercellWxConan(ConanFile): "libpng/1.6.47", "libxml2/2.13.6", "openssl/3.4.1", + "range-v3/0.12.0", "re2/20240702", "spdlog/1.15.1", "sqlite3/3.49.1", diff --git a/wxdata/wxdata.cmake b/wxdata/wxdata.cmake index 468f124b..32db05ab 100644 --- a/wxdata/wxdata.cmake +++ b/wxdata/wxdata.cmake @@ -6,6 +6,7 @@ find_package(Boost) find_package(cpr) find_package(LibXml2) find_package(OpenSSL) +find_package(range-v3) find_package(re2) find_package(spdlog) @@ -303,6 +304,7 @@ target_link_libraries(wxdata PUBLIC aws-cpp-sdk-core cpr::cpr LibXml2::LibXml2 OpenSSL::Crypto + range-v3::range-v3 re2::re2 spdlog::spdlog units::units) From 7a8a0302e0407a3c673f5efcfbe2db9521779b2d Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 12 Apr 2025 22:41:23 -0500 Subject: [PATCH 39/67] Provide interface to request multiple text product lists in parallel --- .../scwx/provider/iem_api_provider.hpp | 5 + .../source/scwx/provider/iem_api_provider.cpp | 181 +++++++++++------- 2 files changed, 118 insertions(+), 68 deletions(-) diff --git a/wxdata/include/scwx/provider/iem_api_provider.hpp b/wxdata/include/scwx/provider/iem_api_provider.hpp index a0183cd4..7b568d83 100644 --- a/wxdata/include/scwx/provider/iem_api_provider.hpp +++ b/wxdata/include/scwx/provider/iem_api_provider.hpp @@ -4,6 +4,7 @@ #include #include +#include #include @@ -29,6 +30,10 @@ public: ListTextProducts(std::chrono::sys_time date, std::optional cccc = {}, std::optional pil = {}); + static boost::outcome_v2::result> + ListTextProducts(std::vector> dates, + std::vector ccccs = {}, + std::vector pils = {}); static std::vector> LoadTextProducts(const std::vector& textProducts); diff --git a/wxdata/source/scwx/provider/iem_api_provider.cpp b/wxdata/source/scwx/provider/iem_api_provider.cpp index f2704e54..00fff608 100644 --- a/wxdata/source/scwx/provider/iem_api_provider.cpp +++ b/wxdata/source/scwx/provider/iem_api_provider.cpp @@ -6,6 +6,7 @@ #include #include +#include #if (__cpp_lib_chrono < 201907L) # include @@ -41,8 +42,25 @@ IemApiProvider& IemApiProvider::operator=(IemApiProvider&&) noexcept = default; boost::outcome_v2::result> IemApiProvider::ListTextProducts(std::chrono::sys_time date, - std::optional cccc, - std::optional pil) + std::optional optionalCccc, + std::optional optionalPil) +{ + std::string_view cccc = + optionalCccc.has_value() ? optionalCccc.value() : std::string_view {}; + std::string_view pil = + optionalPil.has_value() ? optionalPil.value() : std::string_view {}; + + return ListTextProducts( + std::vector> {date}, + {cccc}, + {pil}); +} + +boost::outcome_v2::result> +IemApiProvider::ListTextProducts( + std::vector> dates, + std::vector ccccs, + std::vector pils) { using namespace std::chrono; @@ -57,93 +75,120 @@ IemApiProvider::ListTextProducts(std::chrono::sys_time date, # define kDateFormat "%Y-%m-%d" #endif - auto parameters = cpr::Parameters {{"date", df::format(kDateFormat, date)}}; - - // WMO Source Code - if (cccc.has_value()) + if (ccccs.empty()) { - parameters.Add({"cccc", std::string {cccc.value()}}); + ccccs.push_back({}); } - // AFOS / AWIPS ID / 3-6 length identifier - if (pil.has_value()) + if (pils.empty()) { - parameters.Add({"pil", std::string {pil.value()}}); + pils.push_back({}); } - auto response = - cpr::Get(cpr::Url {kBaseUrl_ + kListNwsTextProductsEndpoint_}, - network::cpr::GetHeader(), - parameters); - boost::json::value json = util::json::ReadJsonString(response.text); + std::vector responses {}; + + for (const auto& [date, cccc, pil] : + ranges::views::cartesian_product(dates, ccccs, pils)) + { + auto parameters = + cpr::Parameters {{"date", df::format(kDateFormat, date)}}; + + // WMO Source Code + if (!cccc.empty()) + { + parameters.Add({"cccc", std::string {cccc}}); + } + + // AFOS / AWIPS ID / 3-6 length identifier + if (!pil.empty()) + { + parameters.Add({"pil", std::string {pil}}); + } + + responses.emplace_back( + cpr::GetAsync(cpr::Url {kBaseUrl_ + kListNwsTextProductsEndpoint_}, + network::cpr::GetHeader(), + parameters)); + } std::vector textProducts {}; - if (response.status_code == cpr::status::HTTP_OK) + for (auto& asyncResponse : responses) { - try - { - // Get AFOS list from response - auto entries = boost::json::value_to(json); + auto response = asyncResponse.get(); - for (auto& entry : entries.data_) + boost::json::value json = util::json::ReadJsonString(response.text); + + if (response.status_code == cpr::status::HTTP_OK) + { + try { - textProducts.push_back(entry.productId_); + // Get AFOS list from response + auto entries = boost::json::value_to(json); + + for (auto& entry : entries.data_) + { + textProducts.push_back(entry.productId_); + } + + logger_->trace("Found {} products", entries.data_.size()); + } + catch (const std::exception& ex) + { + // Unexpected bad response + logger_->warn("Error parsing JSON: {}", ex.what()); + return boost::system::errc::make_error_code( + boost::system::errc::bad_message); + } + } + else if (response.status_code == cpr::status::HTTP_BAD_REQUEST && + json != nullptr) + { + try + { + // Log bad request details + auto badRequest = + boost::json::value_to(json); + logger_->warn("ListTextProducts bad request: {}", + badRequest.detail_); + } + catch (const std::exception& ex) + { + // Unexpected bad response + logger_->warn("Error parsing bad response: {}", ex.what()); } - logger_->trace("Found {} products", entries.data_.size()); - } - catch (const std::exception& ex) - { - // Unexpected bad response - logger_->warn("Error parsing JSON: {}", ex.what()); return boost::system::errc::make_error_code( - boost::system::errc::bad_message); + boost::system::errc::invalid_argument); } - } - else if (response.status_code == cpr::status::HTTP_BAD_REQUEST && - json != nullptr) - { - try + else if (response.status_code == cpr::status::HTTP_UNPROCESSABLE_ENTITY && + json != nullptr) { - // Log bad request details - auto badRequest = boost::json::value_to(json); - logger_->warn("ListTextProducts bad request: {}", badRequest.detail_); - } - catch (const std::exception& ex) - { - // Unexpected bad response - logger_->warn("Error parsing bad response: {}", ex.what()); - } + try + { + // Log validation error details + auto error = + boost::json::value_to(json); + logger_->warn("ListTextProducts validation error: {}", + error.detail_.at(0).msg_); + } + catch (const std::exception& ex) + { + // Unexpected bad response + logger_->warn("Error parsing validation error: {}", ex.what()); + } - return boost::system::errc::make_error_code( - boost::system::errc::invalid_argument); - } - else if (response.status_code == cpr::status::HTTP_UNPROCESSABLE_ENTITY && - json != nullptr) - { - try - { - // Log validation error details - auto error = boost::json::value_to(json); - logger_->warn("ListTextProducts validation error: {}", - error.detail_.at(0).msg_); + return boost::system::errc::make_error_code( + boost::system::errc::no_message_available); } - catch (const std::exception& ex) + else { - // Unexpected bad response - logger_->warn("Error parsing validation error: {}", ex.what()); + logger_->warn("Could not list text products: {}", + response.status_line); + + return boost::system::errc::make_error_code( + boost::system::errc::no_message); } - - return boost::system::errc::make_error_code( - boost::system::errc::no_message_available); - } - else - { - logger_->warn("Could not list text products: {}", response.status_line); - - return boost::system::errc::make_error_code( - boost::system::errc::no_message); } return textProducts; From e82fa93fb0e2ab1fcb72ad66785db8eae899ff30 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sun, 13 Apr 2025 01:20:52 -0500 Subject: [PATCH 40/67] Use ranges instead of vectors for listing text products --- .../scwx/provider/iem_api_provider.hpp | 20 +++++++++--- .../source/scwx/provider/iem_api_provider.cpp | 31 ++++++++++++------- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/wxdata/include/scwx/provider/iem_api_provider.hpp b/wxdata/include/scwx/provider/iem_api_provider.hpp index 7b568d83..316890df 100644 --- a/wxdata/include/scwx/provider/iem_api_provider.hpp +++ b/wxdata/include/scwx/provider/iem_api_provider.hpp @@ -4,10 +4,20 @@ #include #include -#include #include +#if defined(_MSC_VER) +# pragma warning(push) +# pragma warning(disable : 4702) +#endif + +#include + +#if defined(_MSC_VER) +# pragma warning(pop) +#endif + namespace scwx::provider { @@ -30,10 +40,10 @@ public: ListTextProducts(std::chrono::sys_time date, std::optional cccc = {}, std::optional pil = {}); - static boost::outcome_v2::result> - ListTextProducts(std::vector> dates, - std::vector ccccs = {}, - std::vector pils = {}); + static boost::outcome_v2::result> ListTextProducts( + ranges::any_view> dates, + ranges::any_view ccccs = {}, + ranges::any_view pils = {}); static std::vector> LoadTextProducts(const std::vector& textProducts); diff --git a/wxdata/source/scwx/provider/iem_api_provider.cpp b/wxdata/source/scwx/provider/iem_api_provider.cpp index 00fff608..b72b3b18 100644 --- a/wxdata/source/scwx/provider/iem_api_provider.cpp +++ b/wxdata/source/scwx/provider/iem_api_provider.cpp @@ -6,7 +6,9 @@ #include #include +#include #include +#include #if (__cpp_lib_chrono < 201907L) # include @@ -50,17 +52,18 @@ IemApiProvider::ListTextProducts(std::chrono::sys_time date, std::string_view pil = optionalPil.has_value() ? optionalPil.value() : std::string_view {}; - return ListTextProducts( - std::vector> {date}, - {cccc}, - {pil}); + const auto dateArray = std::array {date}; + const auto ccccArray = std::array {cccc}; + const auto pilArray = std::array {pil}; + + return ListTextProducts(dateArray, ccccArray, pilArray); } boost::outcome_v2::result> IemApiProvider::ListTextProducts( - std::vector> dates, - std::vector ccccs, - std::vector pils) + ranges::any_view> dates, + ranges::any_view ccccs, + ranges::any_view pils) { using namespace std::chrono; @@ -75,20 +78,24 @@ IemApiProvider::ListTextProducts( # define kDateFormat "%Y-%m-%d" #endif - if (ccccs.empty()) + if (ccccs.begin() == ccccs.end()) { - ccccs.push_back({}); + ccccs = ranges::views::single(std::string_view {}); } - if (pils.empty()) + if (pils.begin() == pils.end()) { - pils.push_back({}); + pils = ranges::views::single(std::string_view {}); } + const auto dv = ranges::to(dates); + const auto cv = ranges::to(ccccs); + const auto pv = ranges::to(pils); + std::vector responses {}; for (const auto& [date, cccc, pil] : - ranges::views::cartesian_product(dates, ccccs, pils)) + ranges::views::cartesian_product(dv, cv, pv)) { auto parameters = cpr::Parameters {{"date", df::format(kDateFormat, date)}}; From e3ccce5d5bbbe522993d65684fc27dd95ac220c9 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sun, 13 Apr 2025 01:27:47 -0500 Subject: [PATCH 41/67] Text event manager should use filtered ranges to request archived products --- .../scwx/qt/manager/text_event_manager.cpp | 99 +++++++++---------- 1 file changed, 45 insertions(+), 54 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp index bac8b0e7..6574c1cb 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp @@ -15,6 +15,8 @@ #include #include #include +#include +#include namespace scwx { @@ -89,18 +91,15 @@ public: void HandleMessage(const std::shared_ptr& message); - void LoadArchive(std::chrono::sys_days date, const std::string& pil); - void LoadArchives(std::chrono::sys_days date); + void LoadArchives(ranges::any_view dates); void RefreshAsync(); void Refresh(); - void UpdateArchiveDates(std::chrono::sys_days date); + void UpdateArchiveDates(ranges::any_view dates); // Thread pool sized for: // - Live Refresh (1x) - // - Archive Loading (15x) - // - 3 day window (3x) - // - TOR, SVR, SVS, FFW, FFS (5x) - boost::asio::thread_pool threadPool_ {16u}; + // - Archive Loading (1x) + boost::asio::thread_pool threadPool_ {2u}; TextEventManager* self_; @@ -123,10 +122,8 @@ public: std::mutex archiveMutex_ {}; std::list archiveDates_ {}; - std::map< - std::chrono::sys_days, - std::unordered_map>>> + std::map>> archiveMap_; boost::uuids::uuid warningsProviderChangedCallbackUuid_ {}; @@ -212,16 +209,17 @@ void TextEventManager::SelectTime( const auto today = std::chrono::floor(dateTime); const auto yesterday = today - std::chrono::days {1}; const auto tomorrow = today + std::chrono::days {1}; - const auto dates = {today, yesterday, tomorrow}; + const auto dateArray = std::array {today, yesterday, tomorrow}; - for (auto& date : dates) - { - if (p->archiveLimit_ == std::chrono::sys_days {} || - date < p->archiveLimit_) - { - p->LoadArchives(date); - } - } + const ranges::any_view dates = + dateArray | ranges::views::filter( + [this](const auto& date) + { + return p->archiveLimit_ == std::chrono::sys_days {} || + date < p->archiveLimit_; + }); + + p->LoadArchives(dates); } void TextEventManager::Impl::HandleMessage( @@ -283,7 +281,8 @@ void TextEventManager::Impl::HandleMessage( it->second.end(), message, [](const std::shared_ptr& a, - const std::shared_ptr& b) { + const std::shared_ptr& b) + { return a->wmo_header()->GetDateTime() < b->wmo_header()->GetDateTime(); }); @@ -302,26 +301,27 @@ void TextEventManager::Impl::HandleMessage( } } -void TextEventManager::Impl::LoadArchive(std::chrono::sys_days date, - const std::string& pil) +void TextEventManager::Impl::LoadArchives( + ranges::any_view dates) { + UpdateArchiveDates(dates); + std::unique_lock lock {archiveMutex_}; - auto& dateArchive = archiveMap_[date]; - if (dateArchive.contains(pil)) - { - // Don't reload data that has already been loaded - return; - } + + // Don't reload data that has already been loaded + const ranges::any_view filteredDates = + dates | ranges::views::filter([this](const auto& date) + { return !archiveMap_.contains(date); }); + lock.unlock(); - logger_->debug("Load Archive: {}, {}", util::TimeString(date), pil); - // Query for products - const auto& productIds = iemApiProvider_->ListTextProducts(date, {}, pil); + const auto& productIds = + iemApiProvider_->ListTextProducts(filteredDates, {}, kPils_); if (productIds.has_value()) { - logger_->debug("Loading {} {} products", productIds.value().size(), pil); + logger_->debug("Loading {} products", productIds.value().size()); // Load listed products auto products = iemApiProvider_->LoadTextProducts(productIds.value()); @@ -338,30 +338,17 @@ void TextEventManager::Impl::LoadArchive(std::chrono::sys_days date, lock.lock(); - // Ensure the archive map still contains the date, and has not been pruned - if (archiveMap_.contains(date)) + for (const auto& date : dates) { - // Store the products associated with the PIL in the archive - dateArchive.try_emplace(pil, std::move(products)); + archiveMap_[date]; + + // TODO: Store the products in the archive } lock.unlock(); } } -void TextEventManager::Impl::LoadArchives(std::chrono::sys_days date) -{ - logger_->trace("Load Archives: {}", util::TimeString(date)); - - UpdateArchiveDates(date); - - for (auto& pil : kPils_) - { - boost::asio::post(threadPool_, - [this, date, &pil]() { LoadArchive(date, pil); }); - } -} - void TextEventManager::Impl::RefreshAsync() { boost::asio::post(threadPool_, @@ -445,13 +432,17 @@ void TextEventManager::Impl::Refresh() }); } -void TextEventManager::Impl::UpdateArchiveDates(std::chrono::sys_days date) +void TextEventManager::Impl::UpdateArchiveDates( + ranges::any_view dates) { std::unique_lock lock {archiveMutex_}; - // Remove any existing occurrences of day, and add to the back of the list - archiveDates_.remove(date); - archiveDates_.push_back(date); + for (const auto& date : dates) + { + // Remove any existing occurrences of day, and add to the back of the list + archiveDates_.remove(date); + archiveDates_.push_back(date); + } } std::shared_ptr TextEventManager::Instance() From 33e18765b7316af6beecf466acab726fde3a2464 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Tue, 15 Apr 2025 00:16:39 -0500 Subject: [PATCH 42/67] Start of implementation to load a window of archive warning data, currently broken --- .../scwx/qt/manager/text_event_manager.cpp | 162 ++++++++++++++++-- .../scwx/provider/iem_api_provider.test.cpp | 6 +- .../scwx/provider/iem_api_provider.hpp | 6 +- .../source/scwx/provider/iem_api_provider.cpp | 19 +- 4 files changed, 163 insertions(+), 30 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp index 6574c1cb..ef56496b 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -15,9 +16,16 @@ #include #include #include +#include +#include +#include #include #include +#if (__cpp_lib_chrono < 201907L) +# include +#endif + namespace scwx { namespace qt @@ -25,6 +33,8 @@ namespace qt namespace manager { +using namespace std::chrono_literals; + static const std::string logPrefix_ = "scwx::qt::manager::text_event_manager"; static const auto logger_ = scwx::util::Logger::Create(logPrefix_); @@ -33,8 +43,23 @@ static constexpr std::chrono::hours kInitialLoadHistoryDuration_ = static constexpr std::chrono::hours kDefaultLoadHistoryDuration_ = std::chrono::hours {1}; -static const std::array kPils_ = { - "TOR", "SVR", "SVS", "FFW", "FFS"}; +static const std::array kPils_ = { + "FFS", "FFW", "MWS", "SMW", "SQW", "SVR", "SVS", "TOR"}; + +static const std:: + unordered_map> + kPilLoadWindows_ {{"FFS", {-24h, 1h}}, + {"FFW", {-24h, 1h}}, + {"MWS", {-4h, 1h}}, + {"SMW", {-4h, 1h}}, + {"SQW", {-4h, 1h}}, + {"SVR", {-4h, 1h}}, + {"SVS", {-4h, 1h}}, + {"TOR", {-4h, 1h}}}; + +// Widest load window provided by kPilLoadWindows_ +static const std::pair + kArchiveLoadWindow_ {-24h, 1h}; class TextEventManager::Impl { @@ -91,7 +116,8 @@ public: void HandleMessage(const std::shared_ptr& message); - void LoadArchives(ranges::any_view dates); + void ListArchives(ranges::any_view dates); + void LoadArchives(std::chrono::system_clock::time_point dateTime); void RefreshAsync(); void Refresh(); void UpdateArchiveDates(ranges::any_view dates); @@ -125,6 +151,9 @@ public: std::map>> archiveMap_; + std::map> + unloadedProductMap_; boost::uuids::uuid warningsProviderChangedCallbackUuid_ {}; }; @@ -219,7 +248,8 @@ void TextEventManager::SelectTime( date < p->archiveLimit_; }); - p->LoadArchives(dates); + p->ListArchives(dates); + p->LoadArchives(dateTime); } void TextEventManager::Impl::HandleMessage( @@ -301,23 +331,127 @@ void TextEventManager::Impl::HandleMessage( } } -void TextEventManager::Impl::LoadArchives( +void TextEventManager::Impl::ListArchives( ranges::any_view dates) { - UpdateArchiveDates(dates); - std::unique_lock lock {archiveMutex_}; + UpdateArchiveDates(dates); + // Don't reload data that has already been loaded - const ranges::any_view filteredDates = - dates | ranges::views::filter([this](const auto& date) - { return !archiveMap_.contains(date); }); + ranges::any_view filteredDates = + dates | + ranges::views::filter([this](const auto& date) + { return !unloadedProductMap_.contains(date); }); lock.unlock(); - // Query for products - const auto& productIds = - iemApiProvider_->ListTextProducts(filteredDates, {}, kPils_); + const auto dv = ranges::to(filteredDates); + + std::for_each( + std::execution::par, + dv.begin(), + dv.end(), + [this](const auto& date) + { + const auto dateArray = std::array {date}; + + auto productEntries = + iemApiProvider_->ListTextProducts(dateArray, {}, kPils_); + + std::unique_lock lock {archiveMutex_}; + + if (productEntries.has_value()) + { + unloadedProductMap_.try_emplace( + date, + {std::make_move_iterator(productEntries.value().begin()), + std::make_move_iterator(productEntries.value().end())}); + } + }); +} + +void TextEventManager::Impl::LoadArchives( + std::chrono::system_clock::time_point dateTime) +{ + using namespace std::chrono; + +#if (__cpp_lib_chrono >= 201907L) + namespace df = std; + + static constexpr std::string_view kDateFormat {"{:%Y%m%d%H%M}"}; +#else + using namespace date; + namespace df = date; + +# define kDateFormat "%Y%m%d%H%M" +#endif + + // Search unloaded products in the widest archive load window + const std::chrono::sys_days startDate = + std::chrono::floor(dateTime + + kArchiveLoadWindow_.first); + const std::chrono::sys_days endDate = std::chrono::floor( + dateTime + kArchiveLoadWindow_.second + std::chrono::days {1}); + + // Determine load windows for each PIL + std::unordered_map> + pilLoadWindowStrings; + + for (auto& loadWindow : kPilLoadWindows_) + { + const std::string& pil = loadWindow.first; + + pilLoadWindowStrings.insert_or_assign( + pil, + std::pair { + df::format(kDateFormat, (dateTime + loadWindow.second.first)), + df::format(kDateFormat, (dateTime + loadWindow.second.second))}); + } + + std::vector loadList {}; + + std::unique_lock lock {archiveMutex_}; + + for (auto date : boost::irange(startDate, endDate)) + { + auto mapIt = unloadedProductMap_.find(date); + if (mapIt == unloadedProductMap_.cend()) + { + continue; + } + + for (auto it = mapIt->second.begin(); it != mapIt->second.end();) + { + const auto& pil = it->pil_; + + // Check PIL + if (pil.size() >= 3) + { + auto pilPrefix = pil.substr(0, 3); + auto windowIt = pilLoadWindowStrings.find(pilPrefix); + + // Check Window + if (windowIt != pilLoadWindowStrings.cend()) + { + const auto& productId = it->productId_; + const auto& windowStart = windowIt->second.first; + const auto& windowEnd = windowIt->second.second; + + if (windowStart <= productId && productId <= windowEnd) + { + // Product matches, move it to the load list + loadList.emplace_back(std::move(*it)); + it = mapIt->second.erase(it); + continue; + } + } + } + + // Current iterator was not matched + ++it; + } + } if (productIds.has_value()) { @@ -435,8 +569,6 @@ void TextEventManager::Impl::Refresh() void TextEventManager::Impl::UpdateArchiveDates( ranges::any_view dates) { - std::unique_lock lock {archiveMutex_}; - for (const auto& date : dates) { // Remove any existing occurrences of day, and add to the back of the list diff --git a/test/source/scwx/provider/iem_api_provider.test.cpp b/test/source/scwx/provider/iem_api_provider.test.cpp index 854f0d60..4f964b81 100644 --- a/test/source/scwx/provider/iem_api_provider.test.cpp +++ b/test/source/scwx/provider/iem_api_provider.test.cpp @@ -23,11 +23,13 @@ TEST(IemApiProviderTest, ListTextProducts) if (torProducts.value().size() >= 1) { - EXPECT_EQ(torProducts.value().at(0), "202303250016-KMEG-WFUS54-TORMEG"); + EXPECT_EQ(torProducts.value().at(0).productId_, + "202303250016-KMEG-WFUS54-TORMEG"); } if (torProducts.value().size() >= 35) { - EXPECT_EQ(torProducts.value().at(34), "202303252015-KFFC-WFUS52-TORFFC"); + EXPECT_EQ(torProducts.value().at(34).productId_, + "202303252015-KFFC-WFUS52-TORFFC"); } } diff --git a/wxdata/include/scwx/provider/iem_api_provider.hpp b/wxdata/include/scwx/provider/iem_api_provider.hpp index 316890df..b862f73d 100644 --- a/wxdata/include/scwx/provider/iem_api_provider.hpp +++ b/wxdata/include/scwx/provider/iem_api_provider.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -36,11 +37,12 @@ public: IemApiProvider(IemApiProvider&&) noexcept; IemApiProvider& operator=(IemApiProvider&&) noexcept; - static boost::outcome_v2::result> + static boost::outcome_v2::result> ListTextProducts(std::chrono::sys_time date, std::optional cccc = {}, std::optional pil = {}); - static boost::outcome_v2::result> ListTextProducts( + static boost::outcome_v2::result> + ListTextProducts( ranges::any_view> dates, ranges::any_view ccccs = {}, ranges::any_view pils = {}); diff --git a/wxdata/source/scwx/provider/iem_api_provider.cpp b/wxdata/source/scwx/provider/iem_api_provider.cpp index b72b3b18..bab18864 100644 --- a/wxdata/source/scwx/provider/iem_api_provider.cpp +++ b/wxdata/source/scwx/provider/iem_api_provider.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include @@ -42,7 +41,7 @@ IemApiProvider::~IemApiProvider() = default; IemApiProvider::IemApiProvider(IemApiProvider&&) noexcept = default; IemApiProvider& IemApiProvider::operator=(IemApiProvider&&) noexcept = default; -boost::outcome_v2::result> +boost::outcome_v2::result> IemApiProvider::ListTextProducts(std::chrono::sys_time date, std::optional optionalCccc, std::optional optionalPil) @@ -59,7 +58,7 @@ IemApiProvider::ListTextProducts(std::chrono::sys_time date, return ListTextProducts(dateArray, ccccArray, pilArray); } -boost::outcome_v2::result> +boost::outcome_v2::result> IemApiProvider::ListTextProducts( ranges::any_view> dates, ranges::any_view ccccs, @@ -118,7 +117,7 @@ IemApiProvider::ListTextProducts( parameters)); } - std::vector textProducts {}; + std::vector textProducts {}; for (auto& asyncResponse : responses) { @@ -132,13 +131,9 @@ IemApiProvider::ListTextProducts( { // Get AFOS list from response auto entries = boost::json::value_to(json); - - for (auto& entry : entries.data_) - { - textProducts.push_back(entry.productId_); - } - - logger_->trace("Found {} products", entries.data_.size()); + textProducts.insert(textProducts.end(), + std::make_move_iterator(entries.data_.begin()), + std::make_move_iterator(entries.data_.end())); } catch (const std::exception& ex) { @@ -198,6 +193,8 @@ IemApiProvider::ListTextProducts( } } + logger_->trace("Found {} products", textProducts.size()); + return textProducts; } From 1a1c668d62b857e82d67b44229920d1ee3938f2a Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Wed, 16 Apr 2025 23:29:40 -0500 Subject: [PATCH 43/67] Finish windowed load. Not all polygon updates are shown on the map. --- .../scwx/qt/manager/text_event_manager.cpp | 62 +++++++------------ .../scwx/provider/iem_api_provider.hpp | 35 ++++++++++- .../source/scwx/provider/iem_api_provider.cpp | 30 +++------ 3 files changed, 65 insertions(+), 62 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp index ef56496b..153696ff 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -148,9 +149,8 @@ public: std::mutex archiveMutex_ {}; std::list archiveDates_ {}; - std::map>> - archiveMap_; + + std::mutex unloadedProductMapMutex_ {}; std::map> unloadedProductMap_; @@ -248,6 +248,9 @@ void TextEventManager::SelectTime( date < p->archiveLimit_; }); + std::unique_lock lock {p->archiveMutex_}; + + p->UpdateArchiveDates(dates); p->ListArchives(dates); p->LoadArchives(dateTime); } @@ -334,18 +337,12 @@ void TextEventManager::Impl::HandleMessage( void TextEventManager::Impl::ListArchives( ranges::any_view dates) { - std::unique_lock lock {archiveMutex_}; - - UpdateArchiveDates(dates); - // Don't reload data that has already been loaded ranges::any_view filteredDates = dates | ranges::views::filter([this](const auto& date) { return !unloadedProductMap_.contains(date); }); - lock.unlock(); - const auto dv = ranges::to(filteredDates); std::for_each( @@ -359,14 +356,15 @@ void TextEventManager::Impl::ListArchives( auto productEntries = iemApiProvider_->ListTextProducts(dateArray, {}, kPils_); - std::unique_lock lock {archiveMutex_}; + std::unique_lock lock {unloadedProductMapMutex_}; if (productEntries.has_value()) { unloadedProductMap_.try_emplace( date, - {std::make_move_iterator(productEntries.value().begin()), - std::make_move_iterator(productEntries.value().end())}); + boost::container::stable_vector { + std::make_move_iterator(productEntries.value().begin()), + std::make_move_iterator(productEntries.value().end())}); } }); } @@ -409,9 +407,7 @@ void TextEventManager::Impl::LoadArchives( df::format(kDateFormat, (dateTime + loadWindow.second.second))}); } - std::vector loadList {}; - - std::unique_lock lock {archiveMutex_}; + std::vector loadListEntries {}; for (auto date : boost::irange(startDate, endDate)) { @@ -441,7 +437,7 @@ void TextEventManager::Impl::LoadArchives( if (windowStart <= productId && productId <= windowEnd) { // Product matches, move it to the load list - loadList.emplace_back(std::move(*it)); + loadListEntries.emplace_back(std::move(*it)); it = mapIt->second.erase(it); continue; } @@ -453,33 +449,21 @@ void TextEventManager::Impl::LoadArchives( } } - if (productIds.has_value()) + // Load the load list + auto loadView = loadListEntries | + std::ranges::views::transform([](const auto& entry) + { return entry.productId_; }); + auto products = iemApiProvider_->LoadTextProducts(loadView); + + // Process loaded products + for (auto& product : products) { - logger_->debug("Loading {} products", productIds.value().size()); + const auto& messages = product->messages(); - // Load listed products - auto products = iemApiProvider_->LoadTextProducts(productIds.value()); - - for (auto& product : products) + for (auto& message : messages) { - const auto& messages = product->messages(); - - for (auto& message : messages) - { - HandleMessage(message); - } + HandleMessage(message); } - - lock.lock(); - - for (const auto& date : dates) - { - archiveMap_[date]; - - // TODO: Store the products in the archive - } - - lock.unlock(); } } diff --git a/wxdata/include/scwx/provider/iem_api_provider.hpp b/wxdata/include/scwx/provider/iem_api_provider.hpp index b862f73d..7d697919 100644 --- a/wxdata/include/scwx/provider/iem_api_provider.hpp +++ b/wxdata/include/scwx/provider/iem_api_provider.hpp @@ -1,12 +1,16 @@ #pragma once #include +#include #include #include +#include #include +#include #include +#include #if defined(_MSC_VER) # pragma warning(push) @@ -47,12 +51,41 @@ public: ranges::any_view ccccs = {}, ranges::any_view pils = {}); + template + requires std::same_as, std::string> static std::vector> - LoadTextProducts(const std::vector& textProducts); + LoadTextProducts(const Range& textProducts) + { + auto parameters = cpr::Parameters {{"nolimit", "true"}}; + + std::vector> asyncResponses {}; + asyncResponses.reserve(textProducts.size()); + + const std::string endpointUrl = kBaseUrl_ + kNwsTextProductEndpoint_; + + for (const auto& productId : textProducts) + { + asyncResponses.emplace_back( + productId, + cpr::GetAsync(cpr::Url {endpointUrl + productId}, + network::cpr::GetHeader(), + parameters)); + } + + return ProcessTextProductResponses(asyncResponses); + } private: class Impl; std::unique_ptr p; + + static const std::string kBaseUrl_; + static const std::string kListNwsTextProductsEndpoint_; + static const std::string kNwsTextProductEndpoint_; + + static std::vector> + ProcessTextProductResponses( + std::vector>& asyncResponses); }; } // namespace scwx::provider diff --git a/wxdata/source/scwx/provider/iem_api_provider.cpp b/wxdata/source/scwx/provider/iem_api_provider.cpp index bab18864..7e3ff2c4 100644 --- a/wxdata/source/scwx/provider/iem_api_provider.cpp +++ b/wxdata/source/scwx/provider/iem_api_provider.cpp @@ -1,10 +1,10 @@ #include -#include #include #include #include #include +#include #include #include #include @@ -19,10 +19,12 @@ namespace scwx::provider static const std::string logPrefix_ = "scwx::provider::iem_api_provider"; static const auto logger_ = util::Logger::Create(logPrefix_); -static const std::string kBaseUrl_ = "https://mesonet.agron.iastate.edu/api/1"; +const std::string IemApiProvider::kBaseUrl_ = + "https://mesonet.agron.iastate.edu/api/1"; -static const std::string kListNwsTextProductsEndpoint_ = "/nws/afos/list.json"; -static const std::string kNwsTextProductEndpoint_ = "/nwstext/"; +const std::string IemApiProvider::kListNwsTextProductsEndpoint_ = + "/nws/afos/list.json"; +const std::string IemApiProvider::kNwsTextProductEndpoint_ = "/nwstext/"; class IemApiProvider::Impl { @@ -199,25 +201,9 @@ IemApiProvider::ListTextProducts( } std::vector> -IemApiProvider::LoadTextProducts(const std::vector& textProducts) +IemApiProvider::ProcessTextProductResponses( + std::vector>& asyncResponses) { - auto parameters = cpr::Parameters {{"nolimit", "true"}}; - - std::vector> - asyncResponses {}; - asyncResponses.reserve(textProducts.size()); - - const std::string endpointUrl = kBaseUrl_ + kNwsTextProductEndpoint_; - - for (auto& productId : textProducts) - { - asyncResponses.emplace_back( - productId, - cpr::GetAsync(cpr::Url {endpointUrl + productId}, - network::cpr::GetHeader(), - parameters)); - } - std::vector> textProductFiles; for (auto& asyncResponse : asyncResponses) From 65e3a667500e7073aeb4bf09bef33015a545173d Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Fri, 25 Apr 2025 00:15:05 -0500 Subject: [PATCH 44/67] Update IemApiProvider to use template functions --- .../scwx/qt/manager/text_event_manager.cpp | 49 ++++++---- .../scwx/provider/iem_api_provider.hpp | 64 +++++-------- .../scwx/provider/iem_api_provider.ipp | 95 +++++++++++++++++++ .../source/scwx/provider/iem_api_provider.cpp | 65 +------------ wxdata/wxdata.cmake | 1 + 5 files changed, 159 insertions(+), 115 deletions(-) create mode 100644 wxdata/include/scwx/provider/iem_api_provider.ipp diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp index 153696ff..95d86354 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include @@ -20,8 +19,9 @@ #include #include #include -#include #include +#include +#include #if (__cpp_lib_chrono < 201907L) # include @@ -117,11 +117,17 @@ public: void HandleMessage(const std::shared_ptr& message); - void ListArchives(ranges::any_view dates); + template + requires std::same_as, + std::chrono::sys_days> + void ListArchives(DateRange dates); void LoadArchives(std::chrono::system_clock::time_point dateTime); void RefreshAsync(); void Refresh(); - void UpdateArchiveDates(ranges::any_view dates); + template + requires std::same_as, + std::chrono::sys_days> + void UpdateArchiveDates(DateRange dates); // Thread pool sized for: // - Live Refresh (1x) @@ -139,8 +145,6 @@ public: textEventMap_; std::shared_mutex textEventMutex_; - std::unique_ptr iemApiProvider_ { - std::make_unique()}; std::shared_ptr warningsProvider_ {nullptr}; std::chrono::hours loadHistoryDuration_ {kInitialLoadHistoryDuration_}; @@ -240,7 +244,7 @@ void TextEventManager::SelectTime( const auto tomorrow = today + std::chrono::days {1}; const auto dateArray = std::array {today, yesterday, tomorrow}; - const ranges::any_view dates = + const auto dates = dateArray | ranges::views::filter( [this](const auto& date) { @@ -334,11 +338,13 @@ void TextEventManager::Impl::HandleMessage( } } -void TextEventManager::Impl::ListArchives( - ranges::any_view dates) +template + requires std::same_as, + std::chrono::sys_days> +void TextEventManager::Impl::ListArchives(DateRange dates) { // Don't reload data that has already been loaded - ranges::any_view filteredDates = + auto filteredDates = dates | ranges::views::filter([this](const auto& date) { return !unloadedProductMap_.contains(date); }); @@ -351,10 +357,17 @@ void TextEventManager::Impl::ListArchives( dv.end(), [this](const auto& date) { + static const auto kEmptyRange_ = + ranges::views::single(std::string_view {}); + static const auto kPilsView_ = + kPils_ | + ranges::views::transform([](const std::string& pil) + { return std::string_view {pil}; }); + const auto dateArray = std::array {date}; - auto productEntries = - iemApiProvider_->ListTextProducts(dateArray, {}, kPils_); + auto productEntries = provider::IemApiProvider::ListTextProducts( + dateArray | ranges::views::all, kEmptyRange_, kPilsView_); std::unique_lock lock {unloadedProductMapMutex_}; @@ -450,10 +463,10 @@ void TextEventManager::Impl::LoadArchives( } // Load the load list - auto loadView = loadListEntries | - std::ranges::views::transform([](const auto& entry) + auto loadView = + loadListEntries | ranges::views::transform([](const auto& entry) { return entry.productId_; }); - auto products = iemApiProvider_->LoadTextProducts(loadView); + auto products = provider::IemApiProvider::LoadTextProducts(loadView); // Process loaded products for (auto& product : products) @@ -550,8 +563,10 @@ void TextEventManager::Impl::Refresh() }); } -void TextEventManager::Impl::UpdateArchiveDates( - ranges::any_view dates) +template + requires std::same_as, + std::chrono::sys_days> +void TextEventManager::Impl::UpdateArchiveDates(DateRange dates) { for (const auto& date : dates) { diff --git a/wxdata/include/scwx/provider/iem_api_provider.hpp b/wxdata/include/scwx/provider/iem_api_provider.hpp index 7d697919..3af9f545 100644 --- a/wxdata/include/scwx/provider/iem_api_provider.hpp +++ b/wxdata/include/scwx/provider/iem_api_provider.hpp @@ -1,24 +1,21 @@ #pragma once #include -#include #include #include -#include #include #include #include -#include +#include +#include #if defined(_MSC_VER) # pragma warning(push) # pragma warning(disable : 4702) #endif -#include - #if defined(_MSC_VER) # pragma warning(pop) #endif @@ -42,50 +39,41 @@ public: IemApiProvider& operator=(IemApiProvider&&) noexcept; static boost::outcome_v2::result> - ListTextProducts(std::chrono::sys_time date, - std::optional cccc = {}, - std::optional pil = {}); + ListTextProducts(std::chrono::sys_days date, + std::optional cccc = {}, + std::optional pil = {}); + + template + requires std::same_as, + std::chrono::sys_days> && + std::same_as, + std::string_view> && + std::same_as, std::string_view> static boost::outcome_v2::result> - ListTextProducts( - ranges::any_view> dates, - ranges::any_view ccccs = {}, - ranges::any_view pils = {}); + ListTextProducts(DateRange dates, CcccRange ccccs, PilRange pils); - template - requires std::same_as, std::string> + template + requires std::same_as, std::string> static std::vector> - LoadTextProducts(const Range& textProducts) - { - auto parameters = cpr::Parameters {{"nolimit", "true"}}; - - std::vector> asyncResponses {}; - asyncResponses.reserve(textProducts.size()); - - const std::string endpointUrl = kBaseUrl_ + kNwsTextProductEndpoint_; - - for (const auto& productId : textProducts) - { - asyncResponses.emplace_back( - productId, - cpr::GetAsync(cpr::Url {endpointUrl + productId}, - network::cpr::GetHeader(), - parameters)); - } - - return ProcessTextProductResponses(asyncResponses); - } + LoadTextProducts(const Range& textProducts); private: class Impl; std::unique_ptr p; + static boost::outcome_v2::result> + ProcessTextProductLists(std::vector& asyncResponses); + static std::vector> + ProcessTextProductFiles( + std::vector>& asyncResponses); + static const std::string kBaseUrl_; static const std::string kListNwsTextProductsEndpoint_; static const std::string kNwsTextProductEndpoint_; - - static std::vector> - ProcessTextProductResponses( - std::vector>& asyncResponses); }; } // namespace scwx::provider + +#include diff --git a/wxdata/include/scwx/provider/iem_api_provider.ipp b/wxdata/include/scwx/provider/iem_api_provider.ipp new file mode 100644 index 00000000..3fc94dad --- /dev/null +++ b/wxdata/include/scwx/provider/iem_api_provider.ipp @@ -0,0 +1,95 @@ +#pragma once + +#include +#include + +#include +#include +#include + +#if (__cpp_lib_chrono < 201907L) +# include +#endif + +namespace scwx::provider +{ + +template + requires std::same_as, + std::chrono::sys_days> && + std::same_as, std::string_view> && + std::same_as, std::string_view> +boost::outcome_v2::result> +IemApiProvider::ListTextProducts(DateRange dates, + CcccRange ccccs, + PilRange pils) +{ + using namespace std::chrono; + +#if (__cpp_lib_chrono >= 201907L) + namespace df = std; + + static constexpr std::string_view kDateFormat {"{:%Y-%m-%d}"}; +#else + using namespace date; + namespace df = date; + +# define kDateFormat "%Y-%m-%d" +#endif + + std::vector asyncResponses {}; + + for (const auto& [date, cccc, pil] : + ranges::views::cartesian_product(dates, ccccs, pils)) + { + auto parameters = + cpr::Parameters {{"date", df::format(kDateFormat, date)}}; + + // WMO Source Code + if (!cccc.empty()) + { + parameters.Add({"cccc", std::string {cccc}}); + } + + // AFOS / AWIPS ID / 3-6 length identifier + if (!pil.empty()) + { + parameters.Add({"pil", std::string {pil}}); + } + + asyncResponses.emplace_back( + cpr::GetAsync(cpr::Url {kBaseUrl_ + kListNwsTextProductsEndpoint_}, + network::cpr::GetHeader(), + parameters)); + } + + return ProcessTextProductLists(asyncResponses); +} + +template + requires std::same_as, std::string> +std::vector> +IemApiProvider::LoadTextProducts(const Range& textProducts) +{ + auto parameters = cpr::Parameters {{"nolimit", "true"}}; + + std::vector> asyncResponses {}; + asyncResponses.reserve(textProducts.size()); + + const std::string endpointUrl = kBaseUrl_ + kNwsTextProductEndpoint_; + + for (const auto& productId : textProducts) + { + asyncResponses.emplace_back( + productId, + cpr::GetAsync(cpr::Url {endpointUrl + productId}, + network::cpr::GetHeader(), + parameters)); + } + + return ProcessTextProductFiles(asyncResponses); +} + +} // namespace scwx::provider diff --git a/wxdata/source/scwx/provider/iem_api_provider.cpp b/wxdata/source/scwx/provider/iem_api_provider.cpp index 7e3ff2c4..c4fb1ccd 100644 --- a/wxdata/source/scwx/provider/iem_api_provider.cpp +++ b/wxdata/source/scwx/provider/iem_api_provider.cpp @@ -44,7 +44,7 @@ IemApiProvider::IemApiProvider(IemApiProvider&&) noexcept = default; IemApiProvider& IemApiProvider::operator=(IemApiProvider&&) noexcept = default; boost::outcome_v2::result> -IemApiProvider::ListTextProducts(std::chrono::sys_time date, +IemApiProvider::ListTextProducts(std::chrono::sys_days date, std::optional optionalCccc, std::optional optionalPil) { @@ -61,67 +61,12 @@ IemApiProvider::ListTextProducts(std::chrono::sys_time date, } boost::outcome_v2::result> -IemApiProvider::ListTextProducts( - ranges::any_view> dates, - ranges::any_view ccccs, - ranges::any_view pils) +IemApiProvider::ProcessTextProductLists( + std::vector& asyncResponses) { - using namespace std::chrono; - -#if (__cpp_lib_chrono >= 201907L) - namespace df = std; - - static constexpr std::string_view kDateFormat {"{:%Y-%m-%d}"}; -#else - using namespace date; - namespace df = date; - -# define kDateFormat "%Y-%m-%d" -#endif - - if (ccccs.begin() == ccccs.end()) - { - ccccs = ranges::views::single(std::string_view {}); - } - - if (pils.begin() == pils.end()) - { - pils = ranges::views::single(std::string_view {}); - } - - const auto dv = ranges::to(dates); - const auto cv = ranges::to(ccccs); - const auto pv = ranges::to(pils); - - std::vector responses {}; - - for (const auto& [date, cccc, pil] : - ranges::views::cartesian_product(dv, cv, pv)) - { - auto parameters = - cpr::Parameters {{"date", df::format(kDateFormat, date)}}; - - // WMO Source Code - if (!cccc.empty()) - { - parameters.Add({"cccc", std::string {cccc}}); - } - - // AFOS / AWIPS ID / 3-6 length identifier - if (!pil.empty()) - { - parameters.Add({"pil", std::string {pil}}); - } - - responses.emplace_back( - cpr::GetAsync(cpr::Url {kBaseUrl_ + kListNwsTextProductsEndpoint_}, - network::cpr::GetHeader(), - parameters)); - } - std::vector textProducts {}; - for (auto& asyncResponse : responses) + for (auto& asyncResponse : asyncResponses) { auto response = asyncResponse.get(); @@ -201,7 +146,7 @@ IemApiProvider::ListTextProducts( } std::vector> -IemApiProvider::ProcessTextProductResponses( +IemApiProvider::ProcessTextProductFiles( std::vector>& asyncResponses) { std::vector> textProductFiles; diff --git a/wxdata/wxdata.cmake b/wxdata/wxdata.cmake index 32db05ab..8d2e15b4 100644 --- a/wxdata/wxdata.cmake +++ b/wxdata/wxdata.cmake @@ -63,6 +63,7 @@ set(HDR_PROVIDER include/scwx/provider/aws_level2_data_provider.hpp include/scwx/provider/aws_level3_data_provider.hpp include/scwx/provider/aws_nexrad_data_provider.hpp include/scwx/provider/iem_api_provider.hpp + include/scwx/provider/iem_api_provider.ipp include/scwx/provider/nexrad_data_provider.hpp include/scwx/provider/nexrad_data_provider_factory.hpp include/scwx/provider/warnings_provider.hpp) From 8dde98d2a9117410784c55f19f09b2e1fd08a483 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 26 Apr 2025 09:47:50 -0500 Subject: [PATCH 45/67] Add debug log statements to archive warning loading --- .../source/scwx/qt/manager/text_event_manager.cpp | 13 +++++++++---- wxdata/include/scwx/provider/iem_api_provider.hpp | 3 +++ wxdata/include/scwx/provider/iem_api_provider.ipp | 12 ++++++++++++ wxdata/source/scwx/provider/iem_api_provider.cpp | 7 +++++-- 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp index 95d86354..72c7531f 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp @@ -462,11 +462,16 @@ void TextEventManager::Impl::LoadArchives( } } + std::vector> products {}; + // Load the load list - auto loadView = - loadListEntries | ranges::views::transform([](const auto& entry) - { return entry.productId_; }); - auto products = provider::IemApiProvider::LoadTextProducts(loadView); + if (!loadListEntries.empty()) + { + auto loadView = loadListEntries | + ranges::views::transform([](const auto& entry) + { return entry.productId_; }); + products = provider::IemApiProvider::LoadTextProducts(loadView); + } // Process loaded products for (auto& product : products) diff --git a/wxdata/include/scwx/provider/iem_api_provider.hpp b/wxdata/include/scwx/provider/iem_api_provider.hpp index 3af9f545..a3a65614 100644 --- a/wxdata/include/scwx/provider/iem_api_provider.hpp +++ b/wxdata/include/scwx/provider/iem_api_provider.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -69,6 +70,8 @@ private: ProcessTextProductFiles( std::vector>& asyncResponses); + static const std::shared_ptr logger_; + static const std::string kBaseUrl_; static const std::string kListNwsTextProductsEndpoint_; static const std::string kNwsTextProductEndpoint_; diff --git a/wxdata/include/scwx/provider/iem_api_provider.ipp b/wxdata/include/scwx/provider/iem_api_provider.ipp index 3fc94dad..08ffbd34 100644 --- a/wxdata/include/scwx/provider/iem_api_provider.ipp +++ b/wxdata/include/scwx/provider/iem_api_provider.ipp @@ -2,10 +2,13 @@ #include #include +#include +#include #include #include #include +#include #if (__cpp_lib_chrono < 201907L) # include @@ -39,6 +42,13 @@ IemApiProvider::ListTextProducts(DateRange dates, # define kDateFormat "%Y-%m-%d" #endif + auto formattedDates = dates | ranges::views::transform( + [](const std::chrono::sys_days& date) + { return df::format(kDateFormat, date); }); + + logger_->debug("Listing text products for: {}", + boost::algorithm::join(formattedDates, ", ")); + std::vector asyncResponses {}; for (const auto& [date, cccc, pil] : @@ -75,6 +85,8 @@ IemApiProvider::LoadTextProducts(const Range& textProducts) { auto parameters = cpr::Parameters {{"nolimit", "true"}}; + logger_->debug("Loading {} text products", textProducts.size()); + std::vector> asyncResponses {}; asyncResponses.reserve(textProducts.size()); diff --git a/wxdata/source/scwx/provider/iem_api_provider.cpp b/wxdata/source/scwx/provider/iem_api_provider.cpp index c4fb1ccd..e9b8ed71 100644 --- a/wxdata/source/scwx/provider/iem_api_provider.cpp +++ b/wxdata/source/scwx/provider/iem_api_provider.cpp @@ -17,7 +17,8 @@ namespace scwx::provider { static const std::string logPrefix_ = "scwx::provider::iem_api_provider"; -static const auto logger_ = util::Logger::Create(logPrefix_); + +const auto IemApiProvider::logger_ = util::Logger::Create(logPrefix_); const std::string IemApiProvider::kBaseUrl_ = "https://mesonet.agron.iastate.edu/api/1"; @@ -140,7 +141,7 @@ IemApiProvider::ProcessTextProductLists( } } - logger_->trace("Found {} products", textProducts.size()); + logger_->debug("Found {} products", textProducts.size()); return textProducts; } @@ -175,6 +176,8 @@ IemApiProvider::ProcessTextProductFiles( } } + logger_->debug("Loaded {} text products", textProductFiles.size()); + return textProductFiles; } From ae24991432dc0495fb10d5333625d8f799e48d77 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 26 Apr 2025 15:05:15 -0500 Subject: [PATCH 46/67] Load archive warnings in a dedicated thread --- .../scwx/qt/manager/text_event_manager.cpp | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp index 72c7531f..2300a7b1 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp @@ -239,24 +239,37 @@ void TextEventManager::SelectTime( logger_->trace("Select Time: {}", util::TimeString(dateTime)); - const auto today = std::chrono::floor(dateTime); - const auto yesterday = today - std::chrono::days {1}; - const auto tomorrow = today + std::chrono::days {1}; - const auto dateArray = std::array {today, yesterday, tomorrow}; + boost::asio::post( + p->threadPool_, + [=, this]() + { + try + { + const auto today = std::chrono::floor(dateTime); + const auto yesterday = today - std::chrono::days {1}; + const auto tomorrow = today + std::chrono::days {1}; + const auto dateArray = std::array {today, yesterday, tomorrow}; - const auto dates = - dateArray | ranges::views::filter( - [this](const auto& date) - { - return p->archiveLimit_ == std::chrono::sys_days {} || - date < p->archiveLimit_; - }); + const auto dates = + dateArray | + ranges::views::filter( + [this](const auto& date) + { + return p->archiveLimit_ == std::chrono::sys_days {} || + date < p->archiveLimit_; + }); - std::unique_lock lock {p->archiveMutex_}; + std::unique_lock lock {p->archiveMutex_}; - p->UpdateArchiveDates(dates); - p->ListArchives(dates); - p->LoadArchives(dateTime); + p->UpdateArchiveDates(dates); + p->ListArchives(dates); + p->LoadArchives(dateTime); + } + catch (const std::exception& ex) + { + logger_->error(ex.what()); + } + }); } void TextEventManager::Impl::HandleMessage( From 104fe790fbaa1cda232bb5e5011aef695d3ca596 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 26 Apr 2025 22:31:23 -0500 Subject: [PATCH 47/67] Update segment end time logic for alert layer - Only earlier segments should have their end time updated - The current message should end when the next message begins --- scwx-qt/source/scwx/qt/map/alert_layer.cpp | 64 +++++++++++++++------- 1 file changed, 45 insertions(+), 19 deletions(-) diff --git a/scwx-qt/source/scwx/qt/map/alert_layer.cpp b/scwx-qt/source/scwx/qt/map/alert_layer.cpp index 77a2332f..9188cc4c 100644 --- a/scwx-qt/source/scwx/qt/map/alert_layer.cpp +++ b/scwx-qt/source/scwx/qt/map/alert_layer.cpp @@ -336,27 +336,28 @@ void AlertLayerHandler::HandleAlert(const types::TextEventKey& key, alertsUpdated {}; const auto& messageList = textEventManager_->message_list(key); - auto message = messageList.at(messageIndex); - if (message->uuid() != uuid) + // Find message by UUID instead of index, as the message index could have + // changed between the signal being emitted and the handler being called + auto messageIt = std::find_if(messageList.cbegin(), + messageList.cend(), + [&uuid](const auto& message) + { return uuid == message->uuid(); }); + + if (messageIt == messageList.cend()) { - // Find message by UUID instead of index, as the message index could have - // changed between the signal being emitted and the handler being called - auto it = std::find_if(messageList.cbegin(), - messageList.cend(), - [&uuid](const auto& message) - { return uuid == message->uuid(); }); - - if (it == messageList.cend()) - { - logger_->warn( - "Could not find alert uuid: {} ({})", key.ToString(), messageIndex); - return; - } - - message = *it; + logger_->warn( + "Could not find alert uuid: {} ({})", key.ToString(), messageIndex); + return; } + auto& message = *messageIt; + auto nextMessageIt = std::next(messageIt); + + // Store the current message index + messageIndex = + static_cast(std::distance(messageList.cbegin(), messageIt)); + // Determine start time for first segment std::chrono::system_clock::time_point segmentBegin {}; if (message->segment_count() > 0) @@ -364,14 +365,31 @@ void AlertLayerHandler::HandleAlert(const types::TextEventKey& key, segmentBegin = message->segment(0)->event_begin(); } + // Determine the start time for the first segment of the next message + std::optional nextMessageBegin {}; + if (nextMessageIt != messageList.cend()) + { + nextMessageBegin = + (*nextMessageIt) + ->wmo_header() + ->GetDateTime((*nextMessageIt)->segment(0)->event_begin()); + } + // Take a unique mutex before modifying segments std::unique_lock lock {alertMutex_}; - // Update any existing segments with new end time + // Update any existing earlier segments with new end time auto& segmentsForKey = segmentsByKey_[key]; for (auto& segmentRecord : segmentsForKey) { - if (segmentRecord->segmentEnd_ > segmentBegin) + // Determine if the segment is earlier than the current message + auto it = std::find( + messageList.cbegin(), messageList.cend(), segmentRecord->message_); + auto segmentIndex = + static_cast(std::distance(messageList.cbegin(), it)); + + if (segmentIndex < messageIndex && + segmentRecord->segmentEnd_ > segmentBegin) { segmentRecord->segmentEnd_ = segmentBegin; @@ -398,6 +416,14 @@ void AlertLayerHandler::HandleAlert(const types::TextEventKey& key, std::shared_ptr segmentRecord = std::make_shared(segment, key, message); + // Update segment end time to be no later than the begin time of the next + // message (if present) + if (nextMessageBegin.has_value() && + segmentRecord->segmentEnd_ > nextMessageBegin) + { + segmentRecord->segmentEnd_ = nextMessageBegin.value(); + } + segmentsForKey.push_back(segmentRecord); segmentsForType.push_back(segmentRecord); From 3ba569354e7314dc54918dcb5f1800b4f4a9accc Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 26 Apr 2025 22:43:44 -0500 Subject: [PATCH 48/67] Ensure the alert layer schedules a render when an alert is added or updated --- scwx-qt/source/scwx/qt/map/alert_layer.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scwx-qt/source/scwx/qt/map/alert_layer.cpp b/scwx-qt/source/scwx/qt/map/alert_layer.cpp index 9188cc4c..b0dafcfc 100644 --- a/scwx-qt/source/scwx/qt/map/alert_layer.cpp +++ b/scwx-qt/source/scwx/qt/map/alert_layer.cpp @@ -598,6 +598,8 @@ void AlertLayer::Impl::AddAlert( lineHover, drawItems.first->second); } + + Q_EMIT self_->NeedsRendering(); } void AlertLayer::Impl::UpdateAlert( @@ -621,6 +623,8 @@ void AlertLayer::Impl::UpdateAlert( geoLines->SetLineEndTime(line, segmentRecord->segmentEnd_); } } + + Q_EMIT self_->NeedsRendering(); } void AlertLayer::Impl::AddLines( From 2f2516b998d68e2e1799c9192c45e24c68b6d811 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 26 Apr 2025 23:03:20 -0500 Subject: [PATCH 49/67] Fix: selected time is uninitialized on layer initialization --- scwx-qt/source/scwx/qt/manager/timeline_manager.cpp | 5 +++++ scwx-qt/source/scwx/qt/manager/timeline_manager.hpp | 2 ++ scwx-qt/source/scwx/qt/map/alert_layer.cpp | 2 ++ scwx-qt/source/scwx/qt/map/placefile_layer.cpp | 2 ++ 4 files changed, 11 insertions(+) diff --git a/scwx-qt/source/scwx/qt/manager/timeline_manager.cpp b/scwx-qt/source/scwx/qt/manager/timeline_manager.cpp index f0c95e53..0bcf4f68 100644 --- a/scwx-qt/source/scwx/qt/manager/timeline_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/timeline_manager.cpp @@ -112,6 +112,11 @@ public: TimelineManager::TimelineManager() : p(std::make_unique(this)) {} TimelineManager::~TimelineManager() = default; +std::chrono::system_clock::time_point TimelineManager::GetSelectedTime() const +{ + return p->selectedTime_; +} + void TimelineManager::SetMapCount(std::size_t mapCount) { p->mapCount_ = mapCount; diff --git a/scwx-qt/source/scwx/qt/manager/timeline_manager.hpp b/scwx-qt/source/scwx/qt/manager/timeline_manager.hpp index 1a4154ac..bbff19f4 100644 --- a/scwx-qt/source/scwx/qt/manager/timeline_manager.hpp +++ b/scwx-qt/source/scwx/qt/manager/timeline_manager.hpp @@ -24,6 +24,8 @@ public: static std::shared_ptr Instance(); + std::chrono::system_clock::time_point GetSelectedTime() const; + void SetMapCount(std::size_t mapCount); public slots: diff --git a/scwx-qt/source/scwx/qt/map/alert_layer.cpp b/scwx-qt/source/scwx/qt/map/alert_layer.cpp index b0dafcfc..e6d731be 100644 --- a/scwx-qt/source/scwx/qt/map/alert_layer.cpp +++ b/scwx-qt/source/scwx/qt/map/alert_layer.cpp @@ -268,6 +268,8 @@ void AlertLayer::Initialize() auto& alertLayerHandler = AlertLayerHandler::Instance(); + p->selectedTime_ = manager::TimelineManager::Instance()->GetSelectedTime(); + // Take a shared lock to prevent handling additional alerts while populating // initial lists std::shared_lock lock {alertLayerHandler.alertMutex_}; diff --git a/scwx-qt/source/scwx/qt/map/placefile_layer.cpp b/scwx-qt/source/scwx/qt/map/placefile_layer.cpp index df9828eb..dcead2a1 100644 --- a/scwx-qt/source/scwx/qt/map/placefile_layer.cpp +++ b/scwx-qt/source/scwx/qt/map/placefile_layer.cpp @@ -122,6 +122,8 @@ void PlacefileLayer::Initialize() logger_->debug("Initialize()"); DrawLayer::Initialize(); + + p->selectedTime_ = manager::TimelineManager::Instance()->GetSelectedTime(); } void PlacefileLayer::Render( From 8cdd8526ebc31861262afe04b27ec986e87a9ac0 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sun, 27 Apr 2025 10:40:17 -0500 Subject: [PATCH 50/67] Archive warning fixes for gcc --- wxdata/include/scwx/awips/wmo_header.hpp | 1 + wxdata/include/scwx/provider/iem_api_provider.ipp | 4 ++++ wxdata/source/scwx/provider/iem_api_provider.cpp | 3 ++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/wxdata/include/scwx/awips/wmo_header.hpp b/wxdata/include/scwx/awips/wmo_header.hpp index f3e24faf..889c955e 100644 --- a/wxdata/include/scwx/awips/wmo_header.hpp +++ b/wxdata/include/scwx/awips/wmo_header.hpp @@ -2,6 +2,7 @@ #include #include +#include #include namespace scwx::awips diff --git a/wxdata/include/scwx/provider/iem_api_provider.ipp b/wxdata/include/scwx/provider/iem_api_provider.ipp index 08ffbd34..25187376 100644 --- a/wxdata/include/scwx/provider/iem_api_provider.ipp +++ b/wxdata/include/scwx/provider/iem_api_provider.ipp @@ -104,4 +104,8 @@ IemApiProvider::LoadTextProducts(const Range& textProducts) return ProcessTextProductFiles(asyncResponses); } +#ifdef kDateFormat +# undef kDateFormat +#endif + } // namespace scwx::provider diff --git a/wxdata/source/scwx/provider/iem_api_provider.cpp b/wxdata/source/scwx/provider/iem_api_provider.cpp index e9b8ed71..c3d7dce0 100644 --- a/wxdata/source/scwx/provider/iem_api_provider.cpp +++ b/wxdata/source/scwx/provider/iem_api_provider.cpp @@ -18,7 +18,8 @@ namespace scwx::provider static const std::string logPrefix_ = "scwx::provider::iem_api_provider"; -const auto IemApiProvider::logger_ = util::Logger::Create(logPrefix_); +const std::shared_ptr IemApiProvider::logger_ = + util::Logger::Create(logPrefix_); const std::string IemApiProvider::kBaseUrl_ = "https://mesonet.agron.iastate.edu/api/1"; From 81f09e07f0cbebe6bf1a008e17772331875ce9bf Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sun, 27 Apr 2025 13:40:04 -0500 Subject: [PATCH 51/67] Archive warnings clang-tidy fixes --- .clang-tidy | 1 + scwx-qt/source/scwx/qt/manager/marker_manager.cpp | 3 ++- .../source/scwx/qt/manager/text_event_manager.cpp | 6 +++--- .../source/scwx/qt/manager/timeline_manager.hpp | 2 +- scwx-qt/source/scwx/qt/manager/update_manager.cpp | 2 +- test/source/scwx/awips/wmo_header.test.cpp | 14 +++++++------- .../scwx/provider/iem_api_provider.test.cpp | 15 ++++----------- .../scwx/provider/warnings_provider.test.cpp | 4 ++-- .../include/scwx/awips/text_product_message.hpp | 2 +- wxdata/source/scwx/awips/text_product_message.cpp | 4 ++-- wxdata/source/scwx/awips/wmo_header.cpp | 10 ++++++---- wxdata/source/scwx/provider/iem_api_provider.cpp | 8 ++++---- wxdata/source/scwx/provider/warnings_provider.cpp | 10 ++++++---- wxdata/source/scwx/util/time.cpp | 8 +++----- 14 files changed, 43 insertions(+), 46 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index dbf9fbd7..645c9c05 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -10,6 +10,7 @@ Checks: - '-cppcoreguidelines-pro-type-reinterpret-cast' - '-misc-include-cleaner' - '-misc-non-private-member-variables-in-classes' + - '-misc-use-anonymous-namespace' - '-modernize-return-braced-init-list' - '-modernize-use-trailing-return-type' FormatStyle: 'file' diff --git a/scwx-qt/source/scwx/qt/manager/marker_manager.cpp b/scwx-qt/source/scwx/qt/manager/marker_manager.cpp index ea21b211..8af310ec 100644 --- a/scwx-qt/source/scwx/qt/manager/marker_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/marker_manager.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -70,7 +71,7 @@ public: class MarkerManager::Impl::MarkerRecord { public: - MarkerRecord(const types::MarkerInfo& info) : markerInfo_ {info} {} + MarkerRecord(types::MarkerInfo info) : markerInfo_ {std::move(info)} {} const types::MarkerInfo& toMarkerInfo() { return markerInfo_; } diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp index 2300a7b1..77b8143b 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp @@ -241,7 +241,7 @@ void TextEventManager::SelectTime( boost::asio::post( p->threadPool_, - [=, this]() + [this]() { try { @@ -259,7 +259,7 @@ void TextEventManager::SelectTime( date < p->archiveLimit_; }); - std::unique_lock lock {p->archiveMutex_}; + const std::unique_lock lock {p->archiveMutex_}; p->UpdateArchiveDates(dates); p->ListArchives(dates); @@ -382,7 +382,7 @@ void TextEventManager::Impl::ListArchives(DateRange dates) auto productEntries = provider::IemApiProvider::ListTextProducts( dateArray | ranges::views::all, kEmptyRange_, kPilsView_); - std::unique_lock lock {unloadedProductMapMutex_}; + const std::unique_lock lock {unloadedProductMapMutex_}; if (productEntries.has_value()) { diff --git a/scwx-qt/source/scwx/qt/manager/timeline_manager.hpp b/scwx-qt/source/scwx/qt/manager/timeline_manager.hpp index bbff19f4..054a8201 100644 --- a/scwx-qt/source/scwx/qt/manager/timeline_manager.hpp +++ b/scwx-qt/source/scwx/qt/manager/timeline_manager.hpp @@ -24,7 +24,7 @@ public: static std::shared_ptr Instance(); - std::chrono::system_clock::time_point GetSelectedTime() const; + [[nodiscard]] std::chrono::system_clock::time_point GetSelectedTime() const; void SetMapCount(std::size_t mapCount); diff --git a/scwx-qt/source/scwx/qt/manager/update_manager.cpp b/scwx-qt/source/scwx/qt/manager/update_manager.cpp index 5910fcaf..05a9c0d1 100644 --- a/scwx-qt/source/scwx/qt/manager/update_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/update_manager.cpp @@ -126,7 +126,7 @@ size_t UpdateManager::Impl::PopulateReleases() // Successful REST API query if (r.status_code == 200) { - boost::json::value json = util::json::ReadJsonString(r.text); + const boost::json::value json = util::json::ReadJsonString(r.text); if (json == nullptr) { logger_->warn("Response not JSON: {}", r.header["content-type"]); diff --git a/test/source/scwx/awips/wmo_header.test.cpp b/test/source/scwx/awips/wmo_header.test.cpp index bdc4406f..17cd9706 100644 --- a/test/source/scwx/awips/wmo_header.test.cpp +++ b/test/source/scwx/awips/wmo_header.test.cpp @@ -16,7 +16,7 @@ TEST(WmoHeader, WmoFields) { std::stringstream ss {kWmoHeaderSample_}; WmoHeader header; - bool valid = header.Parse(ss); + const bool valid = header.Parse(ss); EXPECT_EQ(valid, true); EXPECT_EQ(header.sequence_number(), "887"); @@ -40,7 +40,7 @@ TEST(WmoHeader, DateHintBeforeParse) WmoHeader header; header.SetDateHint(2022y / October); - bool valid = header.Parse(ss); + const bool valid = header.Parse(ss); EXPECT_EQ(valid, true); EXPECT_EQ(header.GetDateTime(), @@ -54,7 +54,7 @@ TEST(WmoHeader, DateHintAfterParse) std::stringstream ss {kWmoHeaderSample_}; WmoHeader header; - bool valid = header.Parse(ss); + const bool valid = header.Parse(ss); header.SetDateHint(2022y / October); EXPECT_EQ(valid, true); @@ -69,7 +69,7 @@ TEST(WmoHeader, EndTimeHintSameMonth) std::stringstream ss {kWmoHeaderSample_}; WmoHeader header; - bool valid = header.Parse(ss); + const bool valid = header.Parse(ss); auto endTimeHint = sys_days {2022y / October / 29d} + 0h + 0min + 0s; @@ -85,7 +85,7 @@ TEST(WmoHeader, EndTimeHintPreviousMonth) std::stringstream ss {kWmoHeaderSample_}; WmoHeader header; - bool valid = header.Parse(ss); + const bool valid = header.Parse(ss); auto endTimeHint = sys_days {2022y / October / 27d} + 0h + 0min + 0s; @@ -101,7 +101,7 @@ TEST(WmoHeader, EndTimeHintPreviousYear) std::stringstream ss {kWmoHeaderSample_}; WmoHeader header; - bool valid = header.Parse(ss); + const bool valid = header.Parse(ss); auto endTimeHint = sys_days {2022y / January / 27d} + 0h + 0min + 0s; @@ -118,7 +118,7 @@ TEST(WmoHeader, EndTimeHintIgnored) WmoHeader header; header.SetDateHint(2022y / October); - bool valid = header.Parse(ss); + const bool valid = header.Parse(ss); auto endTimeHint = sys_days {2020y / January / 1d} + 0h + 0min + 0s; diff --git a/test/source/scwx/provider/iem_api_provider.test.cpp b/test/source/scwx/provider/iem_api_provider.test.cpp index 4f964b81..0cf133e0 100644 --- a/test/source/scwx/provider/iem_api_provider.test.cpp +++ b/test/source/scwx/provider/iem_api_provider.test.cpp @@ -2,9 +2,7 @@ #include -namespace scwx -{ -namespace provider +namespace scwx::provider { TEST(IemApiProviderTest, ListTextProducts) @@ -12,11 +10,9 @@ TEST(IemApiProviderTest, ListTextProducts) using namespace std::chrono; using sys_days = time_point; - IemApiProvider provider {}; - auto date = sys_days {2023y / March / 25d}; - auto torProducts = provider.ListTextProducts(date, {}, "TOR"); + auto torProducts = IemApiProvider::ListTextProducts(date, {}, "TOR"); ASSERT_EQ(torProducts.has_value(), true); EXPECT_EQ(torProducts.value().size(), 35); @@ -40,9 +36,7 @@ TEST(IemApiProviderTest, LoadTextProducts) "202303252015-KFFC-WFUS52-TORFFC", "202303311942-KLZK-WWUS54-SVSLZK"}; - IemApiProvider provider {}; - - auto textProducts = provider.LoadTextProducts(productIds); + auto textProducts = IemApiProvider::LoadTextProducts(productIds); EXPECT_EQ(textProducts.size(), 3); @@ -60,5 +54,4 @@ TEST(IemApiProviderTest, LoadTextProducts) } } -} // namespace provider -} // namespace scwx +} // namespace scwx::provider diff --git a/test/source/scwx/provider/warnings_provider.test.cpp b/test/source/scwx/provider/warnings_provider.test.cpp index de315b4b..c1c824da 100644 --- a/test/source/scwx/provider/warnings_provider.test.cpp +++ b/test/source/scwx/provider/warnings_provider.test.cpp @@ -18,9 +18,9 @@ TEST_P(WarningsProviderTest, LoadUpdatedFiles) { WarningsProvider provider(GetParam()); - std::chrono::sys_time now = + const std::chrono::sys_time now = std::chrono::floor(std::chrono::system_clock::now()); - std::chrono::sys_time startTime = + const std::chrono::sys_time startTime = now - std::chrono::days {3}; auto updatedFiles = provider.LoadUpdatedFiles(startTime); diff --git a/wxdata/include/scwx/awips/text_product_message.hpp b/wxdata/include/scwx/awips/text_product_message.hpp index 6830f91a..373dc223 100644 --- a/wxdata/include/scwx/awips/text_product_message.hpp +++ b/wxdata/include/scwx/awips/text_product_message.hpp @@ -96,7 +96,7 @@ public: TextProductMessage(TextProductMessage&&) noexcept; TextProductMessage& operator=(TextProductMessage&&) noexcept; - boost::uuids::uuid uuid() const; + [[nodiscard]] boost::uuids::uuid uuid() const; std::string message_content() const; std::shared_ptr wmo_header() const; std::vector mnd_header() const; diff --git a/wxdata/source/scwx/awips/text_product_message.cpp b/wxdata/source/scwx/awips/text_product_message.cpp index f8716b2b..53ec502d 100644 --- a/wxdata/source/scwx/awips/text_product_message.cpp +++ b/wxdata/source/scwx/awips/text_product_message.cpp @@ -593,8 +593,8 @@ std::optional TryParseVtecString(std::istream& is) if (RE2::PartialMatch(line, *rePVtecString)) { - vtec = Vtec(); - bool vtecValid = vtec->pVtec_.Parse(line); + vtec = Vtec(); + const bool vtecValid = vtec->pVtec_.Parse(line); isBegin = is.tellg(); diff --git a/wxdata/source/scwx/awips/wmo_header.cpp b/wxdata/source/scwx/awips/wmo_header.cpp index fad1231b..f89db609 100644 --- a/wxdata/source/scwx/awips/wmo_header.cpp +++ b/wxdata/source/scwx/awips/wmo_header.cpp @@ -143,9 +143,11 @@ std::chrono::sys_time WmoHeader::GetDateTime( { std::chrono::sys_time wmoDateTime {}; - if (p->absoluteDateTime_.has_value()) + const auto absoluteDateTime = p->absoluteDateTime_; + + if (absoluteDateTime.has_value()) { - wmoDateTime = p->absoluteDateTime_.value(); + wmoDateTime = absoluteDateTime.value(); } else if (endTimeHint.has_value()) { @@ -160,8 +162,8 @@ std::chrono::sys_time WmoHeader::GetDateTime( { using namespace std::chrono; - auto endDays = floor(endTimeHint.value()); - year_month_day endDate {endDays}; + const auto endDays = floor(endTimeHint.value()); + const year_month_day endDate {endDays}; // Combine end date year and month with WMO date time wmoDateTime = diff --git a/wxdata/source/scwx/provider/iem_api_provider.cpp b/wxdata/source/scwx/provider/iem_api_provider.cpp index c3d7dce0..85c37ec5 100644 --- a/wxdata/source/scwx/provider/iem_api_provider.cpp +++ b/wxdata/source/scwx/provider/iem_api_provider.cpp @@ -50,9 +50,9 @@ IemApiProvider::ListTextProducts(std::chrono::sys_days date, std::optional optionalCccc, std::optional optionalPil) { - std::string_view cccc = + const std::string_view cccc = optionalCccc.has_value() ? optionalCccc.value() : std::string_view {}; - std::string_view pil = + const std::string_view pil = optionalPil.has_value() ? optionalPil.value() : std::string_view {}; const auto dateArray = std::array {date}; @@ -72,7 +72,7 @@ IemApiProvider::ProcessTextProductLists( { auto response = asyncResponse.get(); - boost::json::value json = util::json::ReadJsonString(response.text); + const boost::json::value json = util::json::ReadJsonString(response.text); if (response.status_code == cpr::status::HTTP_OK) { @@ -161,7 +161,7 @@ IemApiProvider::ProcessTextProductFiles( { // Load file auto& productId = asyncResponse.first; - std::shared_ptr textProductFile { + const std::shared_ptr textProductFile { std::make_shared()}; std::istringstream responseBody {response.text}; if (textProductFile->LoadData(productId, responseBody)) diff --git a/wxdata/source/scwx/provider/warnings_provider.cpp b/wxdata/source/scwx/provider/warnings_provider.cpp index f0d6b8dd..78eb687c 100644 --- a/wxdata/source/scwx/provider/warnings_provider.cpp +++ b/wxdata/source/scwx/provider/warnings_provider.cpp @@ -5,6 +5,7 @@ // Enable chrono formatters #ifndef __cpp_lib_format +// NOLINTNEXTLINE(bugprone-reserved-identifier, cppcoreguidelines-macro-usage) # define __cpp_lib_format 202110L #endif @@ -106,7 +107,7 @@ WarningsProvider::LoadUpdatedFiles( asyncCallbacks; std::vector> updatedFiles; - std::chrono::sys_time now = + const std::chrono::sys_time now = std::chrono::floor(std::chrono::system_clock::now()); std::chrono::sys_time currentHour = (startTime != std::chrono::sys_time {}) ? @@ -130,7 +131,8 @@ WarningsProvider::LoadUpdatedFiles( { if (headResponse.status_code == cpr::status::HTTP_OK) { - bool updated = p->UpdateFileRecord(headResponse, filename); + const bool updated = + p->UpdateFileRecord(headResponse, filename); if (updated) { @@ -173,7 +175,7 @@ WarningsProvider::LoadUpdatedFiles( logger_->debug("Loading file: {}", filename); // Load file - std::shared_ptr textProductFile { + const std::shared_ptr textProductFile { std::make_shared()}; std::istringstream responseBody {response.text}; if (textProductFile->LoadData(filename, responseBody)) @@ -218,7 +220,7 @@ bool WarningsProvider::Impl::UpdateFileRecord(const cpr::Response& response, lastModified = lastModifiedIt->second; } - std::unique_lock lock(filesMutex_); + const std::unique_lock lock(filesMutex_); auto it = files_.find(filename); if (it != files_.cend()) diff --git a/wxdata/source/scwx/util/time.cpp b/wxdata/source/scwx/util/time.cpp index 20c6121d..563aea1b 100644 --- a/wxdata/source/scwx/util/time.cpp +++ b/wxdata/source/scwx/util/time.cpp @@ -21,9 +21,7 @@ # include #endif -namespace scwx -{ -namespace util +namespace scwx::util { static const std::string logPrefix_ = "scwx::util::time"; @@ -48,6 +46,7 @@ std::chrono::system_clock::time_point TimePoint(uint32_t modifiedJulianDate, using sys_days = time_point; constexpr auto epoch = sys_days {1969y / December / 31d}; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers): literals are used return epoch + (modifiedJulianDate * 24h) + std::chrono::milliseconds {milliseconds}; } @@ -154,5 +153,4 @@ template std::optional> TryParseDateTime(const std::string& dateTimeFormat, const std::string& str); -} // namespace util -} // namespace scwx +} // namespace scwx::util From 56961efe769bd1447244867cecd5b47c142ba117 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sun, 27 Apr 2025 14:30:29 -0500 Subject: [PATCH 52/67] Correcting clang-tidy fix compile error --- scwx-qt/source/scwx/qt/manager/text_event_manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp index 77b8143b..7572beb7 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp @@ -241,7 +241,7 @@ void TextEventManager::SelectTime( boost::asio::post( p->threadPool_, - [this]() + [dateTime, this]() { try { From 82ba51909ef193f878aca0784b809bcbcbb087c0 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sun, 27 Apr 2025 15:09:58 -0500 Subject: [PATCH 53/67] Breaking circular header dependency --- scwx-qt/source/scwx/qt/manager/text_event_manager.cpp | 2 +- wxdata/include/scwx/provider/iem_api_provider.hpp | 2 -- wxdata/source/scwx/provider/iem_api_provider.cpp | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp index 7572beb7..4c17e819 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/wxdata/include/scwx/provider/iem_api_provider.hpp b/wxdata/include/scwx/provider/iem_api_provider.hpp index a3a65614..1aac31b2 100644 --- a/wxdata/include/scwx/provider/iem_api_provider.hpp +++ b/wxdata/include/scwx/provider/iem_api_provider.hpp @@ -78,5 +78,3 @@ private: }; } // namespace scwx::provider - -#include diff --git a/wxdata/source/scwx/provider/iem_api_provider.cpp b/wxdata/source/scwx/provider/iem_api_provider.cpp index 85c37ec5..ef0576e7 100644 --- a/wxdata/source/scwx/provider/iem_api_provider.cpp +++ b/wxdata/source/scwx/provider/iem_api_provider.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include From 34fc6d584f469dd28991c193c999bf0eee92c95d Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sun, 27 Apr 2025 18:34:46 -0500 Subject: [PATCH 54/67] Updating include for IEM API Provider test --- test/source/scwx/provider/iem_api_provider.test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/source/scwx/provider/iem_api_provider.test.cpp b/test/source/scwx/provider/iem_api_provider.test.cpp index 0cf133e0..e3e25669 100644 --- a/test/source/scwx/provider/iem_api_provider.test.cpp +++ b/test/source/scwx/provider/iem_api_provider.test.cpp @@ -1,4 +1,4 @@ -#include +#include #include From 228ec191f6da72db08ee63195d716887bc55c40c Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Fri, 2 May 2025 22:56:41 -0500 Subject: [PATCH 55/67] Add year to Text Event Key --- .../scwx/qt/manager/text_event_manager.cpp | 24 ++++++++++++++++- .../source/scwx/qt/types/text_event_key.cpp | 14 ++++++---- .../source/scwx/qt/types/text_event_key.hpp | 26 +++++++++++++++++-- 3 files changed, 56 insertions(+), 8 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp index 4c17e819..6337e97e 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp @@ -275,6 +275,8 @@ void TextEventManager::SelectTime( void TextEventManager::Impl::HandleMessage( const std::shared_ptr& message) { + using namespace std::chrono_literals; + auto segments = message->segments(); // If there are no segments, skip this message @@ -295,15 +297,35 @@ void TextEventManager::Impl::HandleMessage( } } + // Determine year + std::chrono::year_month_day wmoDate = std::chrono::floor( + message->wmo_header()->GetDateTime()); + std::chrono::year wmoYear = wmoDate.year(); + std::unique_lock lock(textEventMutex_); // Find a matching event in the event map auto& vtecString = segments[0]->header_->vtecString_; - types::TextEventKey key {vtecString[0].pVtec_}; + types::TextEventKey key {vtecString[0].pVtec_, wmoYear}; size_t messageIndex = 0; auto it = textEventMap_.find(key); bool updated = false; + if ( + // If there was no matching event + it == textEventMap_.cend() && + // The event is not new + vtecString[0].pVtec_.action() != awips::PVtec::Action::New && + // The message was on January 1 + wmoDate.month() == std::chrono::January && wmoDate.day() == 1d && + // This is at least the 10th ETN of the year + vtecString[0].pVtec_.event_tracking_number() > 10) + { + // Attempt to find a matching event from last year + key = {vtecString[0].pVtec_, wmoYear - std::chrono::years {1}}; + it = textEventMap_.find(key); + } + if (it == textEventMap_.cend()) { // If there was no matching event, add the message to a new event diff --git a/scwx-qt/source/scwx/qt/types/text_event_key.cpp b/scwx-qt/source/scwx/qt/types/text_event_key.cpp index bebf6f63..be5d0443 100644 --- a/scwx-qt/source/scwx/qt/types/text_event_key.cpp +++ b/scwx-qt/source/scwx/qt/types/text_event_key.cpp @@ -14,26 +14,29 @@ static const std::string logPrefix_ = "scwx::qt::types::text_event_key"; std::string TextEventKey::ToFullString() const { - return fmt::format("{} {} {} {:04}", + return fmt::format("{} {} {} {:04} ({:04})", officeId_, awips::GetPhenomenonText(phenomenon_), awips::GetSignificanceText(significance_), - etn_); + etn_, + static_cast(year_)); } std::string TextEventKey::ToString() const { - return fmt::format("{}.{}.{}.{:04}", + return fmt::format("{}.{}.{}.{:04}.{:04}", officeId_, awips::GetPhenomenonCode(phenomenon_), awips::GetSignificanceCode(significance_), - etn_); + etn_, + static_cast(year_)); } bool TextEventKey::operator==(const TextEventKey& o) const { return (officeId_ == o.officeId_ && phenomenon_ == o.phenomenon_ && - significance_ == o.significance_ && etn_ == o.etn_); + significance_ == o.significance_ && etn_ == o.etn_ && + year_ == o.year_); } size_t TextEventHash::operator()(const TextEventKey& x) const @@ -43,6 +46,7 @@ size_t TextEventHash::operator()(const TextEventKey& x) const boost::hash_combine(seed, x.phenomenon_); boost::hash_combine(seed, x.significance_); boost::hash_combine(seed, x.etn_); + boost::hash_combine(seed, static_cast(x.year_)); return seed; } diff --git a/scwx-qt/source/scwx/qt/types/text_event_key.hpp b/scwx-qt/source/scwx/qt/types/text_event_key.hpp index f962bcdf..4cc57dca 100644 --- a/scwx-qt/source/scwx/qt/types/text_event_key.hpp +++ b/scwx-qt/source/scwx/qt/types/text_event_key.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include namespace scwx { @@ -12,12 +13,32 @@ namespace types struct TextEventKey { TextEventKey() : TextEventKey(awips::PVtec {}) {} - TextEventKey(const awips::PVtec& pvtec) : + TextEventKey(const awips::PVtec& pvtec, std::chrono::year yearHint = {}) : officeId_ {pvtec.office_id()}, phenomenon_ {pvtec.phenomenon()}, significance_ {pvtec.significance()}, etn_ {pvtec.event_tracking_number()} { + using namespace std::chrono_literals; + + std::chrono::year_month_day ymd = + std::chrono::floor(pvtec.event_begin()); + if (ymd.year() > 1970y) + { + // Prefer the year from the event begin + year_ = ymd.year(); + } + else if (yearHint > 1970y) + { + // Otherwise, use the year hint + year_ = yearHint; + } + else + { + // If there was no year hint, use the event end + ymd = std::chrono::floor(pvtec.event_end()); + year_ = ymd.year(); + } } std::string ToFullString() const; @@ -27,7 +48,8 @@ struct TextEventKey std::string officeId_; awips::Phenomenon phenomenon_; awips::Significance significance_; - int16_t etn_; + std::int16_t etn_; + std::chrono::year year_; }; template From 4719badc5425568d99ef25aff65c25867d29150e Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 3 May 2025 09:27:58 -0500 Subject: [PATCH 56/67] clang-tidy fixes --- .../scwx/qt/manager/text_event_manager.cpp | 23 ++++++++++--------- .../source/scwx/qt/types/text_event_key.hpp | 8 ++++--- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp index 6337e97e..c6da90f7 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp @@ -27,11 +27,7 @@ # include #endif -namespace scwx -{ -namespace qt -{ -namespace manager +namespace scwx::qt::manager { using namespace std::chrono_literals; @@ -115,6 +111,11 @@ public: threadPool_.join(); } + Impl(const Impl&) = delete; + Impl& operator=(const Impl&) = delete; + Impl(const Impl&&) = delete; + Impl& operator=(const Impl&&) = delete; + void HandleMessage(const std::shared_ptr& message); template @@ -298,9 +299,10 @@ void TextEventManager::Impl::HandleMessage( } // Determine year - std::chrono::year_month_day wmoDate = std::chrono::floor( - message->wmo_header()->GetDateTime()); - std::chrono::year wmoYear = wmoDate.year(); + const std::chrono::year_month_day wmoDate = + std::chrono::floor( + message->wmo_header()->GetDateTime()); + const std::chrono::year wmoYear = wmoDate.year(); std::unique_lock lock(textEventMutex_); @@ -319,6 +321,7 @@ void TextEventManager::Impl::HandleMessage( // The message was on January 1 wmoDate.month() == std::chrono::January && wmoDate.day() == 1d && // This is at least the 10th ETN of the year + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers): Readability vtecString[0].pVtec_.event_tracking_number() > 10) { // Attempt to find a matching event from last year @@ -635,6 +638,4 @@ std::shared_ptr TextEventManager::Instance() return textEventManager; } -} // namespace manager -} // namespace qt -} // namespace scwx +} // namespace scwx::qt::manager diff --git a/scwx-qt/source/scwx/qt/types/text_event_key.hpp b/scwx-qt/source/scwx/qt/types/text_event_key.hpp index 4cc57dca..15eec31c 100644 --- a/scwx-qt/source/scwx/qt/types/text_event_key.hpp +++ b/scwx-qt/source/scwx/qt/types/text_event_key.hpp @@ -21,14 +21,16 @@ struct TextEventKey { using namespace std::chrono_literals; + static constexpr std::chrono::year kMinYear_ = 1970y; + std::chrono::year_month_day ymd = std::chrono::floor(pvtec.event_begin()); - if (ymd.year() > 1970y) + if (ymd.year() > kMinYear_) { // Prefer the year from the event begin year_ = ymd.year(); } - else if (yearHint > 1970y) + else if (yearHint > kMinYear_) { // Otherwise, use the year hint year_ = yearHint; @@ -49,7 +51,7 @@ struct TextEventKey awips::Phenomenon phenomenon_; awips::Significance significance_; std::int16_t etn_; - std::chrono::year year_; + std::chrono::year year_ {}; }; template From f37a77a9f7222aab3e1bdb7c7d9826808145759f Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 3 May 2025 23:20:26 -0500 Subject: [PATCH 57/67] Add text event pruning - Still need to prune AlertLayer - Still need to test alerts reload after being pruned --- .../source/scwx/qt/manager/alert_manager.cpp | 6 +- .../scwx/qt/manager/text_event_manager.cpp | 114 +++++++++++++++++- .../scwx/qt/manager/text_event_manager.hpp | 5 + scwx-qt/source/scwx/qt/model/alert_model.cpp | 31 ++++- scwx-qt/source/scwx/qt/model/alert_model.hpp | 5 + .../source/scwx/qt/ui/alert_dock_widget.cpp | 5 + 6 files changed, 158 insertions(+), 8 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/alert_manager.cpp b/scwx-qt/source/scwx/qt/manager/alert_manager.cpp index 757754a9..748e0943 100644 --- a/scwx-qt/source/scwx/qt/manager/alert_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/alert_manager.cpp @@ -138,8 +138,10 @@ common::Coordinate AlertManager::Impl::CurrentCoordinate( void AlertManager::Impl::HandleAlert(const types::TextEventKey& key, size_t messageIndex) const { + auto messages = textEventManager_->message_list(key); + // Skip alert if there are more messages to be processed - if (messageIndex + 1 < textEventManager_->message_count(key)) + if (messages.empty() || messageIndex + 1 < messages.size()) { return; } @@ -153,7 +155,7 @@ void AlertManager::Impl::HandleAlert(const types::TextEventKey& key, audioSettings.alert_radius().GetValue()); std::string alertWFO = audioSettings.alert_wfo().GetValue(); - auto message = textEventManager_->message_list(key).at(messageIndex); + auto message = messages.at(messageIndex); for (auto& segment : message->segments()) { diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp index c6da90f7..70505d44 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp @@ -116,13 +116,14 @@ public: Impl(const Impl&&) = delete; Impl& operator=(const Impl&&) = delete; - void - HandleMessage(const std::shared_ptr& message); + void HandleMessage(const std::shared_ptr& message, + bool archiveEvent = false); template requires std::same_as, std::chrono::sys_days> void ListArchives(DateRange dates); void LoadArchives(std::chrono::system_clock::time_point dateTime); + void PruneArchives(); void RefreshAsync(); void Refresh(); template @@ -155,6 +156,15 @@ public: std::mutex archiveMutex_ {}; std::list archiveDates_ {}; + std::mutex archiveEventKeyMutex_ {}; + std::map>> + archiveEventKeys_ {}; + std::unordered_set> + liveEventKeys_ {}; + std::mutex unloadedProductMapMutex_ {}; std::map> @@ -249,7 +259,7 @@ void TextEventManager::SelectTime( const auto today = std::chrono::floor(dateTime); const auto yesterday = today - std::chrono::days {1}; const auto tomorrow = today + std::chrono::days {1}; - const auto dateArray = std::array {today, yesterday, tomorrow}; + const auto dateArray = std::array {yesterday, today, tomorrow}; const auto dates = dateArray | @@ -265,6 +275,7 @@ void TextEventManager::SelectTime( p->UpdateArchiveDates(dates); p->ListArchives(dates); p->LoadArchives(dateTime); + p->PruneArchives(); } catch (const std::exception& ex) { @@ -274,7 +285,7 @@ void TextEventManager::SelectTime( } void TextEventManager::Impl::HandleMessage( - const std::shared_ptr& message) + const std::shared_ptr& message, bool archiveEvent) { using namespace std::chrono_literals; @@ -335,6 +346,12 @@ void TextEventManager::Impl::HandleMessage( textEventMap_.emplace(key, std::vector {message}); messageIndex = 0; updated = true; + + if (!archiveEvent) + { + // Add the Text Event Key to the list of live events to prevent pruning + liveEventKeys_.insert(key); + } } else if (std::find_if(it->second.cbegin(), it->second.cend(), @@ -368,6 +385,17 @@ void TextEventManager::Impl::HandleMessage( updated = true; }; + // If this is an archive event, and the key does not exist in the live events + // Assumption: A live event will always be loaded before a duplicate archive + // event + if (archiveEvent && !liveEventKeys_.contains(key)) + { + // Add the Text Event Key to the current date's archive + const std::unique_lock archiveEventKeyLock {archiveEventKeyMutex_}; + auto& archiveKeys = archiveEventKeys_[wmoDate]; + archiveKeys.insert(key); + } + lock.unlock(); if (updated) @@ -518,11 +546,87 @@ void TextEventManager::Impl::LoadArchives( for (auto& message : messages) { - HandleMessage(message); + HandleMessage(message, true); } } } +void TextEventManager::Impl::PruneArchives() +{ + static constexpr std::size_t kMaxArchiveDates_ = 5; + + std::unordered_set> + eventKeysToKeep {}; + std::unordered_set> + eventKeysToPrune {}; + + // Remove oldest dates from the archive + while (archiveDates_.size() > kMaxArchiveDates_) + { + archiveDates_.pop_front(); + } + + const std::unique_lock archiveEventKeyLock {archiveEventKeyMutex_}; + + // If there are the same number of dates in both archiveEventKeys_ and + // archiveDates_, there is nothing to prune + if (archiveEventKeys_.size() == archiveDates_.size()) + { + // Nothing to prune + return; + } + + const std::unique_lock unloadedProductMapLock {unloadedProductMapMutex_}; + + for (auto it = archiveEventKeys_.begin(); it != archiveEventKeys_.end();) + { + const auto& date = it->first; + const auto& eventKeys = it->second; + + // If date is not in recent days map + if (std::find(archiveDates_.cbegin(), archiveDates_.cend(), date) == + archiveDates_.cend()) + { + // Prune these keys (unless they are in the eventKeysToKeep set) + eventKeysToPrune.insert(eventKeys.begin(), eventKeys.end()); + + // The date is not in the list of recent dates, remove it + it = archiveEventKeys_.erase(it); + unloadedProductMap_.erase(date); + } + else + { + // Make sure these keys don't get pruned + eventKeysToKeep.insert(eventKeys.begin(), eventKeys.end()); + + // The date is recent, keep it + ++it; + } + } + + // Remove elements from eventKeysToPrune if they are in eventKeysToKeep + for (const auto& eventKey : eventKeysToKeep) + { + eventKeysToPrune.erase(eventKey); + } + + // Remove eventKeysToPrune from textEventMap + for (const auto& eventKey : eventKeysToPrune) + { + textEventMap_.erase(eventKey); + } + + // If event keys were pruned, emit a signal + if (!eventKeysToPrune.empty()) + { + logger_->debug("Pruned {} archive events", eventKeysToPrune.size()); + + Q_EMIT self_->AlertsRemoved(eventKeysToPrune); + } +} + void TextEventManager::Impl::RefreshAsync() { boost::asio::post(threadPool_, diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.hpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.hpp index 312dece0..61affe6c 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.hpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.hpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -35,6 +36,10 @@ public: static std::shared_ptr Instance(); signals: + void AlertsRemoved( + const std::unordered_set>& + keys); void AlertUpdated(const types::TextEventKey& key, std::size_t messageIndex, boost::uuids::uuid uuid); diff --git a/scwx-qt/source/scwx/qt/model/alert_model.cpp b/scwx-qt/source/scwx/qt/model/alert_model.cpp index 516dbfae..9baaf211 100644 --- a/scwx-qt/source/scwx/qt/model/alert_model.cpp +++ b/scwx-qt/source/scwx/qt/model/alert_model.cpp @@ -338,7 +338,7 @@ void AlertModel::HandleAlert(const types::TextEventKey& alertKey, auto alertMessages = p->textEventManager_->message_list(alertKey); // Skip alert if this is not the most recent message - if (messageIndex + 1 < alertMessages.size()) + if (alertMessages.empty() || messageIndex + 1 < alertMessages.size()) { return; } @@ -393,6 +393,35 @@ void AlertModel::HandleAlert(const types::TextEventKey& alertKey, } } +void AlertModel::HandleAlertsRemoved( + const std::unordered_set>& + alertKeys) +{ + logger_->trace("Handle alerts removed"); + + for (const auto& alertKey : alertKeys) + { + // Remove from the list of text event keys + auto it = std::find( + p->textEventKeys_.begin(), p->textEventKeys_.end(), alertKey); + if (it != p->textEventKeys_.end()) + { + int row = std::distance(p->textEventKeys_.begin(), it); + beginRemoveRows(QModelIndex(), row, row); + p->textEventKeys_.erase(it); + endRemoveRows(); + } + + // Remove from internal maps + p->observedMap_.erase(alertKey); + p->threatCategoryMap_.erase(alertKey); + p->tornadoPossibleMap_.erase(alertKey); + p->centroidMap_.erase(alertKey); + p->distanceMap_.erase(alertKey); + } +} + void AlertModel::HandleMapUpdate(double latitude, double longitude) { logger_->trace("Handle map update: {}, {}", latitude, longitude); diff --git a/scwx-qt/source/scwx/qt/model/alert_model.hpp b/scwx-qt/source/scwx/qt/model/alert_model.hpp index df6d561e..02781b6b 100644 --- a/scwx-qt/source/scwx/qt/model/alert_model.hpp +++ b/scwx-qt/source/scwx/qt/model/alert_model.hpp @@ -4,6 +4,7 @@ #include #include +#include #include @@ -51,6 +52,10 @@ public: public slots: void HandleAlert(const types::TextEventKey& alertKey, size_t messageIndex); + void HandleAlertsRemoved( + const std::unordered_set>& + alertKeys); void HandleMapUpdate(double latitude, double longitude); private: diff --git a/scwx-qt/source/scwx/qt/ui/alert_dock_widget.cpp b/scwx-qt/source/scwx/qt/ui/alert_dock_widget.cpp index 61fd160a..5e22071a 100644 --- a/scwx-qt/source/scwx/qt/ui/alert_dock_widget.cpp +++ b/scwx-qt/source/scwx/qt/ui/alert_dock_widget.cpp @@ -131,6 +131,11 @@ void AlertDockWidgetImpl::ConnectSignals() &QAction::toggled, proxyModel_.get(), &model::AlertProxyModel::SetAlertActiveFilter); + connect(textEventManager_.get(), + &manager::TextEventManager::AlertsRemoved, + alertModel_.get(), + &model::AlertModel::HandleAlertsRemoved, + Qt::QueuedConnection); connect(textEventManager_.get(), &manager::TextEventManager::AlertUpdated, alertModel_.get(), From 671ec1d6583f671575d433cc3c1b8ff6abd60b43 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sun, 4 May 2025 23:05:21 -0500 Subject: [PATCH 58/67] Handle removed alerts from alert layer --- scwx-qt/source/scwx/qt/map/alert_layer.cpp | 128 +++++++++++++++++---- 1 file changed, 108 insertions(+), 20 deletions(-) diff --git a/scwx-qt/source/scwx/qt/map/alert_layer.cpp b/scwx-qt/source/scwx/qt/map/alert_layer.cpp index e6d731be..811f3b88 100644 --- a/scwx-qt/source/scwx/qt/map/alert_layer.cpp +++ b/scwx-qt/source/scwx/qt/map/alert_layer.cpp @@ -73,10 +73,11 @@ public: connect(textEventManager_.get(), &manager::TextEventManager::AlertUpdated, this, - [this](const types::TextEventKey& key, - std::size_t messageIndex, - boost::uuids::uuid uuid) - { HandleAlert(key, messageIndex, uuid); }); + &AlertLayerHandler::HandleAlert); + connect(textEventManager_.get(), + &manager::TextEventManager::AlertsRemoved, + this, + &AlertLayerHandler::HandleAlertsRemoved); } ~AlertLayerHandler() { @@ -100,6 +101,10 @@ public: void HandleAlert(const types::TextEventKey& key, size_t messageIndex, boost::uuids::uuid uuid); + void HandleAlertsRemoved( + const std::unordered_set>& + keys); static AlertLayerHandler& Instance(); @@ -112,6 +117,7 @@ signals: void AlertAdded(const std::shared_ptr& segmentRecord, awips::Phenomenon phenomenon); void AlertUpdated(const std::shared_ptr& segmentRecord); + void AlertsRemoved(awips::Phenomenon phenomenon, bool alertActive); void AlertsUpdated(awips::Phenomenon phenomenon, bool alertActive); }; @@ -190,6 +196,7 @@ public: bool enableHover, boost::container::stable_vector< std::shared_ptr>& drawItems); + void PopulateLines(bool alertActive); void UpdateLines(); static LineData CreateLineData(const settings::LineSettings& lineSettings); @@ -276,22 +283,7 @@ void AlertLayer::Initialize() for (auto alertActive : {false, true}) { - auto& geoLines = p->geoLines_.at(alertActive); - - geoLines->StartLines(); - - // Populate initial segments - auto segmentsIt = - alertLayerHandler.segmentsByType_.find({p->phenomenon_, alertActive}); - if (segmentsIt != alertLayerHandler.segmentsByType_.cend()) - { - for (auto& segment : segmentsIt->second) - { - p->AddAlert(segment); - } - } - - geoLines->FinishLines(); + p->PopulateLines(alertActive); } p->ConnectAlertHandlerSignals(); @@ -444,6 +436,65 @@ void AlertLayerHandler::HandleAlert(const types::TextEventKey& key, } } +void AlertLayerHandler::HandleAlertsRemoved( + const std::unordered_set>& keys) +{ + logger_->trace("HandleAlertsRemoved: {} keys", keys.size()); + + std::unordered_set, + AlertTypeHash>> + alertsRemoved {}; + + // Take a unique lock before modifying segments + std::unique_lock lock {alertMutex_}; + + for (const auto& key : keys) + { + // Remove segments associated with the key + auto segmentsIt = segmentsByKey_.find(key); + if (segmentsIt != segmentsByKey_.end()) + { + for (const auto& segmentRecord : segmentsIt->second) + { + auto& segment = segmentRecord->segment_; + bool alertActive = IsAlertActive(segment); + + // Remove from segmentsByType_ + auto typeIt = segmentsByType_.find({key.phenomenon_, alertActive}); + if (typeIt != segmentsByType_.end()) + { + auto& segmentsForType = typeIt->second; + segmentsForType.erase(std::remove(segmentsForType.begin(), + segmentsForType.end(), + segmentRecord), + segmentsForType.end()); + + // If no segments remain for this type, erase the entry + if (segmentsForType.empty()) + { + segmentsByType_.erase(typeIt); + } + } + + alertsRemoved.emplace(key.phenomenon_, alertActive); + } + + // Remove the key from segmentsByKey_ + segmentsByKey_.erase(segmentsIt); + } + } + + // Release the lock after completing segment updates + lock.unlock(); + + // Emit signal to notify that alerts have been removed + for (auto& alert : alertsRemoved) + { + Q_EMIT AlertsRemoved(alert.first, alert.second); + } +} + void AlertLayer::Impl::ConnectAlertHandlerSignals() { auto& alertLayerHandler = AlertLayerHandler::Instance(); @@ -473,6 +524,22 @@ void AlertLayer::Impl::ConnectAlertHandlerSignals() UpdateAlert(segmentRecord); } }); + QObject::connect( + &alertLayerHandler, + &AlertLayerHandler::AlertsRemoved, + receiver_.get(), + [&alertLayerHandler, this](awips::Phenomenon phenomenon, bool alertActive) + { + if (phenomenon == phenomenon_) + { + // Take a shared lock to prevent handling additional alerts while + // populating initial lists + const std::shared_lock lock {alertLayerHandler.alertMutex_}; + + // Re-populate the lines if multiple alerts were removed + PopulateLines(alertActive); + } + }); } void AlertLayer::Impl::ConnectSignals() @@ -704,6 +771,27 @@ void AlertLayer::Impl::AddLine(std::shared_ptr& geoLines, } } +void AlertLayer::Impl::PopulateLines(bool alertActive) +{ + auto& alertLayerHandler = AlertLayerHandler::Instance(); + auto& geoLines = geoLines_.at(alertActive); + + geoLines->StartLines(); + + // Populate initial segments + auto segmentsIt = + alertLayerHandler.segmentsByType_.find({phenomenon_, alertActive}); + if (segmentsIt != alertLayerHandler.segmentsByType_.cend()) + { + for (auto& segment : segmentsIt->second) + { + AddAlert(segment); + } + } + + geoLines->FinishLines(); +} + void AlertLayer::Impl::UpdateLines() { std::unique_lock lock {linesMutex_}; From f40c24ce6fa5665461c00e4c8c6ae24a24ce99ff Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sun, 4 May 2025 23:05:36 -0500 Subject: [PATCH 59/67] Alert layer warning cleanup --- scwx-qt/source/scwx/qt/map/alert_layer.cpp | 53 ++++++++++++---------- scwx-qt/source/scwx/qt/map/alert_layer.hpp | 4 +- 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/scwx-qt/source/scwx/qt/map/alert_layer.cpp b/scwx-qt/source/scwx/qt/map/alert_layer.cpp index 811f3b88..e1e43965 100644 --- a/scwx-qt/source/scwx/qt/map/alert_layer.cpp +++ b/scwx-qt/source/scwx/qt/map/alert_layer.cpp @@ -19,12 +19,9 @@ #include #include #include +#include -namespace scwx -{ -namespace qt -{ -namespace map +namespace scwx::qt::map { static const std::string logPrefix_ = "scwx::qt::map::alert_layer"; @@ -46,6 +43,8 @@ static bool IsAlertActive(const std::shared_ptr& segment); class AlertLayerHandler : public QObject { Q_OBJECT + Q_DISABLE_COPY_MOVE(AlertLayerHandler) + public: struct SegmentRecord { @@ -57,10 +56,10 @@ public: SegmentRecord( const std::shared_ptr& segment, - const types::TextEventKey& key, + types::TextEventKey key, const std::shared_ptr& message) : segment_ {segment}, - key_ {key}, + key_ {std::move(key)}, message_ {message}, segmentBegin_ {segment->event_begin()}, segmentEnd_ {segment->event_end()} @@ -161,6 +160,11 @@ public: std::unique_lock lock(linesMutex_); }; + Impl(const Impl&) = delete; + Impl& operator=(const Impl&) = delete; + Impl(const Impl&&) = delete; + Impl& operator=(const Impl&&) = delete; + void AddAlert( const std::shared_ptr& segmentRecord); void UpdateAlert( @@ -182,14 +186,14 @@ public: std::shared_ptr& di, const common::Coordinate& p1, const common::Coordinate& p2, - boost::gil::rgba32f_pixel_t color, + const boost::gil::rgba32f_pixel_t& color, float width, std::chrono::system_clock::time_point startTime, std::chrono::system_clock::time_point endTime, bool enableHover); void AddLines(std::shared_ptr& geoLines, const std::vector& coordinates, - boost::gil::rgba32f_pixel_t color, + const boost::gil::rgba32f_pixel_t& color, float width, std::chrono::system_clock::time_point startTime, std::chrono::system_clock::time_point endTime, @@ -238,8 +242,8 @@ public: std::vector connections_ {}; }; -AlertLayer::AlertLayer(std::shared_ptr context, - awips::Phenomenon phenomenon) : +AlertLayer::AlertLayer(const std::shared_ptr& context, + awips::Phenomenon phenomenon) : DrawLayer( context, fmt::format("AlertLayer {}", awips::GetPhenomenonText(phenomenon))), @@ -620,9 +624,9 @@ void AlertLayer::Impl::AddAlert( // If draw items were added if (drawItems.second) { - const float borderWidth = lineData.borderWidth_; - const float highlightWidth = lineData.highlightWidth_; - const float lineWidth = lineData.lineWidth_; + const float borderWidth = static_cast(lineData.borderWidth_); + const float highlightWidth = static_cast(lineData.highlightWidth_); + const float lineWidth = static_cast(lineData.lineWidth_); const float totalHighlightWidth = lineWidth + (highlightWidth * 2.0f); const float totalBorderWidth = totalHighlightWidth + (borderWidth * 2.0f); @@ -699,7 +703,7 @@ void AlertLayer::Impl::UpdateAlert( void AlertLayer::Impl::AddLines( std::shared_ptr& geoLines, const std::vector& coordinates, - boost::gil::rgba32f_pixel_t color, + const boost::gil::rgba32f_pixel_t& color, float width, std::chrono::system_clock::time_point startTime, std::chrono::system_clock::time_point endTime, @@ -739,14 +743,17 @@ void AlertLayer::Impl::AddLine(std::shared_ptr& geoLines, std::shared_ptr& di, const common::Coordinate& p1, const common::Coordinate& p2, - boost::gil::rgba32f_pixel_t color, + const boost::gil::rgba32f_pixel_t& color, float width, std::chrono::system_clock::time_point startTime, std::chrono::system_clock::time_point endTime, bool enableHover) { - geoLines->SetLineLocation( - di, p1.latitude_, p1.longitude_, p2.latitude_, p2.longitude_); + geoLines->SetLineLocation(di, + static_cast(p1.latitude_), + static_cast(p1.longitude_), + static_cast(p2.latitude_), + static_cast(p2.longitude_)); geoLines->SetLineModulate(di, color); geoLines->SetLineWidth(di, width); geoLines->SetLineStartTime(di, startTime); @@ -805,9 +812,9 @@ void AlertLayer::Impl::UpdateLines() auto& lineData = GetLineData(segment, alertActive); auto& geoLines = geoLines_.at(alertActive); - const float borderWidth = lineData.borderWidth_; - const float highlightWidth = lineData.highlightWidth_; - const float lineWidth = lineData.lineWidth_; + const float borderWidth = static_cast(lineData.borderWidth_); + const float highlightWidth = static_cast(lineData.highlightWidth_); + const float lineWidth = static_cast(lineData.lineWidth_); const float totalHighlightWidth = lineWidth + (highlightWidth * 2.0f); const float totalBorderWidth = totalHighlightWidth + (borderWidth * 2.0f); @@ -982,8 +989,6 @@ size_t AlertTypeHash>::operator()( return seed; } -} // namespace map -} // namespace qt -} // namespace scwx +} // namespace scwx::qt::map #include "alert_layer.moc" diff --git a/scwx-qt/source/scwx/qt/map/alert_layer.hpp b/scwx-qt/source/scwx/qt/map/alert_layer.hpp index d51391e3..60905680 100644 --- a/scwx-qt/source/scwx/qt/map/alert_layer.hpp +++ b/scwx-qt/source/scwx/qt/map/alert_layer.hpp @@ -22,8 +22,8 @@ class AlertLayer : public DrawLayer Q_DISABLE_COPY_MOVE(AlertLayer) public: - explicit AlertLayer(std::shared_ptr context, - scwx::awips::Phenomenon phenomenon); + explicit AlertLayer(const std::shared_ptr& context, + scwx::awips::Phenomenon phenomenon); ~AlertLayer(); void Initialize() override final; From 0c59a0d4d27952ce2f8cc3616b473cf278b37ef3 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sun, 4 May 2025 23:09:57 -0500 Subject: [PATCH 60/67] Alert model clang-tidy cleanupp --- scwx-qt/source/scwx/qt/model/alert_model.cpp | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/scwx-qt/source/scwx/qt/model/alert_model.cpp b/scwx-qt/source/scwx/qt/model/alert_model.cpp index 9baaf211..a41a033f 100644 --- a/scwx-qt/source/scwx/qt/model/alert_model.cpp +++ b/scwx-qt/source/scwx/qt/model/alert_model.cpp @@ -10,16 +10,10 @@ #include #include -#include - #include #include -namespace scwx -{ -namespace qt -{ -namespace model +namespace scwx::qt::model { static const std::string logPrefix_ = "scwx::qt::model::alert_model"; @@ -407,7 +401,8 @@ void AlertModel::HandleAlertsRemoved( p->textEventKeys_.begin(), p->textEventKeys_.end(), alertKey); if (it != p->textEventKeys_.end()) { - int row = std::distance(p->textEventKeys_.begin(), it); + const int row = + static_cast(std::distance(p->textEventKeys_.begin(), it)); beginRemoveRows(QModelIndex(), row, row); p->textEventKeys_.erase(it); endRemoveRows(); @@ -597,6 +592,4 @@ std::string AlertModelImpl::GetEndTimeString(const types::TextEventKey& key) return scwx::util::TimeString(GetEndTime(key)); } -} // namespace model -} // namespace qt -} // namespace scwx +} // namespace scwx::qt::model From 490989ac2ad5345eb46ec63e14279ca5b508f31d Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Mon, 5 May 2025 00:25:28 -0500 Subject: [PATCH 61/67] Make archive event pruning more robust --- .../scwx/qt/manager/text_event_manager.cpp | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp index 70505d44..8aa4c611 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp @@ -570,9 +570,10 @@ void TextEventManager::Impl::PruneArchives() const std::unique_lock archiveEventKeyLock {archiveEventKeyMutex_}; - // If there are the same number of dates in both archiveEventKeys_ and - // archiveDates_, there is nothing to prune - if (archiveEventKeys_.size() == archiveDates_.size()) + // If there are the same number of dates in archiveEventKeys_, archiveDates_ + // and unloadedProductMap_, there is nothing to prune + if (archiveEventKeys_.size() == archiveDates_.size() && + unloadedProductMap_.size() == archiveDates_.size()) { // Nothing to prune return; @@ -594,7 +595,6 @@ void TextEventManager::Impl::PruneArchives() // The date is not in the list of recent dates, remove it it = archiveEventKeys_.erase(it); - unloadedProductMap_.erase(date); } else { @@ -606,6 +606,24 @@ void TextEventManager::Impl::PruneArchives() } } + for (auto it = unloadedProductMap_.begin(); it != unloadedProductMap_.end();) + { + const auto& date = it->first; + + // If date is not in recent days map + if (std::find(archiveDates_.cbegin(), archiveDates_.cend(), date) == + archiveDates_.cend()) + { + // The date is not in the list of recent dates, remove it + it = unloadedProductMap_.erase(it); + } + else + { + // The date is recent, keep it + ++it; + } + } + // Remove elements from eventKeysToPrune if they are in eventKeysToKeep for (const auto& eventKey : eventKeysToKeep) { From 1fdefe83de9563a7a4dfa546406a6731f6bdd465 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Mon, 5 May 2025 00:30:23 -0500 Subject: [PATCH 62/67] invalidateRowsFilter must be called from UI thread --- .../scwx/qt/model/alert_proxy_model.cpp | 37 ++++++++++--------- .../scwx/qt/model/alert_proxy_model.hpp | 22 ++++------- 2 files changed, 28 insertions(+), 31 deletions(-) diff --git a/scwx-qt/source/scwx/qt/model/alert_proxy_model.cpp b/scwx-qt/source/scwx/qt/model/alert_proxy_model.cpp index a2afee55..224f7400 100644 --- a/scwx-qt/source/scwx/qt/model/alert_proxy_model.cpp +++ b/scwx-qt/source/scwx/qt/model/alert_proxy_model.cpp @@ -9,21 +9,22 @@ #include -namespace scwx -{ -namespace qt -{ -namespace model +namespace scwx::qt::model { static const std::string logPrefix_ = "scwx::qt::model::alert_proxy_model"; static const auto logger_ = scwx::util::Logger::Create(logPrefix_); -class AlertProxyModelImpl +class AlertProxyModel::Impl { public: - explicit AlertProxyModelImpl(AlertProxyModel* self); - ~AlertProxyModelImpl(); + explicit Impl(AlertProxyModel* self); + ~Impl(); + + Impl(const Impl&) = delete; + Impl& operator=(const Impl&) = delete; + Impl(const Impl&&) = delete; + Impl& operator=(const Impl&&) = delete; void UpdateAlerts(); @@ -36,8 +37,7 @@ public: }; AlertProxyModel::AlertProxyModel(QObject* parent) : - QSortFilterProxyModel(parent), - p(std::make_unique(this)) + QSortFilterProxyModel(parent), p(std::make_unique(this)) { } AlertProxyModel::~AlertProxyModel() = default; @@ -77,7 +77,7 @@ bool AlertProxyModel::filterAcceptsRow(int sourceRow, QSortFilterProxyModel::filterAcceptsRow(sourceRow, sourceParent); } -AlertProxyModelImpl::AlertProxyModelImpl(AlertProxyModel* self) : +AlertProxyModel::Impl::Impl(AlertProxyModel* self) : self_ {self}, alertActiveFilterEnabled_ {false}, alertUpdateTimer_ {scwx::util::io_context()} @@ -86,13 +86,13 @@ AlertProxyModelImpl::AlertProxyModelImpl(AlertProxyModel* self) : UpdateAlerts(); } -AlertProxyModelImpl::~AlertProxyModelImpl() +AlertProxyModel::Impl::~Impl() { std::unique_lock lock(alertMutex_); alertUpdateTimer_.cancel(); } -void AlertProxyModelImpl::UpdateAlerts() +void AlertProxyModel::Impl::UpdateAlerts() { logger_->trace("UpdateAlerts"); @@ -102,10 +102,15 @@ void AlertProxyModelImpl::UpdateAlerts() // Re-evaluate for expired alerts if (alertActiveFilterEnabled_) { - self_->invalidateRowsFilter(); + QMetaObject::invokeMethod( + self_, + static_cast( + &QSortFilterProxyModel::invalidateRowsFilter)); } using namespace std::chrono; + + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers): Readability alertUpdateTimer_.expires_after(15s); alertUpdateTimer_.async_wait( [this](const boost::system::error_code& e) @@ -132,6 +137,4 @@ void AlertProxyModelImpl::UpdateAlerts() }); } -} // namespace model -} // namespace qt -} // namespace scwx +} // namespace scwx::qt::model diff --git a/scwx-qt/source/scwx/qt/model/alert_proxy_model.hpp b/scwx-qt/source/scwx/qt/model/alert_proxy_model.hpp index ee8b81c1..1ee6a138 100644 --- a/scwx-qt/source/scwx/qt/model/alert_proxy_model.hpp +++ b/scwx-qt/source/scwx/qt/model/alert_proxy_model.hpp @@ -4,11 +4,7 @@ #include -namespace scwx -{ -namespace qt -{ -namespace model +namespace scwx::qt::model { class AlertProxyModelImpl; @@ -16,7 +12,7 @@ class AlertProxyModelImpl; class AlertProxyModel : public QSortFilterProxyModel { private: - Q_DISABLE_COPY(AlertProxyModel) + Q_DISABLE_COPY_MOVE(AlertProxyModel) public: explicit AlertProxyModel(QObject* parent = nullptr); @@ -24,15 +20,13 @@ public: void SetAlertActiveFilter(bool enabled); - bool filterAcceptsRow(int sourceRow, - const QModelIndex& sourceParent) const override; + [[nodiscard]] bool + filterAcceptsRow(int sourceRow, + const QModelIndex& sourceParent) const override; private: - std::unique_ptr p; - - friend class AlertProxyModelImpl; + class Impl; + std::unique_ptr p; }; -} // namespace model -} // namespace qt -} // namespace scwx +} // namespace scwx::qt::model From 86926178dfae13916993a263fd8688062e575cee Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Mon, 5 May 2025 21:49:41 -0500 Subject: [PATCH 63/67] Call QSortFilterProxyModel::invalidate instead of invalidateRowsFilter (public vs. protected API) --- scwx-qt/source/scwx/qt/model/alert_proxy_model.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scwx-qt/source/scwx/qt/model/alert_proxy_model.cpp b/scwx-qt/source/scwx/qt/model/alert_proxy_model.cpp index 224f7400..dff55202 100644 --- a/scwx-qt/source/scwx/qt/model/alert_proxy_model.cpp +++ b/scwx-qt/source/scwx/qt/model/alert_proxy_model.cpp @@ -102,10 +102,9 @@ void AlertProxyModel::Impl::UpdateAlerts() // Re-evaluate for expired alerts if (alertActiveFilterEnabled_) { - QMetaObject::invokeMethod( - self_, - static_cast( - &QSortFilterProxyModel::invalidateRowsFilter)); + QMetaObject::invokeMethod(self_, + static_cast( + &QSortFilterProxyModel::invalidate)); } using namespace std::chrono; From dc074b0262ed447b64c935a9e527105f4b2dd7ae Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Mon, 5 May 2025 21:50:01 -0500 Subject: [PATCH 64/67] More clang-tidy fixes --- scwx-qt/source/scwx/qt/map/alert_layer.cpp | 16 ++++++++-------- .../source/scwx/qt/model/alert_proxy_model.cpp | 11 +++++++++-- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/scwx-qt/source/scwx/qt/map/alert_layer.cpp b/scwx-qt/source/scwx/qt/map/alert_layer.cpp index e1e43965..786d80e0 100644 --- a/scwx-qt/source/scwx/qt/map/alert_layer.cpp +++ b/scwx-qt/source/scwx/qt/map/alert_layer.cpp @@ -461,8 +461,8 @@ void AlertLayerHandler::HandleAlertsRemoved( { for (const auto& segmentRecord : segmentsIt->second) { - auto& segment = segmentRecord->segment_; - bool alertActive = IsAlertActive(segment); + auto& segment = segmentRecord->segment_; + const bool alertActive = IsAlertActive(segment); // Remove from segmentsByType_ auto typeIt = segmentsByType_.find({key.phenomenon_, alertActive}); @@ -624,9 +624,9 @@ void AlertLayer::Impl::AddAlert( // If draw items were added if (drawItems.second) { - const float borderWidth = static_cast(lineData.borderWidth_); - const float highlightWidth = static_cast(lineData.highlightWidth_); - const float lineWidth = static_cast(lineData.lineWidth_); + const auto borderWidth = static_cast(lineData.borderWidth_); + const auto highlightWidth = static_cast(lineData.highlightWidth_); + const auto lineWidth = static_cast(lineData.lineWidth_); const float totalHighlightWidth = lineWidth + (highlightWidth * 2.0f); const float totalBorderWidth = totalHighlightWidth + (borderWidth * 2.0f); @@ -812,9 +812,9 @@ void AlertLayer::Impl::UpdateLines() auto& lineData = GetLineData(segment, alertActive); auto& geoLines = geoLines_.at(alertActive); - const float borderWidth = static_cast(lineData.borderWidth_); - const float highlightWidth = static_cast(lineData.highlightWidth_); - const float lineWidth = static_cast(lineData.lineWidth_); + const auto borderWidth = static_cast(lineData.borderWidth_); + const auto highlightWidth = static_cast(lineData.highlightWidth_); + const auto lineWidth = static_cast(lineData.lineWidth_); const float totalHighlightWidth = lineWidth + (highlightWidth * 2.0f); const float totalBorderWidth = totalHighlightWidth + (borderWidth * 2.0f); diff --git a/scwx-qt/source/scwx/qt/model/alert_proxy_model.cpp b/scwx-qt/source/scwx/qt/model/alert_proxy_model.cpp index dff55202..fd5ffc69 100644 --- a/scwx-qt/source/scwx/qt/model/alert_proxy_model.cpp +++ b/scwx-qt/source/scwx/qt/model/alert_proxy_model.cpp @@ -88,8 +88,15 @@ AlertProxyModel::Impl::Impl(AlertProxyModel* self) : AlertProxyModel::Impl::~Impl() { - std::unique_lock lock(alertMutex_); - alertUpdateTimer_.cancel(); + try + { + std::unique_lock lock(alertMutex_); + alertUpdateTimer_.cancel(); + } + catch (const std::exception& ex) + { + logger_->error(ex.what()); + } } void AlertProxyModel::Impl::UpdateAlerts() From 4532327f50f1aeb170125a90f70fc0269f4e5c84 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Mon, 5 May 2025 22:13:18 -0500 Subject: [PATCH 65/67] AlertModel::HandleAlert should find the alert index from the UUID --- scwx-qt/source/scwx/qt/model/alert_model.cpp | 48 ++++++++++++++------ scwx-qt/source/scwx/qt/model/alert_model.hpp | 5 +- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/scwx-qt/source/scwx/qt/model/alert_model.cpp b/scwx-qt/source/scwx/qt/model/alert_model.cpp index a41a033f..20af05f8 100644 --- a/scwx-qt/source/scwx/qt/model/alert_model.cpp +++ b/scwx-qt/source/scwx/qt/model/alert_model.cpp @@ -323,23 +323,45 @@ AlertModel::headerData(int section, Qt::Orientation orientation, int role) const } void AlertModel::HandleAlert(const types::TextEventKey& alertKey, - size_t messageIndex) + std::size_t messageIndex, + boost::uuids::uuid uuid) { logger_->trace("Handle alert: {}", alertKey.ToString()); double distanceInMeters; - auto alertMessages = p->textEventManager_->message_list(alertKey); + const auto& alertMessages = p->textEventManager_->message_list(alertKey); + + // Find message by UUID instead of index, as the message index could have + // changed between the signal being emitted and the handler being called + auto messageIt = std::find_if(alertMessages.cbegin(), + alertMessages.cend(), + [&uuid](const auto& message) + { return uuid == message->uuid(); }); + + if (messageIt == alertMessages.cend()) + { + logger_->warn("Could not find alert uuid: {} ({})", + alertKey.ToString(), + messageIndex); + return; + } + + auto& message = *messageIt; + + // Store the current message index + messageIndex = static_cast( + std::distance(alertMessages.cbegin(), messageIt)); // Skip alert if this is not the most recent message - if (alertMessages.empty() || messageIndex + 1 < alertMessages.size()) + if (messageIndex + 1 < alertMessages.size()) { return; } // Get the most recent segment for the event - std::shared_ptr alertSegment = - alertMessages[messageIndex]->segments().back(); + const std::shared_ptr alertSegment = + message->segments().back(); p->observedMap_.insert_or_assign(alertKey, alertSegment->observed_); p->threatCategoryMap_.insert_or_assign(alertKey, @@ -519,8 +541,8 @@ std::string AlertModelImpl::GetCounties(const types::TextEventKey& key) } else { - logger_->warn("GetCounties(): No message associated with key: {}", - key.ToString()); + logger_->trace("GetCounties(): No message associated with key: {}", + key.ToString()); return {}; } } @@ -538,8 +560,8 @@ std::string AlertModelImpl::GetState(const types::TextEventKey& key) } else { - logger_->warn("GetState(): No message associated with key: {}", - key.ToString()); + logger_->trace("GetState(): No message associated with key: {}", + key.ToString()); return {}; } } @@ -556,8 +578,8 @@ AlertModelImpl::GetStartTime(const types::TextEventKey& key) } else { - logger_->warn("GetStartTime(): No message associated with key: {}", - key.ToString()); + logger_->trace("GetStartTime(): No message associated with key: {}", + key.ToString()); return {}; } } @@ -581,8 +603,8 @@ AlertModelImpl::GetEndTime(const types::TextEventKey& key) } else { - logger_->warn("GetEndTime(): No message associated with key: {}", - key.ToString()); + logger_->trace("GetEndTime(): No message associated with key: {}", + key.ToString()); return {}; } } diff --git a/scwx-qt/source/scwx/qt/model/alert_model.hpp b/scwx-qt/source/scwx/qt/model/alert_model.hpp index 02781b6b..443ca9bb 100644 --- a/scwx-qt/source/scwx/qt/model/alert_model.hpp +++ b/scwx-qt/source/scwx/qt/model/alert_model.hpp @@ -6,6 +6,7 @@ #include #include +#include #include namespace scwx @@ -51,7 +52,9 @@ public: int role = Qt::DisplayRole) const override; public slots: - void HandleAlert(const types::TextEventKey& alertKey, size_t messageIndex); + void HandleAlert(const types::TextEventKey& alertKey, + std::size_t messageIndex, + boost::uuids::uuid uuid); void HandleAlertsRemoved( const std::unordered_set>& From 73355c94247a5e6fafa999b7e3b53f0cf31af661 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Mon, 5 May 2025 22:49:43 -0500 Subject: [PATCH 66/67] Fix line repopulation on alert removal --- scwx-qt/source/scwx/qt/map/alert_layer.cpp | 67 +++++++++++++++------- 1 file changed, 46 insertions(+), 21 deletions(-) diff --git a/scwx-qt/source/scwx/qt/map/alert_layer.cpp b/scwx-qt/source/scwx/qt/map/alert_layer.cpp index 786d80e0..495be87c 100644 --- a/scwx-qt/source/scwx/qt/map/alert_layer.cpp +++ b/scwx-qt/source/scwx/qt/map/alert_layer.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -116,7 +117,7 @@ signals: void AlertAdded(const std::shared_ptr& segmentRecord, awips::Phenomenon phenomenon); void AlertUpdated(const std::shared_ptr& segmentRecord); - void AlertsRemoved(awips::Phenomenon phenomenon, bool alertActive); + void AlertsRemoved(awips::Phenomenon phenomenon); void AlertsUpdated(awips::Phenomenon phenomenon, bool alertActive); }; @@ -201,6 +202,7 @@ public: boost::container::stable_vector< std::shared_ptr>& drawItems); void PopulateLines(bool alertActive); + void RepopulateLines(); void UpdateLines(); static LineData CreateLineData(const settings::LineSettings& lineSettings); @@ -216,6 +218,7 @@ public: const awips::ibw::ImpactBasedWarningInfo& ibw_; std::unique_ptr receiver_ {std::make_unique()}; + std::mutex receiverMutex_ {}; std::unordered_map> geoLines_; @@ -446,9 +449,7 @@ void AlertLayerHandler::HandleAlertsRemoved( { logger_->trace("HandleAlertsRemoved: {} keys", keys.size()); - std::unordered_set, - AlertTypeHash>> - alertsRemoved {}; + std::set alertsRemoved {}; // Take a unique lock before modifying segments std::unique_lock lock {alertMutex_}; @@ -481,7 +482,7 @@ void AlertLayerHandler::HandleAlertsRemoved( } } - alertsRemoved.emplace(key.phenomenon_, alertActive); + alertsRemoved.emplace(key.phenomenon_); } // Remove the key from segmentsByKey_ @@ -495,7 +496,7 @@ void AlertLayerHandler::HandleAlertsRemoved( // Emit signal to notify that alerts have been removed for (auto& alert : alertsRemoved) { - Q_EMIT AlertsRemoved(alert.first, alert.second); + Q_EMIT AlertsRemoved(alert); } } @@ -513,6 +514,9 @@ void AlertLayer::Impl::ConnectAlertHandlerSignals() { if (phenomenon == phenomenon_) { + // Only process one signal at a time + const std::unique_lock lock {receiverMutex_}; + AddAlert(segmentRecord); } }); @@ -525,25 +529,27 @@ void AlertLayer::Impl::ConnectAlertHandlerSignals() { if (segmentRecord->key_.phenomenon_ == phenomenon_) { + // Only process one signal at a time + const std::unique_lock lock {receiverMutex_}; + UpdateAlert(segmentRecord); } }); - QObject::connect( - &alertLayerHandler, - &AlertLayerHandler::AlertsRemoved, - receiver_.get(), - [&alertLayerHandler, this](awips::Phenomenon phenomenon, bool alertActive) - { - if (phenomenon == phenomenon_) - { - // Take a shared lock to prevent handling additional alerts while - // populating initial lists - const std::shared_lock lock {alertLayerHandler.alertMutex_}; + QObject::connect(&alertLayerHandler, + &AlertLayerHandler::AlertsRemoved, + receiver_.get(), + [this](awips::Phenomenon phenomenon) + { + if (phenomenon == phenomenon_) + { + // Only process one signal at a time + const std::unique_lock lock {receiverMutex_}; - // Re-populate the lines if multiple alerts were removed - PopulateLines(alertActive); - } - }); + // Re-populate the lines if multiple alerts were + // removed + RepopulateLines(); + } + }); } void AlertLayer::Impl::ConnectSignals() @@ -799,6 +805,25 @@ void AlertLayer::Impl::PopulateLines(bool alertActive) geoLines->FinishLines(); } +void AlertLayer::Impl::RepopulateLines() +{ + auto& alertLayerHandler = AlertLayerHandler::Instance(); + + // Take a shared lock to prevent handling additional alerts while populating + // initial lists + const std::shared_lock alertLock {alertLayerHandler.alertMutex_}; + + linesBySegment_.clear(); + segmentsByLine_.clear(); + + for (auto alertActive : {false, true}) + { + PopulateLines(alertActive); + } + + Q_EMIT self_->NeedsRendering(); +} + void AlertLayer::Impl::UpdateLines() { std::unique_lock lock {linesMutex_}; From 3c5b126c67c34976e3b496454f34496ab99662c3 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Mon, 5 May 2025 23:24:01 -0500 Subject: [PATCH 67/67] Adding const to locks in AlertProxyModel --- scwx-qt/source/scwx/qt/model/alert_proxy_model.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scwx-qt/source/scwx/qt/model/alert_proxy_model.cpp b/scwx-qt/source/scwx/qt/model/alert_proxy_model.cpp index fd5ffc69..4dfeca41 100644 --- a/scwx-qt/source/scwx/qt/model/alert_proxy_model.cpp +++ b/scwx-qt/source/scwx/qt/model/alert_proxy_model.cpp @@ -90,7 +90,7 @@ AlertProxyModel::Impl::~Impl() { try { - std::unique_lock lock(alertMutex_); + const std::unique_lock lock(alertMutex_); alertUpdateTimer_.cancel(); } catch (const std::exception& ex) @@ -104,7 +104,7 @@ void AlertProxyModel::Impl::UpdateAlerts() logger_->trace("UpdateAlerts"); // Take a unique lock before modifying feature lists - std::unique_lock lock(alertMutex_); + const std::unique_lock lock(alertMutex_); // Re-evaluate for expired alerts if (alertActiveFilterEnabled_)