From 1a1c668d62b857e82d67b44229920d1ee3938f2a Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Wed, 16 Apr 2025 23:29:40 -0500 Subject: [PATCH] Finish windowed load. Not all polygon updates are shown on the map. --- .../scwx/qt/manager/text_event_manager.cpp | 62 +++++++------------ .../scwx/provider/iem_api_provider.hpp | 35 ++++++++++- .../source/scwx/provider/iem_api_provider.cpp | 30 +++------ 3 files changed, 65 insertions(+), 62 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp index ef56496b..153696ff 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -148,9 +149,8 @@ public: std::mutex archiveMutex_ {}; std::list archiveDates_ {}; - std::map>> - archiveMap_; + + std::mutex unloadedProductMapMutex_ {}; std::map> unloadedProductMap_; @@ -248,6 +248,9 @@ void TextEventManager::SelectTime( date < p->archiveLimit_; }); + std::unique_lock lock {p->archiveMutex_}; + + p->UpdateArchiveDates(dates); p->ListArchives(dates); p->LoadArchives(dateTime); } @@ -334,18 +337,12 @@ void TextEventManager::Impl::HandleMessage( void TextEventManager::Impl::ListArchives( ranges::any_view dates) { - std::unique_lock lock {archiveMutex_}; - - UpdateArchiveDates(dates); - // Don't reload data that has already been loaded ranges::any_view filteredDates = dates | ranges::views::filter([this](const auto& date) { return !unloadedProductMap_.contains(date); }); - lock.unlock(); - const auto dv = ranges::to(filteredDates); std::for_each( @@ -359,14 +356,15 @@ void TextEventManager::Impl::ListArchives( auto productEntries = iemApiProvider_->ListTextProducts(dateArray, {}, kPils_); - std::unique_lock lock {archiveMutex_}; + std::unique_lock lock {unloadedProductMapMutex_}; if (productEntries.has_value()) { unloadedProductMap_.try_emplace( date, - {std::make_move_iterator(productEntries.value().begin()), - std::make_move_iterator(productEntries.value().end())}); + boost::container::stable_vector { + std::make_move_iterator(productEntries.value().begin()), + std::make_move_iterator(productEntries.value().end())}); } }); } @@ -409,9 +407,7 @@ void TextEventManager::Impl::LoadArchives( df::format(kDateFormat, (dateTime + loadWindow.second.second))}); } - std::vector loadList {}; - - std::unique_lock lock {archiveMutex_}; + std::vector loadListEntries {}; for (auto date : boost::irange(startDate, endDate)) { @@ -441,7 +437,7 @@ void TextEventManager::Impl::LoadArchives( if (windowStart <= productId && productId <= windowEnd) { // Product matches, move it to the load list - loadList.emplace_back(std::move(*it)); + loadListEntries.emplace_back(std::move(*it)); it = mapIt->second.erase(it); continue; } @@ -453,33 +449,21 @@ void TextEventManager::Impl::LoadArchives( } } - if (productIds.has_value()) + // Load the load list + auto loadView = loadListEntries | + std::ranges::views::transform([](const auto& entry) + { return entry.productId_; }); + auto products = iemApiProvider_->LoadTextProducts(loadView); + + // Process loaded products + for (auto& product : products) { - logger_->debug("Loading {} products", productIds.value().size()); + const auto& messages = product->messages(); - // Load listed products - auto products = iemApiProvider_->LoadTextProducts(productIds.value()); - - for (auto& product : products) + for (auto& message : messages) { - const auto& messages = product->messages(); - - for (auto& message : messages) - { - HandleMessage(message); - } + HandleMessage(message); } - - lock.lock(); - - for (const auto& date : dates) - { - archiveMap_[date]; - - // TODO: Store the products in the archive - } - - lock.unlock(); } } diff --git a/wxdata/include/scwx/provider/iem_api_provider.hpp b/wxdata/include/scwx/provider/iem_api_provider.hpp index b862f73d..7d697919 100644 --- a/wxdata/include/scwx/provider/iem_api_provider.hpp +++ b/wxdata/include/scwx/provider/iem_api_provider.hpp @@ -1,12 +1,16 @@ #pragma once #include +#include #include #include +#include #include +#include #include +#include #if defined(_MSC_VER) # pragma warning(push) @@ -47,12 +51,41 @@ public: ranges::any_view ccccs = {}, ranges::any_view pils = {}); + template + requires std::same_as, std::string> static std::vector> - LoadTextProducts(const std::vector& textProducts); + LoadTextProducts(const Range& textProducts) + { + auto parameters = cpr::Parameters {{"nolimit", "true"}}; + + std::vector> asyncResponses {}; + asyncResponses.reserve(textProducts.size()); + + const std::string endpointUrl = kBaseUrl_ + kNwsTextProductEndpoint_; + + for (const auto& productId : textProducts) + { + asyncResponses.emplace_back( + productId, + cpr::GetAsync(cpr::Url {endpointUrl + productId}, + network::cpr::GetHeader(), + parameters)); + } + + return ProcessTextProductResponses(asyncResponses); + } private: class Impl; std::unique_ptr p; + + static const std::string kBaseUrl_; + static const std::string kListNwsTextProductsEndpoint_; + static const std::string kNwsTextProductEndpoint_; + + static std::vector> + ProcessTextProductResponses( + std::vector>& asyncResponses); }; } // namespace scwx::provider diff --git a/wxdata/source/scwx/provider/iem_api_provider.cpp b/wxdata/source/scwx/provider/iem_api_provider.cpp index bab18864..7e3ff2c4 100644 --- a/wxdata/source/scwx/provider/iem_api_provider.cpp +++ b/wxdata/source/scwx/provider/iem_api_provider.cpp @@ -1,10 +1,10 @@ #include -#include #include #include #include #include +#include #include #include #include @@ -19,10 +19,12 @@ namespace scwx::provider static const std::string logPrefix_ = "scwx::provider::iem_api_provider"; static const auto logger_ = util::Logger::Create(logPrefix_); -static const std::string kBaseUrl_ = "https://mesonet.agron.iastate.edu/api/1"; +const std::string IemApiProvider::kBaseUrl_ = + "https://mesonet.agron.iastate.edu/api/1"; -static const std::string kListNwsTextProductsEndpoint_ = "/nws/afos/list.json"; -static const std::string kNwsTextProductEndpoint_ = "/nwstext/"; +const std::string IemApiProvider::kListNwsTextProductsEndpoint_ = + "/nws/afos/list.json"; +const std::string IemApiProvider::kNwsTextProductEndpoint_ = "/nwstext/"; class IemApiProvider::Impl { @@ -199,25 +201,9 @@ IemApiProvider::ListTextProducts( } std::vector> -IemApiProvider::LoadTextProducts(const std::vector& textProducts) +IemApiProvider::ProcessTextProductResponses( + std::vector>& asyncResponses) { - auto parameters = cpr::Parameters {{"nolimit", "true"}}; - - std::vector> - asyncResponses {}; - asyncResponses.reserve(textProducts.size()); - - const std::string endpointUrl = kBaseUrl_ + kNwsTextProductEndpoint_; - - for (auto& productId : textProducts) - { - asyncResponses.emplace_back( - productId, - cpr::GetAsync(cpr::Url {endpointUrl + productId}, - network::cpr::GetHeader(), - parameters)); - } - std::vector> textProductFiles; for (auto& asyncResponse : asyncResponses)