diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp index 6574c1cb..ef56496b 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -15,9 +16,16 @@ #include #include #include +#include +#include +#include #include #include +#if (__cpp_lib_chrono < 201907L) +# include +#endif + namespace scwx { namespace qt @@ -25,6 +33,8 @@ namespace qt namespace manager { +using namespace std::chrono_literals; + static const std::string logPrefix_ = "scwx::qt::manager::text_event_manager"; static const auto logger_ = scwx::util::Logger::Create(logPrefix_); @@ -33,8 +43,23 @@ static constexpr std::chrono::hours kInitialLoadHistoryDuration_ = static constexpr std::chrono::hours kDefaultLoadHistoryDuration_ = std::chrono::hours {1}; -static const std::array kPils_ = { - "TOR", "SVR", "SVS", "FFW", "FFS"}; +static const std::array kPils_ = { + "FFS", "FFW", "MWS", "SMW", "SQW", "SVR", "SVS", "TOR"}; + +static const std:: + unordered_map> + kPilLoadWindows_ {{"FFS", {-24h, 1h}}, + {"FFW", {-24h, 1h}}, + {"MWS", {-4h, 1h}}, + {"SMW", {-4h, 1h}}, + {"SQW", {-4h, 1h}}, + {"SVR", {-4h, 1h}}, + {"SVS", {-4h, 1h}}, + {"TOR", {-4h, 1h}}}; + +// Widest load window provided by kPilLoadWindows_ +static const std::pair + kArchiveLoadWindow_ {-24h, 1h}; class TextEventManager::Impl { @@ -91,7 +116,8 @@ public: void HandleMessage(const std::shared_ptr& message); - void LoadArchives(ranges::any_view dates); + void ListArchives(ranges::any_view dates); + void LoadArchives(std::chrono::system_clock::time_point dateTime); void RefreshAsync(); void Refresh(); void UpdateArchiveDates(ranges::any_view dates); @@ -125,6 +151,9 @@ public: std::map>> archiveMap_; + std::map> + unloadedProductMap_; boost::uuids::uuid warningsProviderChangedCallbackUuid_ {}; }; @@ -219,7 +248,8 @@ void TextEventManager::SelectTime( date < p->archiveLimit_; }); - p->LoadArchives(dates); + p->ListArchives(dates); + p->LoadArchives(dateTime); } void TextEventManager::Impl::HandleMessage( @@ -301,23 +331,127 @@ void TextEventManager::Impl::HandleMessage( } } -void TextEventManager::Impl::LoadArchives( +void TextEventManager::Impl::ListArchives( ranges::any_view dates) { - UpdateArchiveDates(dates); - std::unique_lock lock {archiveMutex_}; + UpdateArchiveDates(dates); + // Don't reload data that has already been loaded - const ranges::any_view filteredDates = - dates | ranges::views::filter([this](const auto& date) - { return !archiveMap_.contains(date); }); + ranges::any_view filteredDates = + dates | + ranges::views::filter([this](const auto& date) + { return !unloadedProductMap_.contains(date); }); lock.unlock(); - // Query for products - const auto& productIds = - iemApiProvider_->ListTextProducts(filteredDates, {}, kPils_); + const auto dv = ranges::to(filteredDates); + + std::for_each( + std::execution::par, + dv.begin(), + dv.end(), + [this](const auto& date) + { + const auto dateArray = std::array {date}; + + auto productEntries = + iemApiProvider_->ListTextProducts(dateArray, {}, kPils_); + + std::unique_lock lock {archiveMutex_}; + + if (productEntries.has_value()) + { + unloadedProductMap_.try_emplace( + date, + {std::make_move_iterator(productEntries.value().begin()), + std::make_move_iterator(productEntries.value().end())}); + } + }); +} + +void TextEventManager::Impl::LoadArchives( + std::chrono::system_clock::time_point dateTime) +{ + using namespace std::chrono; + +#if (__cpp_lib_chrono >= 201907L) + namespace df = std; + + static constexpr std::string_view kDateFormat {"{:%Y%m%d%H%M}"}; +#else + using namespace date; + namespace df = date; + +# define kDateFormat "%Y%m%d%H%M" +#endif + + // Search unloaded products in the widest archive load window + const std::chrono::sys_days startDate = + std::chrono::floor(dateTime + + kArchiveLoadWindow_.first); + const std::chrono::sys_days endDate = std::chrono::floor( + dateTime + kArchiveLoadWindow_.second + std::chrono::days {1}); + + // Determine load windows for each PIL + std::unordered_map> + pilLoadWindowStrings; + + for (auto& loadWindow : kPilLoadWindows_) + { + const std::string& pil = loadWindow.first; + + pilLoadWindowStrings.insert_or_assign( + pil, + std::pair { + df::format(kDateFormat, (dateTime + loadWindow.second.first)), + df::format(kDateFormat, (dateTime + loadWindow.second.second))}); + } + + std::vector loadList {}; + + std::unique_lock lock {archiveMutex_}; + + for (auto date : boost::irange(startDate, endDate)) + { + auto mapIt = unloadedProductMap_.find(date); + if (mapIt == unloadedProductMap_.cend()) + { + continue; + } + + for (auto it = mapIt->second.begin(); it != mapIt->second.end();) + { + const auto& pil = it->pil_; + + // Check PIL + if (pil.size() >= 3) + { + auto pilPrefix = pil.substr(0, 3); + auto windowIt = pilLoadWindowStrings.find(pilPrefix); + + // Check Window + if (windowIt != pilLoadWindowStrings.cend()) + { + const auto& productId = it->productId_; + const auto& windowStart = windowIt->second.first; + const auto& windowEnd = windowIt->second.second; + + if (windowStart <= productId && productId <= windowEnd) + { + // Product matches, move it to the load list + loadList.emplace_back(std::move(*it)); + it = mapIt->second.erase(it); + continue; + } + } + } + + // Current iterator was not matched + ++it; + } + } if (productIds.has_value()) { @@ -435,8 +569,6 @@ void TextEventManager::Impl::Refresh() void TextEventManager::Impl::UpdateArchiveDates( ranges::any_view dates) { - std::unique_lock lock {archiveMutex_}; - for (const auto& date : dates) { // Remove any existing occurrences of day, and add to the back of the list diff --git a/test/source/scwx/provider/iem_api_provider.test.cpp b/test/source/scwx/provider/iem_api_provider.test.cpp index 854f0d60..4f964b81 100644 --- a/test/source/scwx/provider/iem_api_provider.test.cpp +++ b/test/source/scwx/provider/iem_api_provider.test.cpp @@ -23,11 +23,13 @@ TEST(IemApiProviderTest, ListTextProducts) if (torProducts.value().size() >= 1) { - EXPECT_EQ(torProducts.value().at(0), "202303250016-KMEG-WFUS54-TORMEG"); + EXPECT_EQ(torProducts.value().at(0).productId_, + "202303250016-KMEG-WFUS54-TORMEG"); } if (torProducts.value().size() >= 35) { - EXPECT_EQ(torProducts.value().at(34), "202303252015-KFFC-WFUS52-TORFFC"); + EXPECT_EQ(torProducts.value().at(34).productId_, + "202303252015-KFFC-WFUS52-TORFFC"); } } diff --git a/wxdata/include/scwx/provider/iem_api_provider.hpp b/wxdata/include/scwx/provider/iem_api_provider.hpp index 316890df..b862f73d 100644 --- a/wxdata/include/scwx/provider/iem_api_provider.hpp +++ b/wxdata/include/scwx/provider/iem_api_provider.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -36,11 +37,12 @@ public: IemApiProvider(IemApiProvider&&) noexcept; IemApiProvider& operator=(IemApiProvider&&) noexcept; - static boost::outcome_v2::result> + static boost::outcome_v2::result> ListTextProducts(std::chrono::sys_time date, std::optional cccc = {}, std::optional pil = {}); - static boost::outcome_v2::result> ListTextProducts( + static boost::outcome_v2::result> + ListTextProducts( ranges::any_view> dates, ranges::any_view ccccs = {}, ranges::any_view pils = {}); diff --git a/wxdata/source/scwx/provider/iem_api_provider.cpp b/wxdata/source/scwx/provider/iem_api_provider.cpp index b72b3b18..bab18864 100644 --- a/wxdata/source/scwx/provider/iem_api_provider.cpp +++ b/wxdata/source/scwx/provider/iem_api_provider.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include @@ -42,7 +41,7 @@ IemApiProvider::~IemApiProvider() = default; IemApiProvider::IemApiProvider(IemApiProvider&&) noexcept = default; IemApiProvider& IemApiProvider::operator=(IemApiProvider&&) noexcept = default; -boost::outcome_v2::result> +boost::outcome_v2::result> IemApiProvider::ListTextProducts(std::chrono::sys_time date, std::optional optionalCccc, std::optional optionalPil) @@ -59,7 +58,7 @@ IemApiProvider::ListTextProducts(std::chrono::sys_time date, return ListTextProducts(dateArray, ccccArray, pilArray); } -boost::outcome_v2::result> +boost::outcome_v2::result> IemApiProvider::ListTextProducts( ranges::any_view> dates, ranges::any_view ccccs, @@ -118,7 +117,7 @@ IemApiProvider::ListTextProducts( parameters)); } - std::vector textProducts {}; + std::vector textProducts {}; for (auto& asyncResponse : responses) { @@ -132,13 +131,9 @@ IemApiProvider::ListTextProducts( { // Get AFOS list from response auto entries = boost::json::value_to(json); - - for (auto& entry : entries.data_) - { - textProducts.push_back(entry.productId_); - } - - logger_->trace("Found {} products", entries.data_.size()); + textProducts.insert(textProducts.end(), + std::make_move_iterator(entries.data_.begin()), + std::make_move_iterator(entries.data_.end())); } catch (const std::exception& ex) { @@ -198,6 +193,8 @@ IemApiProvider::ListTextProducts( } } + logger_->trace("Found {} products", textProducts.size()); + return textProducts; }