Don't reload archive text products that have already been loaded

scwx-qt/source/scwx/qt/manager/text_event_manager.cpp

@@ -5,7 +5,10 @@
 #include <scwx/provider/iem_api_provider.hpp>
 #include <scwx/provider/warnings_provider.hpp>
 #include <scwx/util/logger.hpp>
+#include <scwx/util/time.hpp>
 
+#include <list>
+#include <map>
 #include <shared_mutex>
 #include <unordered_map>
 
@@ -90,6 +93,7 @@ public:
    void LoadArchives(std::chrono::sys_days date);
    void RefreshAsync();
    void Refresh();
+   void UpdateArchiveDates(std::chrono::sys_days date);
 
    // Thread pool sized for:
    // - Live Refresh (1x)
@@ -112,9 +116,18 @@ public:
    std::unique_ptr<provider::IemApiProvider> iemApiProvider_ {
       std::make_unique<provider::IemApiProvider>()};
    std::shared_ptr<provider::WarningsProvider> warningsProvider_ {nullptr};
+
    std::chrono::hours loadHistoryDuration_ {kInitialLoadHistoryDuration_};
    std::chrono::sys_time<std::chrono::hours> prevLoadTime_ {};
 
+   std::mutex                       archiveMutex_ {};
+   std::list<std::chrono::sys_days> archiveDates_ {};
+   std::map<
+      std::chrono::sys_days,
+      std::unordered_map<std::string,
+                         std::vector<std::shared_ptr<awips::TextProductFile>>>>
+      archiveMap_;
+
    boost::uuids::uuid warningsProviderChangedCallbackUuid_ {};
 };
 
@@ -187,10 +200,12 @@ void TextEventManager::LoadFile(const std::string& filename)
 void TextEventManager::SelectTime(
    std::chrono::system_clock::time_point dateTime)
 {
+   logger_->trace("Select Time: {}", util::TimeString(dateTime));
+
    const auto today     = std::chrono::floor<std::chrono::days>(dateTime);
    const auto yesterday = today - std::chrono::days {1};
    const auto tomorrow  = today + std::chrono::days {1};
-   const auto dates     = {yesterday, today, tomorrow};
+   const auto dates     = {today, yesterday, tomorrow};
 
    for (auto& date : dates)
    {
@@ -279,22 +294,56 @@ void TextEventManager::Impl::HandleMessage(
 void TextEventManager::Impl::LoadArchive(std::chrono::sys_days date,
                                          const std::string&    pil)
 {
-   const auto& productIds = iemApiProvider_->ListTextProducts(date, {}, pil);
+   std::unique_lock lock {archiveMutex_};
-   const auto& products   = iemApiProvider_->LoadTextProducts(productIds);
+   auto&            dateArchive = archiveMap_[date];
-
+   if (dateArchive.contains(pil))
-   for (auto& product : products)
    {
-      const auto& messages = product->messages();
+      // Don't reload data that has already been loaded
+      return;
+   }
+   lock.unlock();
 
-      for (auto& message : messages)
+   logger_->debug("Load Archive: {}, {}", util::TimeString(date), pil);
+
+   // Query for products
+   const auto& productIds = iemApiProvider_->ListTextProducts(date, {}, pil);
+
+   if (productIds.has_value())
+   {
+      logger_->debug("Loading {} {} products", productIds.value().size(), pil);
+
+      // Load listed products
+      auto products = iemApiProvider_->LoadTextProducts(productIds.value());
+
+      for (auto& product : products)
       {
-         HandleMessage(message);
+         const auto& messages = product->messages();
+
+         for (auto& message : messages)
+         {
+            HandleMessage(message);
+         }
       }
+
+      lock.lock();
+
+      // Ensure the archive map still contains the date, and has not been pruned
+      if (archiveMap_.contains(date))
+      {
+         // Store the products associated with the PIL in the archive
+         dateArchive.try_emplace(pil, std::move(products));
+      }
+
+      lock.unlock();
    }
 }
 
 void TextEventManager::Impl::LoadArchives(std::chrono::sys_days date)
 {
+   logger_->trace("Load Archives: {}", util::TimeString(date));
+
+   UpdateArchiveDates(date);
+
    for (auto& pil : kPils_)
    {
       boost::asio::post(threadPool_,
@@ -380,6 +429,15 @@ void TextEventManager::Impl::Refresh()
       });
 }
 
+void TextEventManager::Impl::UpdateArchiveDates(std::chrono::sys_days date)
+{
+   std::unique_lock lock {archiveMutex_};
+
+   // Remove any existing occurrences of day, and add to the back of the list
+   archiveDates_.remove(date);
+   archiveDates_.push_back(date);
+}
+
 std::shared_ptr<TextEventManager> TextEventManager::Instance()
 {
    static std::weak_ptr<TextEventManager> textEventManagerReference_ {};

test/source/scwx/provider/iem_api_provider.test.cpp

@@ -18,15 +18,16 @@ TEST(IemApiProviderTest, ListTextProducts)
 
    auto torProducts = provider.ListTextProducts(date, {}, "TOR");
 
-   EXPECT_EQ(torProducts.size(), 35);
+   ASSERT_EQ(torProducts.has_value(), true);
+   EXPECT_EQ(torProducts.value().size(), 35);
 
-   if (torProducts.size() >= 1)
+   if (torProducts.value().size() >= 1)
    {
-      EXPECT_EQ(torProducts.at(0), "202303250016-KMEG-WFUS54-TORMEG");
+      EXPECT_EQ(torProducts.value().at(0), "202303250016-KMEG-WFUS54-TORMEG");
    }
-   if (torProducts.size() >= 35)
+   if (torProducts.value().size() >= 35)
    {
-      EXPECT_EQ(torProducts.at(34), "202303252015-KFFC-WFUS52-TORFFC");
+      EXPECT_EQ(torProducts.value().at(34), "202303252015-KFFC-WFUS52-TORFFC");
    }
 }
 

wxdata/include/scwx/provider/iem_api_provider.hpp

@@ -5,6 +5,8 @@
 #include <memory>
 #include <string>
 
+#include <boost/outcome/result.hpp>
+
 namespace scwx::provider
 {
 
@@ -23,7 +25,7 @@ public:
    IemApiProvider(IemApiProvider&&) noexcept;
    IemApiProvider& operator=(IemApiProvider&&) noexcept;
 
-   static std::vector<std::string>
+   static boost::outcome_v2::result<std::vector<std::string>>
    ListTextProducts(std::chrono::sys_time<std::chrono::days> date,
                     std::optional<std::string_view>          cccc = {},
                     std::optional<std::string_view>          pil  = {});

wxdata/source/scwx/provider/iem_api_provider.cpp

@@ -35,7 +35,7 @@ IemApiProvider::~IemApiProvider() = default;
 IemApiProvider::IemApiProvider(IemApiProvider&&) noexcept            = default;
 IemApiProvider& IemApiProvider::operator=(IemApiProvider&&) noexcept = default;
 
-std::vector<std::string>
+boost::outcome_v2::result<std::vector<std::string>>
 IemApiProvider::ListTextProducts(std::chrono::sys_time<std::chrono::days> date,
                                  std::optional<std::string_view>          cccc,
                                  std::optional<std::string_view>          pil)
@@ -93,6 +93,8 @@ IemApiProvider::ListTextProducts(std::chrono::sys_time<std::chrono::days> date,
       {
          // Unexpected bad response
          logger_->warn("Error parsing JSON: {}", ex.what());
+         return boost::system::errc::make_error_code(
+            boost::system::errc::bad_message);
       }
    }
    else if (response.status_code == cpr::status::HTTP_BAD_REQUEST &&
@@ -109,6 +111,9 @@ IemApiProvider::ListTextProducts(std::chrono::sys_time<std::chrono::days> date,
          // Unexpected bad response
          logger_->warn("Error parsing bad response: {}", ex.what());
       }
+
+      return boost::system::errc::make_error_code(
+         boost::system::errc::invalid_argument);
    }
    else if (response.status_code == cpr::status::HTTP_UNPROCESSABLE_ENTITY &&
             json != nullptr)
@@ -125,10 +130,16 @@ IemApiProvider::ListTextProducts(std::chrono::sys_time<std::chrono::days> date,
          // Unexpected bad response
          logger_->warn("Error parsing validation error: {}", ex.what());
       }
+
+      return boost::system::errc::make_error_code(
+         boost::system::errc::no_message_available);
    }
    else
    {
       logger_->warn("Could not list text products: {}", response.status_line);
+
+      return boost::system::errc::make_error_code(
+         boost::system::errc::no_message);
    }
 
    return textProducts;