Rewrite warnings provider to use HEAD requests instead of directory listing to find recent warnings

scwx-qt/source/scwx/qt/manager/text_event_manager.cpp

@@ -22,8 +22,10 @@ namespace manager
 static const std::string logPrefix_ = "scwx::qt::manager::text_event_manager";
 static const auto        logger_    = scwx::util::Logger::Create(logPrefix_);
 
-static const std::string& kDefaultWarningsProviderUrl {
+static constexpr std::chrono::hours kInitialLoadHistoryDuration_ =
-   "https://warnings.allisonhouse.com"};
+   std::chrono::days {3};
+static constexpr std::chrono::hours kDefaultLoadHistoryDuration_ =
+   std::chrono::hours {1};
 
 class TextEventManager::Impl
 {
@@ -42,7 +44,9 @@ public:
 
       warningsProviderChangedCallbackUuid_ =
          generalSettings.warnings_provider().RegisterValueChangedCallback(
-            [this](const std::string& value) {
+            [this](const std::string& value)
+            {
+               loadHistoryDuration_ = kInitialLoadHistoryDuration_;
                warningsProvider_ =
                   std::make_shared<provider::WarningsProvider>(value);
             });
@@ -94,6 +98,8 @@ public:
    std::shared_mutex textEventMutex_;
 
    std::shared_ptr<provider::WarningsProvider> warningsProvider_ {nullptr};
+   std::chrono::hours loadHistoryDuration_ {kInitialLoadHistoryDuration_};
+   std::chrono::sys_time<std::chrono::hours> prevLoadTime_ {};
 
    boost::uuids::uuid warningsProviderChangedCallbackUuid_ {};
 };
@@ -254,13 +260,26 @@ void TextEventManager::Impl::Refresh()
    std::shared_ptr<provider::WarningsProvider> warningsProvider =
       warningsProvider_;
 
-   // Update the file listing from the warnings provider
+   // Load updated files from the warnings provider
-   auto [newFiles, totalFiles] = warningsProvider->ListFiles();
+   // Start time should default to:
+   // - 3 days of history for the first load
+   // - 1 hour of history for subsequent loads
+   // If the time jumps, we should attempt to load from no later than the
+   // previous load time
+   auto loadTime =
+      std::chrono::floor<std::chrono::hours>(std::chrono::system_clock::now());
+   auto startTime = loadTime - loadHistoryDuration_;
 
-   if (newFiles > 0)
+   if (prevLoadTime_ != std::chrono::sys_time<std::chrono::hours> {})
    {
-      // Load new files
+      startTime = std::min(startTime, prevLoadTime_);
-      auto updatedFiles = warningsProvider->LoadUpdatedFiles();
+   }
+
+   auto updatedFiles = warningsProvider->LoadUpdatedFiles(startTime);
+
+   // Store the load time and reset the load history duration
+   prevLoadTime_        = loadTime;
+   loadHistoryDuration_ = kDefaultLoadHistoryDuration_;
 
    // Handle messages
    for (auto& file : updatedFiles)
@@ -270,7 +289,6 @@ void TextEventManager::Impl::Refresh()
          HandleMessage(message);
       }
    }
-   }
 
    // Schedule another update in 15 seconds
    using namespace std::chrono;

wxdata/include/scwx/provider/warnings_provider.hpp

@@ -22,10 +22,8 @@ public:
    WarningsProvider(WarningsProvider&&) noexcept;
    WarningsProvider& operator=(WarningsProvider&&) noexcept;
 
-   std::pair<size_t, size_t>
-   ListFiles(std::chrono::system_clock::time_point newerThan = {});
    std::vector<std::shared_ptr<awips::TextProductFile>>
-   LoadUpdatedFiles(std::chrono::system_clock::time_point newerThan = {});
+   LoadUpdatedFiles(std::chrono::sys_time<std::chrono::hours> newerThan = {});
 
 private:
    class Impl;

wxdata/source/scwx/provider/warnings_provider.cpp

@@ -1,9 +1,18 @@
-#include <scwx/provider/warnings_provider.hpp>
+// Prevent redefinition of __cpp_lib_format
-#include <scwx/network/dir_list.hpp>
+#if defined(_MSC_VER)
-#include <scwx/util/logger.hpp>
+#   include <yvals_core.h>
+#endif
 
-#include <ranges>
+// Enable chrono formatters
-#include <shared_mutex>
+#ifndef __cpp_lib_format
+#   define __cpp_lib_format 202110L
+#endif
+
+#include <scwx/provider/warnings_provider.hpp>
+#include <scwx/util/logger.hpp>
+#include <scwx/util/time.hpp>
+
+#include <mutex>
 
 #if defined(_MSC_VER)
 #   pragma warning(push, 0)
@@ -11,8 +20,6 @@
 
 #define LIBXML_HTML_ENABLED
 #include <cpr/cpr.h>
-#include <libxml/HTMLparser.h>
-#include <re2/re2.h>
 
 #if (__cpp_lib_chrono < 201907L)
 #   include <date/date.h>
@@ -35,13 +42,17 @@ class WarningsProvider::Impl
 public:
    struct FileInfoRecord
    {
-      std::chrono::system_clock::time_point startTime_ {};
+      FileInfoRecord(const std::string& contentLength,
-      std::chrono::system_clock::time_point lastModified_ {};
+                     const std::string& lastModified) :
-      size_t                                size_ {};
+          contentLengthStr_ {contentLength}, lastModifiedStr_ {lastModified}
-      bool                                  updated_ {};
+      {
+      }
+
+      std::string contentLengthStr_ {};
+      std::string lastModifiedStr_ {};
    };
 
-   typedef std::map<std::string, FileInfoRecord> WarningFileMap;
+   using WarningFileMap = std::map<std::string, FileInfoRecord>;
 
    explicit Impl(const std::string& baseUrl) :
        baseUrl_ {baseUrl}, files_ {}, filesMutex_ {}
@@ -50,10 +61,13 @@ public:
 
    ~Impl() {}
 
+   bool UpdateFileRecord(const cpr::Response& response,
+                         const std::string&   filename);
+
    std::string baseUrl_;
 
    WarningFileMap files_;
-   std::shared_mutex filesMutex_;
+   std::mutex     filesMutex_;
 };
 
 WarningsProvider::WarningsProvider(const std::string& baseUrl) :
@@ -66,131 +80,94 @@ WarningsProvider::WarningsProvider(WarningsProvider&&) noexcept = default;
 WarningsProvider&
 WarningsProvider::operator=(WarningsProvider&&) noexcept = default;
 
-std::pair<size_t, size_t>
+std::vector<std::shared_ptr<awips::TextProductFile>>
-WarningsProvider::ListFiles(std::chrono::system_clock::time_point newerThan)
+WarningsProvider::LoadUpdatedFiles(
+   std::chrono::sys_time<std::chrono::hours> startTime)
 {
    using namespace std::chrono;
 
-#if (__cpp_lib_chrono < 201907L)
+#if (__cpp_lib_chrono >= 201907L)
+   namespace date = std::chrono;
+   namespace df   = std;
+#else
    using namespace date;
+   namespace df = date;
 #endif
 
-   static constexpr LazyRE2 reWarningsFilename = {
+   std::vector<
-      "warnings_[0-9]{8}_[0-9]{2}.txt"};
+      std::pair<std::string,
-   static const std::string dateTimeFormat {"warnings_%Y%m%d_%H.txt"};
+                cpr::AsyncWrapper<std::optional<cpr::AsyncResponse>, false>>>
-
+                                                        asyncCallbacks;
-   logger_->trace("Listing files");
-
-   size_t updatedObjects = 0;
-   size_t totalObjects   = 0;
-
-   // Perform a directory listing
-   auto records = network::DirList(p->baseUrl_);
-
-   // Sort records by filename
-   std::sort(records.begin(),
-             records.end(),
-             [](auto& a, auto& b) { return a.filename_ < b.filename_; });
-
-   // Filter warning records
-   auto warningRecords =
-      records |
-      std::views::filter(
-         [](auto& record)
-         {
-            return record.type_ == std::filesystem::file_type::regular &&
-                   RE2::FullMatch(record.filename_, *reWarningsFilename);
-         });
-
-   std::unique_lock lock(p->filesMutex_);
-
-   Impl::WarningFileMap warningFileMap;
-
-   // Store records
-   for (auto& record : warningRecords)
-   {
-      // Determine start time
-      std::chrono::sys_time<hours> startTime;
-      std::istringstream           ssFilename {record.filename_};
-
-      ssFilename >> parse(dateTimeFormat, startTime);
-
-      // If start time is valid
-      if (!ssFilename.fail())
-      {
-         // Determine if the record should be marked updated
-         bool updated = true;
-         auto it      = p->files_.find(record.filename_);
-         if (it != p->files_.cend())
-         {
-            auto& existingRecord = it->second;
-
-            updated = existingRecord.updated_ ||
-                      record.size_ != existingRecord.size_ ||
-                      record.mtime_ != existingRecord.lastModified_;
-         }
-
-         // Update object counts, but only if newer than threshold
-         if (newerThan < startTime)
-         {
-            if (updated)
-            {
-               ++updatedObjects;
-            }
-            ++totalObjects;
-         }
-
-         // Store record
-         warningFileMap.emplace(
-            std::piecewise_construct,
-            std::forward_as_tuple(record.filename_),
-            std::forward_as_tuple(
-               startTime, record.mtime_, record.size_, updated));
-      }
-   }
-
-   p->files_ = std::move(warningFileMap);
-
-   return std::make_pair(updatedObjects, totalObjects);
-}
-
-std::vector<std::shared_ptr<awips::TextProductFile>>
-WarningsProvider::LoadUpdatedFiles(
-   std::chrono::system_clock::time_point newerThan)
-{
-   logger_->debug("Loading updated files");
-
    std::vector<std::shared_ptr<awips::TextProductFile>> updatedFiles;
 
-   std::vector<std::pair<std::string, cpr::AsyncResponse>> asyncResponses;
+   std::chrono::sys_time<std::chrono::hours> now =
+      std::chrono::floor<std::chrono::hours>(std::chrono::system_clock::now());
+   std::chrono::sys_time<std::chrono::hours> currentHour =
+      (startTime != std::chrono::sys_time<std::chrono::hours> {}) ?
+         startTime :
+         now - std::chrono::hours {1};
 
-   std::unique_lock lock(p->filesMutex_);
+   logger_->trace("Querying files newer than: {}", util::TimeString(startTime));
 
-   // For each warning file
+   while (currentHour <= now)
-   for (auto& record : p->files_)
    {
-      // If file is updated, and time is later than the threshold
+      static constexpr std::string_view dateTimeFormat {
-      if (record.second.updated_ && newerThan < record.second.startTime_)
+         "warnings_{:%Y%m%d_%H}.txt"};
-      {
+      const std::string filename = df::format(dateTimeFormat, currentHour);
-         // Retrieve warning file
+      const std::string url      = p->baseUrl_ + "/" + filename;
-         asyncResponses.emplace_back(
-            record.first,
-            cpr::GetAsync(cpr::Url {p->baseUrl_ + "/" + record.first}));
 
-         // Clear updated flag
+      logger_->trace("HEAD request for file: {}", filename);
-         record.second.updated_ = false;
+
+      asyncCallbacks.emplace_back(
+         filename,
+         cpr::HeadCallback(
+            [url, filename, this](
+               cpr::Response headResponse) -> std::optional<cpr::AsyncResponse>
+            {
+               if (headResponse.status_code == cpr::status::HTTP_OK)
+               {
+                  bool updated =
+                     p->UpdateFileRecord(headResponse, url); // TODO: filename
+
+                  if (updated)
+                  {
+                     logger_->trace("GET request for file: {}", filename);
+                     return cpr::GetAsync(cpr::Url {url});
                   }
                }
-
+               else if (headResponse.status_code != cpr::status::HTTP_NOT_FOUND)
-   lock.unlock();
-
-   // Wait for warning files to load
-   for (auto& asyncResponse : asyncResponses)
                {
-      cpr::Response response = asyncResponse.second.get();
+                  logger_->warn("HEAD request for file failed: {} ({})",
+                                url,
+                                headResponse.status_line);
+               }
+
+               return std::nullopt;
+            },
+            cpr::Url {url}));
+
+      // Query the next hour
+      currentHour += 1h;
+   }
+
+   for (auto& asyncCallback : asyncCallbacks)
+   {
+      auto& filename = asyncCallback.first;
+      auto& callback = asyncCallback.second;
+
+      if (callback.valid())
+      {
+         // Wait for futures to complete
+         callback.wait();
+         auto asyncResponse = callback.get();
+
+         if (asyncResponse.has_value())
+         {
+            auto response = asyncResponse.value().get();
+
             if (response.status_code == cpr::status::HTTP_OK)
             {
-         logger_->debug("Loading file: {}", asyncResponse.first);
+               logger_->debug("Loading file: {}", filename);
 
                // Load file
                std::shared_ptr<awips::TextProductFile> textProductFile {
@@ -201,10 +178,78 @@ WarningsProvider::LoadUpdatedFiles(
                   updatedFiles.push_back(textProductFile);
                }
             }
+            else
+            {
+               logger_->warn("Could not load file: {} ({})",
+                             filename,
+                             response.status_line);
+            }
+         }
+      }
+      else
+      {
+         logger_->error("Invalid future state");
+      }
    }
 
    return updatedFiles;
 }
 
+bool WarningsProvider::Impl::UpdateFileRecord(const cpr::Response& response,
+                                              const std::string&   filename)
+{
+   bool updated = false;
+
+   auto contentLengthIt = response.header.find("Content-Length");
+   auto lastModifiedIt  = response.header.find("Last-Modified");
+
+   std::string contentLength {};
+   std::string lastModified {};
+
+   if (contentLengthIt != response.header.cend())
+   {
+      contentLength = contentLengthIt->second;
+   }
+   if (lastModifiedIt != response.header.cend())
+   {
+      lastModified = lastModifiedIt->second;
+   }
+
+   std::unique_lock lock(filesMutex_);
+
+   auto it = files_.find(filename);
+   if (it != files_.cend())
+   {
+      auto& existingRecord = it->second;
+
+      // If the size or last modified changes, request an update
+
+      if (!contentLength.empty() &&
+          contentLength != existingRecord.contentLengthStr_)
+      {
+         // Size changed
+         existingRecord.contentLengthStr_ = contentLengthIt->second;
+         updated                          = true;
+      }
+      else if (!lastModified.empty() &&
+               lastModified != existingRecord.lastModifiedStr_)
+      {
+         // Last modified changed
+         existingRecord.lastModifiedStr_ = lastModifiedIt->second;
+         updated                         = true;
+      }
+   }
+   else
+   {
+      // File not found
+      files_.emplace(std::piecewise_construct,
+                     std::forward_as_tuple(filename),
+                     std::forward_as_tuple(contentLength, lastModified));
+      updated = true;
+   }
+
+   return updated;
+}
+
 } // namespace provider
 } // namespace scwx