// Prevent redefinition of __cpp_lib_format #if defined(_MSC_VER) # include #endif // Enable chrono formatters #ifndef __cpp_lib_format # define __cpp_lib_format 202110L #endif #include #include #include #include #if defined(_MSC_VER) # pragma warning(push, 0) #endif #define LIBXML_HTML_ENABLED #include #if (__cpp_lib_chrono < 201907L) # include #endif #if defined(_MSC_VER) # pragma warning(pop) #endif namespace scwx::provider { static const std::string logPrefix_ = "scwx::provider::warnings_provider"; static const auto logger_ = util::Logger::Create(logPrefix_); class WarningsProvider::Impl { public: struct FileInfoRecord { FileInfoRecord(std::string contentLength, std::string lastModified) : contentLengthStr_ {std::move(contentLength)}, lastModifiedStr_ {std::move(lastModified)} { } std::string contentLengthStr_ {}; std::string lastModifiedStr_ {}; }; using WarningFileMap = std::map; explicit Impl(std::string baseUrl) : baseUrl_ {std::move(baseUrl)}, files_ {}, filesMutex_ {} { } ~Impl() = default; Impl(const Impl&) = delete; Impl& operator=(const Impl&) = delete; Impl(const Impl&&) = delete; Impl& operator=(const Impl&&) = delete; bool UpdateFileRecord(const cpr::Response& response, const std::string& filename); std::string baseUrl_; WarningFileMap files_; std::mutex filesMutex_; }; WarningsProvider::WarningsProvider(const std::string& baseUrl) : p(std::make_unique(baseUrl)) { } WarningsProvider::~WarningsProvider() = default; WarningsProvider::WarningsProvider(WarningsProvider&&) noexcept = default; WarningsProvider& WarningsProvider::operator=(WarningsProvider&&) noexcept = default; std::vector> WarningsProvider::LoadUpdatedFiles( std::chrono::sys_time startTime) { using namespace std::chrono; #if (__cpp_lib_chrono >= 201907L) namespace df = std; static constexpr std::string_view kDateTimeFormat { "warnings_{:%Y%m%d_%H}.txt"}; #else using namespace date; namespace df = date; # define kDateTimeFormat "warnings_%Y%m%d_%H.txt" #endif std::vector< std::pair, false>>> asyncCallbacks; std::vector> updatedFiles; std::chrono::sys_time now = std::chrono::floor(std::chrono::system_clock::now()); std::chrono::sys_time currentHour = (startTime != std::chrono::sys_time {}) ? startTime : now - std::chrono::hours {1}; logger_->trace("Querying files newer than: {}", util::TimeString(startTime)); while (currentHour <= now) { const std::string filename = df::format(kDateTimeFormat, currentHour); const std::string url = p->baseUrl_ + "/" + filename; logger_->trace("HEAD request for file: {}", filename); asyncCallbacks.emplace_back( filename, cpr::HeadCallback( [url, filename, this]( cpr::Response headResponse) -> std::optional { if (headResponse.status_code == cpr::status::HTTP_OK) { bool updated = p->UpdateFileRecord(headResponse, filename); if (updated) { logger_->trace("GET request for file: {}", filename); return cpr::GetAsync(cpr::Url {url}); } } else if (headResponse.status_code != cpr::status::HTTP_NOT_FOUND) { logger_->warn("HEAD request for file failed: {} ({})", url, headResponse.status_line); } return std::nullopt; }, cpr::Url {url})); // Query the next hour currentHour += 1h; } for (auto& asyncCallback : asyncCallbacks) { auto& filename = asyncCallback.first; auto& callback = asyncCallback.second; if (callback.valid()) { // Wait for futures to complete callback.wait(); auto asyncResponse = callback.get(); if (asyncResponse.has_value()) { auto response = asyncResponse.value().get(); if (response.status_code == cpr::status::HTTP_OK) { logger_->debug("Loading file: {}", filename); // Load file std::shared_ptr textProductFile { std::make_shared()}; std::istringstream responseBody {response.text}; if (textProductFile->LoadData(responseBody)) { updatedFiles.push_back(textProductFile); } } else { logger_->warn("Could not load file: {} ({})", filename, response.status_line); } } } else { logger_->error("Invalid future state"); } } return updatedFiles; } bool WarningsProvider::Impl::UpdateFileRecord(const cpr::Response& response, const std::string& filename) { bool updated = false; auto contentLengthIt = response.header.find("Content-Length"); auto lastModifiedIt = response.header.find("Last-Modified"); std::string contentLength {}; std::string lastModified {}; if (contentLengthIt != response.header.cend()) { contentLength = contentLengthIt->second; } if (lastModifiedIt != response.header.cend()) { lastModified = lastModifiedIt->second; } std::unique_lock lock(filesMutex_); auto it = files_.find(filename); if (it != files_.cend()) { auto& existingRecord = it->second; // If the size or last modified changes, request an update if (!contentLength.empty() && contentLength != existingRecord.contentLengthStr_) { // Size changed existingRecord.contentLengthStr_ = contentLengthIt->second; updated = true; } else if (!lastModified.empty() && lastModified != existingRecord.lastModifiedStr_) { // Last modified changed existingRecord.lastModifiedStr_ = lastModifiedIt->second; updated = true; } } else { // File not found files_.emplace(std::piecewise_construct, std::forward_as_tuple(filename), std::forward_as_tuple(contentLength, lastModified)); updated = true; } return updated; } } // namespace scwx::provider