diff --git a/test/source/scwx/provider/warnings_provider.test.cpp b/test/source/scwx/provider/warnings_provider.test.cpp new file mode 100644 index 00000000..bc762dd6 --- /dev/null +++ b/test/source/scwx/provider/warnings_provider.test.cpp @@ -0,0 +1,58 @@ +#include + +#include + +namespace scwx +{ +namespace provider +{ + +static const std::string& kDefaultUrl {"https://warnings.allisonhouse.com"}; +static const std::string& kAlternateUrl {"http://warnings.cod.edu"}; + +class WarningsProviderTest : public testing::TestWithParam +{ +}; +TEST_P(WarningsProviderTest, ListFiles) +{ + WarningsProvider provider(GetParam()); + + auto [newObjects, totalObjects] = provider.ListFiles(); + + EXPECT_GT(newObjects, 0); + EXPECT_GT(totalObjects, 0); + EXPECT_EQ(newObjects, totalObjects); +} + +TEST_P(WarningsProviderTest, LoadUpdatedFiles) +{ + WarningsProvider provider(GetParam()); + + auto [newObjects, totalObjects] = provider.ListFiles(); + auto updatedFiles = provider.LoadUpdatedFiles(); + + EXPECT_GT(newObjects, 0); + EXPECT_GT(totalObjects, 0); + EXPECT_EQ(newObjects, totalObjects); + EXPECT_EQ(updatedFiles.size(), newObjects); + + auto [newObjects2, totalObjects2] = provider.ListFiles(); + auto updatedFiles2 = provider.LoadUpdatedFiles(); + + // There should be no more than 2 updated warnings files since the last query + // (assumption that the previous newest file was updated, and a new file was + // created on the hour) + EXPECT_LE(newObjects2, 2); + EXPECT_EQ(updatedFiles2.size(), newObjects2); + + // The total number of objects may have changed, since the oldest file could + // have dropped off the list + EXPECT_GT(totalObjects2, 0); +} + +INSTANTIATE_TEST_SUITE_P(WarningsProvider, + WarningsProviderTest, + testing::Values(kDefaultUrl, kAlternateUrl)); + +} // namespace provider +} // namespace scwx diff --git a/test/test.cmake b/test/test.cmake index efc9300d..76984a3a 100644 --- a/test/test.cmake +++ b/test/test.cmake @@ -17,7 +17,8 @@ set(SRC_COMMON_TESTS source/scwx/common/color_table.test.cpp source/scwx/common/products.test.cpp) set(SRC_NETWORK_TESTS source/scwx/network/dir_list.test.cpp) set(SRC_PROVIDER_TESTS source/scwx/provider/aws_level2_data_provider.test.cpp - source/scwx/provider/aws_level3_data_provider.test.cpp) + source/scwx/provider/aws_level3_data_provider.test.cpp + source/scwx/provider/warnings_provider.test.cpp) set(SRC_QT_CONFIG_TESTS source/scwx/qt/config/county_database.test.cpp source/scwx/qt/config/radar_site.test.cpp) set(SRC_QT_MANAGER_TESTS source/scwx/qt/manager/settings_manager.test.cpp) diff --git a/wxdata/include/scwx/provider/warnings_provider.hpp b/wxdata/include/scwx/provider/warnings_provider.hpp new file mode 100644 index 00000000..e519ec5d --- /dev/null +++ b/wxdata/include/scwx/provider/warnings_provider.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include + +namespace scwx +{ +namespace provider +{ + +/** + * @brief Warnings Provider + */ +class WarningsProvider +{ +public: + explicit WarningsProvider(const std::string& baseUrl); + ~WarningsProvider(); + + WarningsProvider(const WarningsProvider&) = delete; + WarningsProvider& operator=(const WarningsProvider&) = delete; + + WarningsProvider(WarningsProvider&&) noexcept; + WarningsProvider& operator=(WarningsProvider&&) noexcept; + + std::pair + ListFiles(std::chrono::system_clock::time_point newerThan = {}); + std::vector> + LoadUpdatedFiles(std::chrono::system_clock::time_point newerThan = {}); + +private: + class Impl; + std::unique_ptr p; +}; + +} // namespace provider +} // namespace scwx diff --git a/wxdata/source/scwx/awips/text_product_file.cpp b/wxdata/source/scwx/awips/text_product_file.cpp index ea99aeda..3edc7b2d 100644 --- a/wxdata/source/scwx/awips/text_product_file.cpp +++ b/wxdata/source/scwx/awips/text_product_file.cpp @@ -67,7 +67,7 @@ bool TextProductFile::LoadFile(const std::string& filename) bool TextProductFile::LoadData(std::istream& is) { - logger_->debug("Loading Data"); + logger_->trace("Loading Data"); while (!is.eof()) { diff --git a/wxdata/source/scwx/awips/wmo_header.cpp b/wxdata/source/scwx/awips/wmo_header.cpp index 6fa813cc..8e6a8519 100644 --- a/wxdata/source/scwx/awips/wmo_header.cpp +++ b/wxdata/source/scwx/awips/wmo_header.cpp @@ -53,7 +53,7 @@ public: WmoHeader::WmoHeader() : p(std::make_unique()) {} WmoHeader::~WmoHeader() = default; -WmoHeader::WmoHeader(WmoHeader&&) noexcept = default; +WmoHeader::WmoHeader(WmoHeader&&) noexcept = default; WmoHeader& WmoHeader::operator=(WmoHeader&&) noexcept = default; bool WmoHeader::operator==(const WmoHeader& o) const @@ -139,7 +139,7 @@ bool WmoHeader::Parse(std::istream& is) if (is.eof()) { - logger_->debug("Reached end of file"); + logger_->trace("Reached end of file"); headerValid = false; } else diff --git a/wxdata/source/scwx/provider/warnings_provider.cpp b/wxdata/source/scwx/provider/warnings_provider.cpp new file mode 100644 index 00000000..462d22b8 --- /dev/null +++ b/wxdata/source/scwx/provider/warnings_provider.cpp @@ -0,0 +1,198 @@ +#include +#include +#include + +#include +#include +#include + +#pragma warning(push, 0) +#define LIBXML_HTML_ENABLED +#include +#include +#pragma warning(pop) + +namespace scwx +{ +namespace provider +{ + +static const std::string logPrefix_ = "scwx::provider::warnings_provider"; +static const auto logger_ = util::Logger::Create(logPrefix_); + +static constexpr std::chrono::seconds kUpdatePeriod_ {15}; + +class WarningsProvider::Impl +{ +public: + struct FileInfoRecord + { + std::chrono::system_clock::time_point startTime_ {}; + std::chrono::system_clock::time_point lastModified_ {}; + size_t size_ {}; + bool updated_ {}; + }; + + typedef std::map WarningFileMap; + + explicit Impl(const std::string& baseUrl) : + baseUrl_ {baseUrl}, files_ {}, filesMutex_ {} + { + } + + ~Impl() {} + + std::string baseUrl_; + + WarningFileMap files_; + std::shared_mutex filesMutex_; +}; + +WarningsProvider::WarningsProvider(const std::string& baseUrl) : + p(std::make_unique(baseUrl)) +{ +} +WarningsProvider::~WarningsProvider() = default; + +WarningsProvider::WarningsProvider(WarningsProvider&&) noexcept = default; +WarningsProvider& +WarningsProvider::operator=(WarningsProvider&&) noexcept = default; + +std::pair +WarningsProvider::ListFiles(std::chrono::system_clock::time_point newerThan) +{ + using namespace std::chrono; + + static const std::regex reWarningsFilename { + "warnings_[0-9]{8}_[0-9]{2}.txt"}; + static const std::string dateTimeFormat {"warnings_%Y%m%d_%H.txt"}; + + logger_->trace("Listing files"); + + size_t updatedObjects = 0; + size_t totalObjects = 0; + + // Perform a directory listing + auto records = network::DirList(p->baseUrl_); + + // Sort records by filename + std::sort(records.begin(), + records.end(), + [](auto& a, auto& b) { return a.filename_ < b.filename_; }); + + // Filter warning records + auto warningRecords = + records | + std::views::filter( + [](auto& record) + { + return record.type_ == std::filesystem::file_type::regular && + std::regex_match(record.filename_, reWarningsFilename); + }); + + std::unique_lock lock(p->filesMutex_); + + Impl::WarningFileMap warningFileMap; + + // Store records + for (auto& record : warningRecords) + { + // Determine start time + sys_time startTime; + std::istringstream ssFilename {record.filename_}; + + ssFilename >> parse(dateTimeFormat, startTime); + + // If start time is valid + if (!ssFilename.fail()) + { + // Determine if the record should be marked updated + bool updated = true; + auto it = p->files_.find(record.filename_); + if (it != p->files_.cend()) + { + auto& existingRecord = it->second; + + updated = existingRecord.updated_ || + record.size_ != existingRecord.size_ || + record.mtime_ != existingRecord.lastModified_; + } + + // Update object counts, but only if newer than threshold + if (newerThan < startTime) + { + if (updated) + { + ++updatedObjects; + } + ++totalObjects; + } + + // Store record + warningFileMap.emplace( + std::piecewise_construct, + std::forward_as_tuple(record.filename_), + std::forward_as_tuple( + startTime, record.mtime_, record.size_, updated)); + } + } + + p->files_ = std::move(warningFileMap); + + return std::make_pair(updatedObjects, totalObjects); +} + +std::vector> +WarningsProvider::LoadUpdatedFiles( + std::chrono::system_clock::time_point newerThan) +{ + logger_->debug("Loading updated files"); + + std::vector> updatedFiles; + + std::vector> asyncResponses; + + std::unique_lock lock(p->filesMutex_); + + // For each warning file + for (auto& record : p->files_) + { + // If file is updated, and time is later than the threshold + if (record.second.updated_ && newerThan < record.second.startTime_) + { + // Retrieve warning file + asyncResponses.emplace_back( + record.first, + cpr::GetAsync(cpr::Url {p->baseUrl_ + "/" + record.first})); + + // Clear updated flag + record.second.updated_ = false; + } + } + + lock.unlock(); + + // Wait for warning files to load + for (auto& asyncResponse : asyncResponses) + { + cpr::Response response = asyncResponse.second.get(); + if (response.status_code == cpr::status::HTTP_OK) + { + logger_->debug("Loading file: {}", asyncResponse.first); + + // Load file + std::shared_ptr textProductFile { + std::make_shared()}; + std::istringstream responseBody {response.text}; + if (textProductFile->LoadData(responseBody)) + { + updatedFiles.push_back(textProductFile); + } + } + } + + return updatedFiles; +} + +} // namespace provider +} // namespace scwx diff --git a/wxdata/wxdata.cmake b/wxdata/wxdata.cmake index 3bbfb561..cfad9421 100644 --- a/wxdata/wxdata.cmake +++ b/wxdata/wxdata.cmake @@ -48,12 +48,14 @@ set(HDR_PROVIDER include/scwx/provider/aws_level2_data_provider.hpp include/scwx/provider/aws_level3_data_provider.hpp include/scwx/provider/aws_nexrad_data_provider.hpp include/scwx/provider/nexrad_data_provider.hpp - include/scwx/provider/nexrad_data_provider_factory.hpp) + include/scwx/provider/nexrad_data_provider_factory.hpp + include/scwx/provider/warnings_provider.hpp) set(SRC_PROVIDER source/scwx/provider/aws_level2_data_provider.cpp source/scwx/provider/aws_level3_data_provider.cpp source/scwx/provider/aws_nexrad_data_provider.cpp source/scwx/provider/nexrad_data_provider.cpp - source/scwx/provider/nexrad_data_provider_factory.cpp) + source/scwx/provider/nexrad_data_provider_factory.cpp + source/scwx/provider/warnings_provider.cpp) set(HDR_UTIL include/scwx/util/environment.hpp include/scwx/util/float.hpp include/scwx/util/hash.hpp