diff --git a/.clang-tidy b/.clang-tidy index 3c98e81d..645c9c05 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -6,10 +6,11 @@ Checks: - 'misc-*' - 'modernize-*' - 'performance-*' + - '-bugprone-easily-swappable-parameters' - '-cppcoreguidelines-pro-type-reinterpret-cast' - '-misc-include-cleaner' - '-misc-non-private-member-variables-in-classes' - - '-modernize-use-trailing-return-type' - - '-bugprone-easily-swappable-parameters' + - '-misc-use-anonymous-namespace' - '-modernize-return-braced-init-list' + - '-modernize-use-trailing-return-type' FormatStyle: 'file' diff --git a/.github/workflows/clang-tidy-review.yml b/.github/workflows/clang-tidy-review.yml index c37236d6..a7ec09ff 100644 --- a/.github/workflows/clang-tidy-review.yml +++ b/.github/workflows/clang-tidy-review.yml @@ -126,7 +126,7 @@ jobs: --build_dir='../build' \ --base_dir='${{ github.workspace }}/source' \ --clang_tidy_checks='' \ - --config_file='.clang-tidy' \ + --config_file='' \ --include='*.[ch],*.[ch]xx,*.[chi]pp,*.[ch]++,*.cc,*.hh' \ --exclude='' \ --apt-packages='' \ diff --git a/ACKNOWLEDGEMENTS.md b/ACKNOWLEDGEMENTS.md index 4aec61d2..6f7cd4ef 100644 --- a/ACKNOWLEDGEMENTS.md +++ b/ACKNOWLEDGEMENTS.md @@ -36,6 +36,7 @@ Supercell Wx uses code from the following dependencies: | [OpenSSL](https://www.openssl.org/) | [OpenSSL License](https://spdx.org/licenses/OpenSSL.html) | | [Qt](https://www.qt.io/) | [GNU Lesser General Public License v3.0 only](https://spdx.org/licenses/LGPL-3.0-only.html) | Qt Core, Qt GUI, Qt Multimedia, Qt Network, Qt OpenGL, Qt Positioning, Qt Serial Port, Qt SQL, Qt SVG, Qt Widgets
Additional Licenses: https://doc.qt.io/qt-6/licenses-used-in-qt.html | | [qt6ct](https://github.com/trialuser02/qt6ct) | [BSD 2-Clause "Simplified" License](https://spdx.org/licenses/BSD-2-Clause.html) | +| [range-v3](https://github.com/ericniebler/range-v3) | [Boost Software License 1.0](https://spdx.org/licenses/BSL-1.0.html)
[MIT License](https://spdx.org/licenses/MIT.html)
[Stepanov and McJones, "Elements of Programming" license](https://github.com/ericniebler/range-v3/tree/0.12.0?tab=License-1-ov-file)
[SGI C++ Standard Template Library license](https://github.com/ericniebler/range-v3/tree/0.12.0?tab=License-1-ov-file) | | [re2](https://github.com/google/re2) | [BSD 3-Clause "New" or "Revised" License](https://spdx.org/licenses/BSD-3-Clause.html) | | [spdlog](https://github.com/gabime/spdlog) | [MIT License](https://spdx.org/licenses/MIT.html) | | [SQLite](https://www.sqlite.org/) | Public Domain | diff --git a/conanfile.py b/conanfile.py index e68c9f39..ea9c6ab9 100644 --- a/conanfile.py +++ b/conanfile.py @@ -18,6 +18,7 @@ class SupercellWxConan(ConanFile): "libpng/1.6.47", "libxml2/2.13.6", "openssl/3.4.1", + "range-v3/0.12.0", "re2/20240702", "spdlog/1.15.1", "sqlite3/3.49.1", diff --git a/scwx-qt/scwx-qt.cmake b/scwx-qt/scwx-qt.cmake index 89b31011..1628607a 100644 --- a/scwx-qt/scwx-qt.cmake +++ b/scwx-qt/scwx-qt.cmake @@ -11,6 +11,8 @@ set(CMAKE_AUTORCC ON) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) +OPTION(SCWX_DISABLE_CONSOLE "Disables the Windows console in release mode" ON) + find_package(Boost) find_package(Fontconfig) find_package(geographiclib) @@ -615,7 +617,9 @@ set_target_properties(scwx-qt_update_radar_sites PROPERTIES FOLDER generate) if (WIN32) set(APP_ICON_RESOURCE_WINDOWS ${RESOURCE_OUTPUT}) qt_add_executable(supercell-wx ${EXECUTABLE_SOURCES} ${APP_ICON_RESOURCE_WINDOWS}) - set_target_properties(supercell-wx PROPERTIES WIN32_EXECUTABLE $,TRUE,FALSE>) + if (SCWX_DISABLE_CONSOLE) + set_target_properties(supercell-wx PROPERTIES WIN32_EXECUTABLE $,TRUE,FALSE>) + endif() else() qt_add_executable(supercell-wx ${EXECUTABLE_SOURCES}) endif() diff --git a/scwx-qt/source/scwx/qt/config/radar_site.cpp b/scwx-qt/source/scwx/qt/config/radar_site.cpp index 5c1dba2e..69815636 100644 --- a/scwx-qt/source/scwx/qt/config/radar_site.cpp +++ b/scwx-qt/source/scwx/qt/config/radar_site.cpp @@ -245,7 +245,7 @@ size_t RadarSite::ReadConfig(const std::string& path) bool dataValid = true; size_t sitesAdded = 0; - boost::json::value j = util::json::ReadJsonFile(path); + boost::json::value j = util::json::ReadJsonQFile(path); dataValid = j.is_array(); diff --git a/scwx-qt/source/scwx/qt/main/main_window.cpp b/scwx-qt/source/scwx/qt/main/main_window.cpp index 71f4d00c..d213b4fd 100644 --- a/scwx-qt/source/scwx/qt/main/main_window.cpp +++ b/scwx-qt/source/scwx/qt/main/main_window.cpp @@ -1018,6 +1018,7 @@ void MainWindowImpl::ConnectAnimationSignals() for (auto map : maps_) { map->SelectTime(dateTime); + textEventManager_->SelectTime(dateTime); QMetaObject::invokeMethod( map, static_cast(&QWidget::update)); } diff --git a/scwx-qt/source/scwx/qt/manager/alert_manager.cpp b/scwx-qt/source/scwx/qt/manager/alert_manager.cpp index 757754a9..748e0943 100644 --- a/scwx-qt/source/scwx/qt/manager/alert_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/alert_manager.cpp @@ -138,8 +138,10 @@ common::Coordinate AlertManager::Impl::CurrentCoordinate( void AlertManager::Impl::HandleAlert(const types::TextEventKey& key, size_t messageIndex) const { + auto messages = textEventManager_->message_list(key); + // Skip alert if there are more messages to be processed - if (messageIndex + 1 < textEventManager_->message_count(key)) + if (messages.empty() || messageIndex + 1 < messages.size()) { return; } @@ -153,7 +155,7 @@ void AlertManager::Impl::HandleAlert(const types::TextEventKey& key, audioSettings.alert_radius().GetValue()); std::string alertWFO = audioSettings.alert_wfo().GetValue(); - auto message = textEventManager_->message_list(key).at(messageIndex); + auto message = messages.at(messageIndex); for (auto& segment : message->segments()) { diff --git a/scwx-qt/source/scwx/qt/manager/marker_manager.cpp b/scwx-qt/source/scwx/qt/manager/marker_manager.cpp index 952dea44..8af310ec 100644 --- a/scwx-qt/source/scwx/qt/manager/marker_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/marker_manager.cpp @@ -1,14 +1,15 @@ #include #include #include -#include #include #include #include +#include #include #include #include +#include #include #include #include @@ -62,7 +63,7 @@ public: bool markerFileRead_ {false}; - void InitalizeIds(); + void InitalizeIds(); types::MarkerId NewId(); types::MarkerId lastId_ {0}; }; @@ -70,15 +71,9 @@ public: class MarkerManager::Impl::MarkerRecord { public: - MarkerRecord(const types::MarkerInfo& info) : - markerInfo_ {info} - { - } + MarkerRecord(types::MarkerInfo info) : markerInfo_ {std::move(info)} {} - const types::MarkerInfo& toMarkerInfo() - { - return markerInfo_; - } + const types::MarkerInfo& toMarkerInfo() { return markerInfo_; } types::MarkerInfo markerInfo_; @@ -175,7 +170,7 @@ void MarkerManager::Impl::ReadMarkerSettings() // Determine if marker settings exists if (std::filesystem::exists(markerSettingsPath_)) { - markerJson = util::json::ReadJsonFile(markerSettingsPath_); + markerJson = scwx::util::json::ReadJsonFile(markerSettingsPath_); } if (markerJson != nullptr && markerJson.is_array()) @@ -224,8 +219,8 @@ void MarkerManager::Impl::WriteMarkerSettings() logger_->info("Saving location marker settings"); const std::shared_lock lock(markerRecordLock_); - auto markerJson = boost::json::value_from(markerRecords_); - util::json::WriteJsonFile(markerSettingsPath_, markerJson); + auto markerJson = boost::json::value_from(markerRecords_); + scwx::util::json::WriteJsonFile(markerSettingsPath_, markerJson); } std::shared_ptr @@ -357,10 +352,11 @@ types::MarkerId MarkerManager::add_marker(const types::MarkerInfo& marker) types::MarkerId id; { const std::unique_lock lock(p->markerRecordLock_); - id = p->NewId(); + id = p->NewId(); size_t index = p->markerRecords_.size(); p->idToIndex_.emplace(id, index); - p->markerRecords_.emplace_back(std::make_shared(marker)); + p->markerRecords_.emplace_back( + std::make_shared(marker)); p->markerRecords_[index]->markerInfo_.id = id; add_icon(marker.iconName); @@ -499,7 +495,6 @@ void MarkerManager::set_marker_settings_path(const std::string& path) p->markerSettingsPath_ = path; } - std::shared_ptr MarkerManager::Instance() { static std::weak_ptr markerManagerReference_ {}; diff --git a/scwx-qt/source/scwx/qt/manager/placefile_manager.cpp b/scwx-qt/source/scwx/qt/manager/placefile_manager.cpp index a6158773..d85f9e40 100644 --- a/scwx-qt/source/scwx/qt/manager/placefile_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/placefile_manager.cpp @@ -2,10 +2,10 @@ #include #include #include -#include #include #include #include +#include #include #include @@ -385,7 +385,7 @@ void PlacefileManager::Impl::ReadPlacefileSettings() // Determine if placefile settings exists if (std::filesystem::exists(placefileSettingsPath_)) { - placefileJson = util::json::ReadJsonFile(placefileSettingsPath_); + placefileJson = scwx::util::json::ReadJsonFile(placefileSettingsPath_); } // If placefile settings was successfully read @@ -428,7 +428,7 @@ void PlacefileManager::Impl::WritePlacefileSettings() std::shared_lock lock {placefileRecordLock_}; auto placefileJson = boost::json::value_from(placefileRecords_); - util::json::WriteJsonFile(placefileSettingsPath_, placefileJson); + scwx::util::json::WriteJsonFile(placefileSettingsPath_, placefileJson); } void PlacefileManager::SetRadarSite( diff --git a/scwx-qt/source/scwx/qt/manager/settings_manager.cpp b/scwx-qt/source/scwx/qt/manager/settings_manager.cpp index 5b2e9cbb..a47428bb 100644 --- a/scwx-qt/source/scwx/qt/manager/settings_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/settings_manager.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp index e48ecd48..8aa4c611 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp @@ -2,28 +2,61 @@ #include #include #include +#include #include #include +#include +#include +#include +#include #include #include #include #include #include +#include +#include +#include +#include +#include +#include -namespace scwx -{ -namespace qt -{ -namespace manager +#if (__cpp_lib_chrono < 201907L) +# include +#endif + +namespace scwx::qt::manager { +using namespace std::chrono_literals; + static const std::string logPrefix_ = "scwx::qt::manager::text_event_manager"; static const auto logger_ = scwx::util::Logger::Create(logPrefix_); -static const std::string& kDefaultWarningsProviderUrl { - "https://warnings.allisonhouse.com"}; +static constexpr std::chrono::hours kInitialLoadHistoryDuration_ = + std::chrono::days {3}; +static constexpr std::chrono::hours kDefaultLoadHistoryDuration_ = + std::chrono::hours {1}; + +static const std::array kPils_ = { + "FFS", "FFW", "MWS", "SMW", "SQW", "SVR", "SVS", "TOR"}; + +static const std:: + unordered_map> + kPilLoadWindows_ {{"FFS", {-24h, 1h}}, + {"FFW", {-24h, 1h}}, + {"MWS", {-4h, 1h}}, + {"SMW", {-4h, 1h}}, + {"SQW", {-4h, 1h}}, + {"SVR", {-4h, 1h}}, + {"SVS", {-4h, 1h}}, + {"TOR", {-4h, 1h}}}; + +// Widest load window provided by kPilLoadWindows_ +static const std::pair + kArchiveLoadWindow_ {-24h, 1h}; class TextEventManager::Impl { @@ -42,7 +75,9 @@ public: warningsProviderChangedCallbackUuid_ = generalSettings.warnings_provider().RegisterValueChangedCallback( - [this](const std::string& value) { + [this](const std::string& value) + { + loadHistoryDuration_ = kInitialLoadHistoryDuration_; warningsProvider_ = std::make_shared(value); }); @@ -76,11 +111,30 @@ public: threadPool_.join(); } - void HandleMessage(std::shared_ptr message); + Impl(const Impl&) = delete; + Impl& operator=(const Impl&) = delete; + Impl(const Impl&&) = delete; + Impl& operator=(const Impl&&) = delete; + + void HandleMessage(const std::shared_ptr& message, + bool archiveEvent = false); + template + requires std::same_as, + std::chrono::sys_days> + void ListArchives(DateRange dates); + void LoadArchives(std::chrono::system_clock::time_point dateTime); + void PruneArchives(); void RefreshAsync(); void Refresh(); + template + requires std::same_as, + std::chrono::sys_days> + void UpdateArchiveDates(DateRange dates); - boost::asio::thread_pool threadPool_ {1u}; + // Thread pool sized for: + // - Live Refresh (1x) + // - Archive Loading (1x) + boost::asio::thread_pool threadPool_ {2u}; TextEventManager* self_; @@ -95,6 +149,27 @@ public: std::shared_ptr warningsProvider_ {nullptr}; + std::chrono::hours loadHistoryDuration_ {kInitialLoadHistoryDuration_}; + std::chrono::sys_time prevLoadTime_ {}; + std::chrono::sys_days archiveLimit_ {}; + + std::mutex archiveMutex_ {}; + std::list archiveDates_ {}; + + std::mutex archiveEventKeyMutex_ {}; + std::map>> + archiveEventKeys_ {}; + std::unordered_set> + liveEventKeys_ {}; + + std::mutex unloadedProductMapMutex_ {}; + std::map> + unloadedProductMap_; + boost::uuids::uuid warningsProviderChangedCallbackUuid_ {}; }; @@ -164,9 +239,56 @@ void TextEventManager::LoadFile(const std::string& filename) }); } -void TextEventManager::Impl::HandleMessage( - std::shared_ptr message) +void TextEventManager::SelectTime( + std::chrono::system_clock::time_point dateTime) { + if (dateTime == std::chrono::system_clock::time_point {}) + { + // Ignore a default date/time selection + return; + } + + logger_->trace("Select Time: {}", util::TimeString(dateTime)); + + boost::asio::post( + p->threadPool_, + [dateTime, this]() + { + try + { + const auto today = std::chrono::floor(dateTime); + const auto yesterday = today - std::chrono::days {1}; + const auto tomorrow = today + std::chrono::days {1}; + const auto dateArray = std::array {yesterday, today, tomorrow}; + + const auto dates = + dateArray | + ranges::views::filter( + [this](const auto& date) + { + return p->archiveLimit_ == std::chrono::sys_days {} || + date < p->archiveLimit_; + }); + + const std::unique_lock lock {p->archiveMutex_}; + + p->UpdateArchiveDates(dates); + p->ListArchives(dates); + p->LoadArchives(dateTime); + p->PruneArchives(); + } + catch (const std::exception& ex) + { + logger_->error(ex.what()); + } + }); +} + +void TextEventManager::Impl::HandleMessage( + const std::shared_ptr& message, bool archiveEvent) +{ + using namespace std::chrono_literals; + auto segments = message->segments(); // If there are no segments, skip this message @@ -187,21 +309,49 @@ void TextEventManager::Impl::HandleMessage( } } + // Determine year + const std::chrono::year_month_day wmoDate = + std::chrono::floor( + message->wmo_header()->GetDateTime()); + const std::chrono::year wmoYear = wmoDate.year(); + std::unique_lock lock(textEventMutex_); // Find a matching event in the event map auto& vtecString = segments[0]->header_->vtecString_; - types::TextEventKey key {vtecString[0].pVtec_}; + types::TextEventKey key {vtecString[0].pVtec_, wmoYear}; size_t messageIndex = 0; auto it = textEventMap_.find(key); bool updated = false; + if ( + // If there was no matching event + it == textEventMap_.cend() && + // The event is not new + vtecString[0].pVtec_.action() != awips::PVtec::Action::New && + // The message was on January 1 + wmoDate.month() == std::chrono::January && wmoDate.day() == 1d && + // This is at least the 10th ETN of the year + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers): Readability + vtecString[0].pVtec_.event_tracking_number() > 10) + { + // Attempt to find a matching event from last year + key = {vtecString[0].pVtec_, wmoYear - std::chrono::years {1}}; + it = textEventMap_.find(key); + } + if (it == textEventMap_.cend()) { // If there was no matching event, add the message to a new event textEventMap_.emplace(key, std::vector {message}); messageIndex = 0; updated = true; + + if (!archiveEvent) + { + // Add the Text Event Key to the list of live events to prevent pruning + liveEventKeys_.insert(key); + } } else if (std::find_if(it->second.cbegin(), it->second.cend(), @@ -214,16 +364,284 @@ void TextEventManager::Impl::HandleMessage( // If there was a matching event, and this message has not been stored // (WMO header equivalence check), add the updated message to the existing // event - messageIndex = it->second.size(); - it->second.push_back(message); + + // Determine the chronological sequence of the message. Note, if there + // were no time hints given to the WMO header, this will place the message + // at the end of the vector. + auto insertionPoint = std::upper_bound( + it->second.begin(), + it->second.end(), + message, + [](const std::shared_ptr& a, + const std::shared_ptr& b) + { + return a->wmo_header()->GetDateTime() < + b->wmo_header()->GetDateTime(); + }); + + // Insert the message in chronological order + messageIndex = std::distance(it->second.begin(), insertionPoint); + it->second.insert(insertionPoint, message); updated = true; }; + // If this is an archive event, and the key does not exist in the live events + // Assumption: A live event will always be loaded before a duplicate archive + // event + if (archiveEvent && !liveEventKeys_.contains(key)) + { + // Add the Text Event Key to the current date's archive + const std::unique_lock archiveEventKeyLock {archiveEventKeyMutex_}; + auto& archiveKeys = archiveEventKeys_[wmoDate]; + archiveKeys.insert(key); + } + lock.unlock(); if (updated) { - Q_EMIT self_->AlertUpdated(key, messageIndex); + Q_EMIT self_->AlertUpdated(key, messageIndex, message->uuid()); + } +} + +template + requires std::same_as, + std::chrono::sys_days> +void TextEventManager::Impl::ListArchives(DateRange dates) +{ + // Don't reload data that has already been loaded + auto filteredDates = + dates | + ranges::views::filter([this](const auto& date) + { return !unloadedProductMap_.contains(date); }); + + const auto dv = ranges::to(filteredDates); + + std::for_each( + std::execution::par, + dv.begin(), + dv.end(), + [this](const auto& date) + { + static const auto kEmptyRange_ = + ranges::views::single(std::string_view {}); + static const auto kPilsView_ = + kPils_ | + ranges::views::transform([](const std::string& pil) + { return std::string_view {pil}; }); + + const auto dateArray = std::array {date}; + + auto productEntries = provider::IemApiProvider::ListTextProducts( + dateArray | ranges::views::all, kEmptyRange_, kPilsView_); + + const std::unique_lock lock {unloadedProductMapMutex_}; + + if (productEntries.has_value()) + { + unloadedProductMap_.try_emplace( + date, + boost::container::stable_vector { + std::make_move_iterator(productEntries.value().begin()), + std::make_move_iterator(productEntries.value().end())}); + } + }); +} + +void TextEventManager::Impl::LoadArchives( + std::chrono::system_clock::time_point dateTime) +{ + using namespace std::chrono; + +#if (__cpp_lib_chrono >= 201907L) + namespace df = std; + + static constexpr std::string_view kDateFormat {"{:%Y%m%d%H%M}"}; +#else + using namespace date; + namespace df = date; + +# define kDateFormat "%Y%m%d%H%M" +#endif + + // Search unloaded products in the widest archive load window + const std::chrono::sys_days startDate = + std::chrono::floor(dateTime + + kArchiveLoadWindow_.first); + const std::chrono::sys_days endDate = std::chrono::floor( + dateTime + kArchiveLoadWindow_.second + std::chrono::days {1}); + + // Determine load windows for each PIL + std::unordered_map> + pilLoadWindowStrings; + + for (auto& loadWindow : kPilLoadWindows_) + { + const std::string& pil = loadWindow.first; + + pilLoadWindowStrings.insert_or_assign( + pil, + std::pair { + df::format(kDateFormat, (dateTime + loadWindow.second.first)), + df::format(kDateFormat, (dateTime + loadWindow.second.second))}); + } + + std::vector loadListEntries {}; + + for (auto date : boost::irange(startDate, endDate)) + { + auto mapIt = unloadedProductMap_.find(date); + if (mapIt == unloadedProductMap_.cend()) + { + continue; + } + + for (auto it = mapIt->second.begin(); it != mapIt->second.end();) + { + const auto& pil = it->pil_; + + // Check PIL + if (pil.size() >= 3) + { + auto pilPrefix = pil.substr(0, 3); + auto windowIt = pilLoadWindowStrings.find(pilPrefix); + + // Check Window + if (windowIt != pilLoadWindowStrings.cend()) + { + const auto& productId = it->productId_; + const auto& windowStart = windowIt->second.first; + const auto& windowEnd = windowIt->second.second; + + if (windowStart <= productId && productId <= windowEnd) + { + // Product matches, move it to the load list + loadListEntries.emplace_back(std::move(*it)); + it = mapIt->second.erase(it); + continue; + } + } + } + + // Current iterator was not matched + ++it; + } + } + + std::vector> products {}; + + // Load the load list + if (!loadListEntries.empty()) + { + auto loadView = loadListEntries | + ranges::views::transform([](const auto& entry) + { return entry.productId_; }); + products = provider::IemApiProvider::LoadTextProducts(loadView); + } + + // Process loaded products + for (auto& product : products) + { + const auto& messages = product->messages(); + + for (auto& message : messages) + { + HandleMessage(message, true); + } + } +} + +void TextEventManager::Impl::PruneArchives() +{ + static constexpr std::size_t kMaxArchiveDates_ = 5; + + std::unordered_set> + eventKeysToKeep {}; + std::unordered_set> + eventKeysToPrune {}; + + // Remove oldest dates from the archive + while (archiveDates_.size() > kMaxArchiveDates_) + { + archiveDates_.pop_front(); + } + + const std::unique_lock archiveEventKeyLock {archiveEventKeyMutex_}; + + // If there are the same number of dates in archiveEventKeys_, archiveDates_ + // and unloadedProductMap_, there is nothing to prune + if (archiveEventKeys_.size() == archiveDates_.size() && + unloadedProductMap_.size() == archiveDates_.size()) + { + // Nothing to prune + return; + } + + const std::unique_lock unloadedProductMapLock {unloadedProductMapMutex_}; + + for (auto it = archiveEventKeys_.begin(); it != archiveEventKeys_.end();) + { + const auto& date = it->first; + const auto& eventKeys = it->second; + + // If date is not in recent days map + if (std::find(archiveDates_.cbegin(), archiveDates_.cend(), date) == + archiveDates_.cend()) + { + // Prune these keys (unless they are in the eventKeysToKeep set) + eventKeysToPrune.insert(eventKeys.begin(), eventKeys.end()); + + // The date is not in the list of recent dates, remove it + it = archiveEventKeys_.erase(it); + } + else + { + // Make sure these keys don't get pruned + eventKeysToKeep.insert(eventKeys.begin(), eventKeys.end()); + + // The date is recent, keep it + ++it; + } + } + + for (auto it = unloadedProductMap_.begin(); it != unloadedProductMap_.end();) + { + const auto& date = it->first; + + // If date is not in recent days map + if (std::find(archiveDates_.cbegin(), archiveDates_.cend(), date) == + archiveDates_.cend()) + { + // The date is not in the list of recent dates, remove it + it = unloadedProductMap_.erase(it); + } + else + { + // The date is recent, keep it + ++it; + } + } + + // Remove elements from eventKeysToPrune if they are in eventKeysToKeep + for (const auto& eventKey : eventKeysToKeep) + { + eventKeysToPrune.erase(eventKey); + } + + // Remove eventKeysToPrune from textEventMap + for (const auto& eventKey : eventKeysToPrune) + { + textEventMap_.erase(eventKey); + } + + // If event keys were pruned, emit a signal + if (!eventKeysToPrune.empty()) + { + logger_->debug("Pruned {} archive events", eventKeysToPrune.size()); + + Q_EMIT self_->AlertsRemoved(eventKeysToPrune); } } @@ -254,21 +672,38 @@ void TextEventManager::Impl::Refresh() std::shared_ptr warningsProvider = warningsProvider_; - // Update the file listing from the warnings provider - auto [newFiles, totalFiles] = warningsProvider->ListFiles(); + // Load updated files from the warnings provider + // Start time should default to: + // - 3 days of history for the first load + // - 1 hour of history for subsequent loads + // If the time jumps, we should attempt to load from no later than the + // previous load time + auto loadTime = + std::chrono::floor(std::chrono::system_clock::now()); + auto startTime = loadTime - loadHistoryDuration_; - if (newFiles > 0) + if (prevLoadTime_ != std::chrono::sys_time {}) { - // Load new files - auto updatedFiles = warningsProvider->LoadUpdatedFiles(); + startTime = std::min(startTime, prevLoadTime_); + } - // Handle messages - for (auto& file : updatedFiles) + if (archiveLimit_ == std::chrono::sys_days {}) + { + archiveLimit_ = std::chrono::ceil(startTime); + } + + auto updatedFiles = warningsProvider->LoadUpdatedFiles(startTime); + + // Store the load time and reset the load history duration + prevLoadTime_ = loadTime; + loadHistoryDuration_ = kDefaultLoadHistoryDuration_; + + // Handle messages + for (auto& file : updatedFiles) + { + for (auto& message : file->messages()) { - for (auto& message : file->messages()) - { - HandleMessage(message); - } + HandleMessage(message); } } @@ -293,6 +728,19 @@ void TextEventManager::Impl::Refresh() }); } +template + requires std::same_as, + std::chrono::sys_days> +void TextEventManager::Impl::UpdateArchiveDates(DateRange dates) +{ + for (const auto& date : dates) + { + // Remove any existing occurrences of day, and add to the back of the list + archiveDates_.remove(date); + archiveDates_.push_back(date); + } +} + std::shared_ptr TextEventManager::Instance() { static std::weak_ptr textEventManagerReference_ {}; @@ -312,6 +760,4 @@ std::shared_ptr TextEventManager::Instance() return textEventManager; } -} // namespace manager -} // namespace qt -} // namespace scwx +} // namespace scwx::qt::manager diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.hpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.hpp index f97ca223..61affe6c 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.hpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.hpp @@ -3,9 +3,12 @@ #include #include +#include #include #include +#include +#include #include namespace scwx @@ -28,11 +31,18 @@ public: message_list(const types::TextEventKey& key) const; void LoadFile(const std::string& filename); + void SelectTime(std::chrono::system_clock::time_point dateTime); static std::shared_ptr Instance(); signals: - void AlertUpdated(const types::TextEventKey& key, size_t messageIndex); + void AlertsRemoved( + const std::unordered_set>& + keys); + void AlertUpdated(const types::TextEventKey& key, + std::size_t messageIndex, + boost::uuids::uuid uuid); private: class Impl; diff --git a/scwx-qt/source/scwx/qt/manager/timeline_manager.cpp b/scwx-qt/source/scwx/qt/manager/timeline_manager.cpp index f0c95e53..0bcf4f68 100644 --- a/scwx-qt/source/scwx/qt/manager/timeline_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/timeline_manager.cpp @@ -112,6 +112,11 @@ public: TimelineManager::TimelineManager() : p(std::make_unique(this)) {} TimelineManager::~TimelineManager() = default; +std::chrono::system_clock::time_point TimelineManager::GetSelectedTime() const +{ + return p->selectedTime_; +} + void TimelineManager::SetMapCount(std::size_t mapCount) { p->mapCount_ = mapCount; diff --git a/scwx-qt/source/scwx/qt/manager/timeline_manager.hpp b/scwx-qt/source/scwx/qt/manager/timeline_manager.hpp index 1a4154ac..054a8201 100644 --- a/scwx-qt/source/scwx/qt/manager/timeline_manager.hpp +++ b/scwx-qt/source/scwx/qt/manager/timeline_manager.hpp @@ -24,6 +24,8 @@ public: static std::shared_ptr Instance(); + [[nodiscard]] std::chrono::system_clock::time_point GetSelectedTime() const; + void SetMapCount(std::size_t mapCount); public slots: diff --git a/scwx-qt/source/scwx/qt/manager/update_manager.cpp b/scwx-qt/source/scwx/qt/manager/update_manager.cpp index d21068cb..05a9c0d1 100644 --- a/scwx-qt/source/scwx/qt/manager/update_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/update_manager.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -29,8 +30,7 @@ public: ~Impl() {} - static std::string GetVersionString(const std::string& releaseName); - static boost::json::value ParseResponseText(const std::string& s); + static std::string GetVersionString(const std::string& releaseName); size_t PopulateReleases(); size_t AddReleases(const boost::json::value& json); @@ -70,28 +70,6 @@ UpdateManager::Impl::GetVersionString(const std::string& releaseName) return versionString; } -boost::json::value UpdateManager::Impl::ParseResponseText(const std::string& s) -{ - boost::json::stream_parser p; - boost::system::error_code ec; - - p.write(s, ec); - if (ec) - { - logger_->warn("{}", ec.message()); - return nullptr; - } - - p.finish(ec); - if (ec) - { - logger_->warn("{}", ec.message()); - return nullptr; - } - - return p.release(); -} - bool UpdateManager::CheckForUpdates(const std::string& currentVersion) { std::unique_lock lock(p->updateMutex_); @@ -148,7 +126,7 @@ size_t UpdateManager::Impl::PopulateReleases() // Successful REST API query if (r.status_code == 200) { - boost::json::value json = Impl::ParseResponseText(r.text); + const boost::json::value json = util::json::ReadJsonString(r.text); if (json == nullptr) { logger_->warn("Response not JSON: {}", r.header["content-type"]); diff --git a/scwx-qt/source/scwx/qt/map/alert_layer.cpp b/scwx-qt/source/scwx/qt/map/alert_layer.cpp index 7c5c9db2..495be87c 100644 --- a/scwx-qt/source/scwx/qt/map/alert_layer.cpp +++ b/scwx-qt/source/scwx/qt/map/alert_layer.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -19,12 +20,9 @@ #include #include #include +#include -namespace scwx -{ -namespace qt -{ -namespace map +namespace scwx::qt::map { static const std::string logPrefix_ = "scwx::qt::map::alert_layer"; @@ -46,6 +44,8 @@ static bool IsAlertActive(const std::shared_ptr& segment); class AlertLayerHandler : public QObject { Q_OBJECT + Q_DISABLE_COPY_MOVE(AlertLayerHandler) + public: struct SegmentRecord { @@ -57,10 +57,10 @@ public: SegmentRecord( const std::shared_ptr& segment, - const types::TextEventKey& key, + types::TextEventKey key, const std::shared_ptr& message) : segment_ {segment}, - key_ {key}, + key_ {std::move(key)}, message_ {message}, segmentBegin_ {segment->event_begin()}, segmentEnd_ {segment->event_end()} @@ -73,8 +73,11 @@ public: connect(textEventManager_.get(), &manager::TextEventManager::AlertUpdated, this, - [this](const types::TextEventKey& key, std::size_t messageIndex) - { HandleAlert(key, messageIndex); }); + &AlertLayerHandler::HandleAlert); + connect(textEventManager_.get(), + &manager::TextEventManager::AlertsRemoved, + this, + &AlertLayerHandler::HandleAlertsRemoved); } ~AlertLayerHandler() { @@ -95,7 +98,13 @@ public: types::TextEventHash> segmentsByKey_ {}; - void HandleAlert(const types::TextEventKey& key, size_t messageIndex); + void HandleAlert(const types::TextEventKey& key, + size_t messageIndex, + boost::uuids::uuid uuid); + void HandleAlertsRemoved( + const std::unordered_set>& + keys); static AlertLayerHandler& Instance(); @@ -108,6 +117,7 @@ signals: void AlertAdded(const std::shared_ptr& segmentRecord, awips::Phenomenon phenomenon); void AlertUpdated(const std::shared_ptr& segmentRecord); + void AlertsRemoved(awips::Phenomenon phenomenon); void AlertsUpdated(awips::Phenomenon phenomenon, bool alertActive); }; @@ -151,6 +161,11 @@ public: std::unique_lock lock(linesMutex_); }; + Impl(const Impl&) = delete; + Impl& operator=(const Impl&) = delete; + Impl(const Impl&&) = delete; + Impl& operator=(const Impl&&) = delete; + void AddAlert( const std::shared_ptr& segmentRecord); void UpdateAlert( @@ -172,20 +187,22 @@ public: std::shared_ptr& di, const common::Coordinate& p1, const common::Coordinate& p2, - boost::gil::rgba32f_pixel_t color, + const boost::gil::rgba32f_pixel_t& color, float width, std::chrono::system_clock::time_point startTime, std::chrono::system_clock::time_point endTime, bool enableHover); void AddLines(std::shared_ptr& geoLines, const std::vector& coordinates, - boost::gil::rgba32f_pixel_t color, + const boost::gil::rgba32f_pixel_t& color, float width, std::chrono::system_clock::time_point startTime, std::chrono::system_clock::time_point endTime, bool enableHover, boost::container::stable_vector< std::shared_ptr>& drawItems); + void PopulateLines(bool alertActive); + void RepopulateLines(); void UpdateLines(); static LineData CreateLineData(const settings::LineSettings& lineSettings); @@ -201,6 +218,7 @@ public: const awips::ibw::ImpactBasedWarningInfo& ibw_; std::unique_ptr receiver_ {std::make_unique()}; + std::mutex receiverMutex_ {}; std::unordered_map> geoLines_; @@ -227,8 +245,8 @@ public: std::vector connections_ {}; }; -AlertLayer::AlertLayer(std::shared_ptr context, - awips::Phenomenon phenomenon) : +AlertLayer::AlertLayer(const std::shared_ptr& context, + awips::Phenomenon phenomenon) : DrawLayer( context, fmt::format("AlertLayer {}", awips::GetPhenomenonText(phenomenon))), @@ -264,28 +282,15 @@ void AlertLayer::Initialize() auto& alertLayerHandler = AlertLayerHandler::Instance(); + p->selectedTime_ = manager::TimelineManager::Instance()->GetSelectedTime(); + // Take a shared lock to prevent handling additional alerts while populating // initial lists std::shared_lock lock {alertLayerHandler.alertMutex_}; for (auto alertActive : {false, true}) { - auto& geoLines = p->geoLines_.at(alertActive); - - geoLines->StartLines(); - - // Populate initial segments - auto segmentsIt = - alertLayerHandler.segmentsByType_.find({p->phenomenon_, alertActive}); - if (segmentsIt != alertLayerHandler.segmentsByType_.cend()) - { - for (auto& segment : segmentsIt->second) - { - p->AddAlert(segment); - } - } - - geoLines->FinishLines(); + p->PopulateLines(alertActive); } p->ConnectAlertHandlerSignals(); @@ -322,7 +327,8 @@ bool IsAlertActive(const std::shared_ptr& segment) } void AlertLayerHandler::HandleAlert(const types::TextEventKey& key, - size_t messageIndex) + size_t messageIndex, + boost::uuids::uuid uuid) { logger_->trace("HandleAlert: {}", key.ToString()); @@ -330,7 +336,28 @@ void AlertLayerHandler::HandleAlert(const types::TextEventKey& key, AlertTypeHash>> alertsUpdated {}; - auto message = textEventManager_->message_list(key).at(messageIndex); + const auto& messageList = textEventManager_->message_list(key); + + // Find message by UUID instead of index, as the message index could have + // changed between the signal being emitted and the handler being called + auto messageIt = std::find_if(messageList.cbegin(), + messageList.cend(), + [&uuid](const auto& message) + { return uuid == message->uuid(); }); + + if (messageIt == messageList.cend()) + { + logger_->warn( + "Could not find alert uuid: {} ({})", key.ToString(), messageIndex); + return; + } + + auto& message = *messageIt; + auto nextMessageIt = std::next(messageIt); + + // Store the current message index + messageIndex = + static_cast(std::distance(messageList.cbegin(), messageIt)); // Determine start time for first segment std::chrono::system_clock::time_point segmentBegin {}; @@ -339,14 +366,31 @@ void AlertLayerHandler::HandleAlert(const types::TextEventKey& key, segmentBegin = message->segment(0)->event_begin(); } + // Determine the start time for the first segment of the next message + std::optional nextMessageBegin {}; + if (nextMessageIt != messageList.cend()) + { + nextMessageBegin = + (*nextMessageIt) + ->wmo_header() + ->GetDateTime((*nextMessageIt)->segment(0)->event_begin()); + } + // Take a unique mutex before modifying segments std::unique_lock lock {alertMutex_}; - // Update any existing segments with new end time + // Update any existing earlier segments with new end time auto& segmentsForKey = segmentsByKey_[key]; for (auto& segmentRecord : segmentsForKey) { - if (segmentRecord->segmentEnd_ > segmentBegin) + // Determine if the segment is earlier than the current message + auto it = std::find( + messageList.cbegin(), messageList.cend(), segmentRecord->message_); + auto segmentIndex = + static_cast(std::distance(messageList.cbegin(), it)); + + if (segmentIndex < messageIndex && + segmentRecord->segmentEnd_ > segmentBegin) { segmentRecord->segmentEnd_ = segmentBegin; @@ -373,6 +417,14 @@ void AlertLayerHandler::HandleAlert(const types::TextEventKey& key, std::shared_ptr segmentRecord = std::make_shared(segment, key, message); + // Update segment end time to be no later than the begin time of the next + // message (if present) + if (nextMessageBegin.has_value() && + segmentRecord->segmentEnd_ > nextMessageBegin) + { + segmentRecord->segmentEnd_ = nextMessageBegin.value(); + } + segmentsForKey.push_back(segmentRecord); segmentsForType.push_back(segmentRecord); @@ -391,6 +443,63 @@ void AlertLayerHandler::HandleAlert(const types::TextEventKey& key, } } +void AlertLayerHandler::HandleAlertsRemoved( + const std::unordered_set>& keys) +{ + logger_->trace("HandleAlertsRemoved: {} keys", keys.size()); + + std::set alertsRemoved {}; + + // Take a unique lock before modifying segments + std::unique_lock lock {alertMutex_}; + + for (const auto& key : keys) + { + // Remove segments associated with the key + auto segmentsIt = segmentsByKey_.find(key); + if (segmentsIt != segmentsByKey_.end()) + { + for (const auto& segmentRecord : segmentsIt->second) + { + auto& segment = segmentRecord->segment_; + const bool alertActive = IsAlertActive(segment); + + // Remove from segmentsByType_ + auto typeIt = segmentsByType_.find({key.phenomenon_, alertActive}); + if (typeIt != segmentsByType_.end()) + { + auto& segmentsForType = typeIt->second; + segmentsForType.erase(std::remove(segmentsForType.begin(), + segmentsForType.end(), + segmentRecord), + segmentsForType.end()); + + // If no segments remain for this type, erase the entry + if (segmentsForType.empty()) + { + segmentsByType_.erase(typeIt); + } + } + + alertsRemoved.emplace(key.phenomenon_); + } + + // Remove the key from segmentsByKey_ + segmentsByKey_.erase(segmentsIt); + } + } + + // Release the lock after completing segment updates + lock.unlock(); + + // Emit signal to notify that alerts have been removed + for (auto& alert : alertsRemoved) + { + Q_EMIT AlertsRemoved(alert); + } +} + void AlertLayer::Impl::ConnectAlertHandlerSignals() { auto& alertLayerHandler = AlertLayerHandler::Instance(); @@ -405,6 +514,9 @@ void AlertLayer::Impl::ConnectAlertHandlerSignals() { if (phenomenon == phenomenon_) { + // Only process one signal at a time + const std::unique_lock lock {receiverMutex_}; + AddAlert(segmentRecord); } }); @@ -417,9 +529,27 @@ void AlertLayer::Impl::ConnectAlertHandlerSignals() { if (segmentRecord->key_.phenomenon_ == phenomenon_) { + // Only process one signal at a time + const std::unique_lock lock {receiverMutex_}; + UpdateAlert(segmentRecord); } }); + QObject::connect(&alertLayerHandler, + &AlertLayerHandler::AlertsRemoved, + receiver_.get(), + [this](awips::Phenomenon phenomenon) + { + if (phenomenon == phenomenon_) + { + // Only process one signal at a time + const std::unique_lock lock {receiverMutex_}; + + // Re-populate the lines if multiple alerts were + // removed + RepopulateLines(); + } + }); } void AlertLayer::Impl::ConnectSignals() @@ -500,9 +630,9 @@ void AlertLayer::Impl::AddAlert( // If draw items were added if (drawItems.second) { - const float borderWidth = lineData.borderWidth_; - const float highlightWidth = lineData.highlightWidth_; - const float lineWidth = lineData.lineWidth_; + const auto borderWidth = static_cast(lineData.borderWidth_); + const auto highlightWidth = static_cast(lineData.highlightWidth_); + const auto lineWidth = static_cast(lineData.lineWidth_); const float totalHighlightWidth = lineWidth + (highlightWidth * 2.0f); const float totalBorderWidth = totalHighlightWidth + (borderWidth * 2.0f); @@ -547,6 +677,8 @@ void AlertLayer::Impl::AddAlert( lineHover, drawItems.first->second); } + + Q_EMIT self_->NeedsRendering(); } void AlertLayer::Impl::UpdateAlert( @@ -570,12 +702,14 @@ void AlertLayer::Impl::UpdateAlert( geoLines->SetLineEndTime(line, segmentRecord->segmentEnd_); } } + + Q_EMIT self_->NeedsRendering(); } void AlertLayer::Impl::AddLines( std::shared_ptr& geoLines, const std::vector& coordinates, - boost::gil::rgba32f_pixel_t color, + const boost::gil::rgba32f_pixel_t& color, float width, std::chrono::system_clock::time_point startTime, std::chrono::system_clock::time_point endTime, @@ -615,14 +749,17 @@ void AlertLayer::Impl::AddLine(std::shared_ptr& geoLines, std::shared_ptr& di, const common::Coordinate& p1, const common::Coordinate& p2, - boost::gil::rgba32f_pixel_t color, + const boost::gil::rgba32f_pixel_t& color, float width, std::chrono::system_clock::time_point startTime, std::chrono::system_clock::time_point endTime, bool enableHover) { - geoLines->SetLineLocation( - di, p1.latitude_, p1.longitude_, p2.latitude_, p2.longitude_); + geoLines->SetLineLocation(di, + static_cast(p1.latitude_), + static_cast(p1.longitude_), + static_cast(p2.latitude_), + static_cast(p2.longitude_)); geoLines->SetLineModulate(di, color); geoLines->SetLineWidth(di, width); geoLines->SetLineStartTime(di, startTime); @@ -647,6 +784,46 @@ void AlertLayer::Impl::AddLine(std::shared_ptr& geoLines, } } +void AlertLayer::Impl::PopulateLines(bool alertActive) +{ + auto& alertLayerHandler = AlertLayerHandler::Instance(); + auto& geoLines = geoLines_.at(alertActive); + + geoLines->StartLines(); + + // Populate initial segments + auto segmentsIt = + alertLayerHandler.segmentsByType_.find({phenomenon_, alertActive}); + if (segmentsIt != alertLayerHandler.segmentsByType_.cend()) + { + for (auto& segment : segmentsIt->second) + { + AddAlert(segment); + } + } + + geoLines->FinishLines(); +} + +void AlertLayer::Impl::RepopulateLines() +{ + auto& alertLayerHandler = AlertLayerHandler::Instance(); + + // Take a shared lock to prevent handling additional alerts while populating + // initial lists + const std::shared_lock alertLock {alertLayerHandler.alertMutex_}; + + linesBySegment_.clear(); + segmentsByLine_.clear(); + + for (auto alertActive : {false, true}) + { + PopulateLines(alertActive); + } + + Q_EMIT self_->NeedsRendering(); +} + void AlertLayer::Impl::UpdateLines() { std::unique_lock lock {linesMutex_}; @@ -660,9 +837,9 @@ void AlertLayer::Impl::UpdateLines() auto& lineData = GetLineData(segment, alertActive); auto& geoLines = geoLines_.at(alertActive); - const float borderWidth = lineData.borderWidth_; - const float highlightWidth = lineData.highlightWidth_; - const float lineWidth = lineData.lineWidth_; + const auto borderWidth = static_cast(lineData.borderWidth_); + const auto highlightWidth = static_cast(lineData.highlightWidth_); + const auto lineWidth = static_cast(lineData.lineWidth_); const float totalHighlightWidth = lineWidth + (highlightWidth * 2.0f); const float totalBorderWidth = totalHighlightWidth + (borderWidth * 2.0f); @@ -837,8 +1014,6 @@ size_t AlertTypeHash>::operator()( return seed; } -} // namespace map -} // namespace qt -} // namespace scwx +} // namespace scwx::qt::map #include "alert_layer.moc" diff --git a/scwx-qt/source/scwx/qt/map/alert_layer.hpp b/scwx-qt/source/scwx/qt/map/alert_layer.hpp index d51391e3..60905680 100644 --- a/scwx-qt/source/scwx/qt/map/alert_layer.hpp +++ b/scwx-qt/source/scwx/qt/map/alert_layer.hpp @@ -22,8 +22,8 @@ class AlertLayer : public DrawLayer Q_DISABLE_COPY_MOVE(AlertLayer) public: - explicit AlertLayer(std::shared_ptr context, - scwx::awips::Phenomenon phenomenon); + explicit AlertLayer(const std::shared_ptr& context, + scwx::awips::Phenomenon phenomenon); ~AlertLayer(); void Initialize() override final; diff --git a/scwx-qt/source/scwx/qt/map/placefile_layer.cpp b/scwx-qt/source/scwx/qt/map/placefile_layer.cpp index df9828eb..dcead2a1 100644 --- a/scwx-qt/source/scwx/qt/map/placefile_layer.cpp +++ b/scwx-qt/source/scwx/qt/map/placefile_layer.cpp @@ -122,6 +122,8 @@ void PlacefileLayer::Initialize() logger_->debug("Initialize()"); DrawLayer::Initialize(); + + p->selectedTime_ = manager::TimelineManager::Instance()->GetSelectedTime(); } void PlacefileLayer::Render( diff --git a/scwx-qt/source/scwx/qt/model/alert_model.cpp b/scwx-qt/source/scwx/qt/model/alert_model.cpp index fed7cc17..20af05f8 100644 --- a/scwx-qt/source/scwx/qt/model/alert_model.cpp +++ b/scwx-qt/source/scwx/qt/model/alert_model.cpp @@ -10,16 +10,10 @@ #include #include -#include - #include #include -namespace scwx -{ -namespace qt -{ -namespace model +namespace scwx::qt::model { static const std::string logPrefix_ = "scwx::qt::model::alert_model"; @@ -329,16 +323,45 @@ AlertModel::headerData(int section, Qt::Orientation orientation, int role) const } void AlertModel::HandleAlert(const types::TextEventKey& alertKey, - size_t messageIndex) + std::size_t messageIndex, + boost::uuids::uuid uuid) { logger_->trace("Handle alert: {}", alertKey.ToString()); double distanceInMeters; + const auto& alertMessages = p->textEventManager_->message_list(alertKey); + + // Find message by UUID instead of index, as the message index could have + // changed between the signal being emitted and the handler being called + auto messageIt = std::find_if(alertMessages.cbegin(), + alertMessages.cend(), + [&uuid](const auto& message) + { return uuid == message->uuid(); }); + + if (messageIt == alertMessages.cend()) + { + logger_->warn("Could not find alert uuid: {} ({})", + alertKey.ToString(), + messageIndex); + return; + } + + auto& message = *messageIt; + + // Store the current message index + messageIndex = static_cast( + std::distance(alertMessages.cbegin(), messageIt)); + + // Skip alert if this is not the most recent message + if (messageIndex + 1 < alertMessages.size()) + { + return; + } + // Get the most recent segment for the event - auto alertMessages = p->textEventManager_->message_list(alertKey); - std::shared_ptr alertSegment = - alertMessages[messageIndex]->segments().back(); + const std::shared_ptr alertSegment = + message->segments().back(); p->observedMap_.insert_or_assign(alertKey, alertSegment->observed_); p->threatCategoryMap_.insert_or_assign(alertKey, @@ -386,6 +409,36 @@ void AlertModel::HandleAlert(const types::TextEventKey& alertKey, } } +void AlertModel::HandleAlertsRemoved( + const std::unordered_set>& + alertKeys) +{ + logger_->trace("Handle alerts removed"); + + for (const auto& alertKey : alertKeys) + { + // Remove from the list of text event keys + auto it = std::find( + p->textEventKeys_.begin(), p->textEventKeys_.end(), alertKey); + if (it != p->textEventKeys_.end()) + { + const int row = + static_cast(std::distance(p->textEventKeys_.begin(), it)); + beginRemoveRows(QModelIndex(), row, row); + p->textEventKeys_.erase(it); + endRemoveRows(); + } + + // Remove from internal maps + p->observedMap_.erase(alertKey); + p->threatCategoryMap_.erase(alertKey); + p->tornadoPossibleMap_.erase(alertKey); + p->centroidMap_.erase(alertKey); + p->distanceMap_.erase(alertKey); + } +} + void AlertModel::HandleMapUpdate(double latitude, double longitude) { logger_->trace("Handle map update: {}, {}", latitude, longitude); @@ -488,8 +541,8 @@ std::string AlertModelImpl::GetCounties(const types::TextEventKey& key) } else { - logger_->warn("GetCounties(): No message associated with key: {}", - key.ToString()); + logger_->trace("GetCounties(): No message associated with key: {}", + key.ToString()); return {}; } } @@ -507,8 +560,8 @@ std::string AlertModelImpl::GetState(const types::TextEventKey& key) } else { - logger_->warn("GetState(): No message associated with key: {}", - key.ToString()); + logger_->trace("GetState(): No message associated with key: {}", + key.ToString()); return {}; } } @@ -525,8 +578,8 @@ AlertModelImpl::GetStartTime(const types::TextEventKey& key) } else { - logger_->warn("GetStartTime(): No message associated with key: {}", - key.ToString()); + logger_->trace("GetStartTime(): No message associated with key: {}", + key.ToString()); return {}; } } @@ -550,8 +603,8 @@ AlertModelImpl::GetEndTime(const types::TextEventKey& key) } else { - logger_->warn("GetEndTime(): No message associated with key: {}", - key.ToString()); + logger_->trace("GetEndTime(): No message associated with key: {}", + key.ToString()); return {}; } } @@ -561,6 +614,4 @@ std::string AlertModelImpl::GetEndTimeString(const types::TextEventKey& key) return scwx::util::TimeString(GetEndTime(key)); } -} // namespace model -} // namespace qt -} // namespace scwx +} // namespace scwx::qt::model diff --git a/scwx-qt/source/scwx/qt/model/alert_model.hpp b/scwx-qt/source/scwx/qt/model/alert_model.hpp index df6d561e..443ca9bb 100644 --- a/scwx-qt/source/scwx/qt/model/alert_model.hpp +++ b/scwx-qt/source/scwx/qt/model/alert_model.hpp @@ -4,7 +4,9 @@ #include #include +#include +#include #include namespace scwx @@ -50,7 +52,13 @@ public: int role = Qt::DisplayRole) const override; public slots: - void HandleAlert(const types::TextEventKey& alertKey, size_t messageIndex); + void HandleAlert(const types::TextEventKey& alertKey, + std::size_t messageIndex, + boost::uuids::uuid uuid); + void HandleAlertsRemoved( + const std::unordered_set>& + alertKeys); void HandleMapUpdate(double latitude, double longitude); private: diff --git a/scwx-qt/source/scwx/qt/model/alert_proxy_model.cpp b/scwx-qt/source/scwx/qt/model/alert_proxy_model.cpp index a2afee55..4dfeca41 100644 --- a/scwx-qt/source/scwx/qt/model/alert_proxy_model.cpp +++ b/scwx-qt/source/scwx/qt/model/alert_proxy_model.cpp @@ -9,21 +9,22 @@ #include -namespace scwx -{ -namespace qt -{ -namespace model +namespace scwx::qt::model { static const std::string logPrefix_ = "scwx::qt::model::alert_proxy_model"; static const auto logger_ = scwx::util::Logger::Create(logPrefix_); -class AlertProxyModelImpl +class AlertProxyModel::Impl { public: - explicit AlertProxyModelImpl(AlertProxyModel* self); - ~AlertProxyModelImpl(); + explicit Impl(AlertProxyModel* self); + ~Impl(); + + Impl(const Impl&) = delete; + Impl& operator=(const Impl&) = delete; + Impl(const Impl&&) = delete; + Impl& operator=(const Impl&&) = delete; void UpdateAlerts(); @@ -36,8 +37,7 @@ public: }; AlertProxyModel::AlertProxyModel(QObject* parent) : - QSortFilterProxyModel(parent), - p(std::make_unique(this)) + QSortFilterProxyModel(parent), p(std::make_unique(this)) { } AlertProxyModel::~AlertProxyModel() = default; @@ -77,7 +77,7 @@ bool AlertProxyModel::filterAcceptsRow(int sourceRow, QSortFilterProxyModel::filterAcceptsRow(sourceRow, sourceParent); } -AlertProxyModelImpl::AlertProxyModelImpl(AlertProxyModel* self) : +AlertProxyModel::Impl::Impl(AlertProxyModel* self) : self_ {self}, alertActiveFilterEnabled_ {false}, alertUpdateTimer_ {scwx::util::io_context()} @@ -86,26 +86,37 @@ AlertProxyModelImpl::AlertProxyModelImpl(AlertProxyModel* self) : UpdateAlerts(); } -AlertProxyModelImpl::~AlertProxyModelImpl() +AlertProxyModel::Impl::~Impl() { - std::unique_lock lock(alertMutex_); - alertUpdateTimer_.cancel(); + try + { + const std::unique_lock lock(alertMutex_); + alertUpdateTimer_.cancel(); + } + catch (const std::exception& ex) + { + logger_->error(ex.what()); + } } -void AlertProxyModelImpl::UpdateAlerts() +void AlertProxyModel::Impl::UpdateAlerts() { logger_->trace("UpdateAlerts"); // Take a unique lock before modifying feature lists - std::unique_lock lock(alertMutex_); + const std::unique_lock lock(alertMutex_); // Re-evaluate for expired alerts if (alertActiveFilterEnabled_) { - self_->invalidateRowsFilter(); + QMetaObject::invokeMethod(self_, + static_cast( + &QSortFilterProxyModel::invalidate)); } using namespace std::chrono; + + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers): Readability alertUpdateTimer_.expires_after(15s); alertUpdateTimer_.async_wait( [this](const boost::system::error_code& e) @@ -132,6 +143,4 @@ void AlertProxyModelImpl::UpdateAlerts() }); } -} // namespace model -} // namespace qt -} // namespace scwx +} // namespace scwx::qt::model diff --git a/scwx-qt/source/scwx/qt/model/alert_proxy_model.hpp b/scwx-qt/source/scwx/qt/model/alert_proxy_model.hpp index ee8b81c1..1ee6a138 100644 --- a/scwx-qt/source/scwx/qt/model/alert_proxy_model.hpp +++ b/scwx-qt/source/scwx/qt/model/alert_proxy_model.hpp @@ -4,11 +4,7 @@ #include -namespace scwx -{ -namespace qt -{ -namespace model +namespace scwx::qt::model { class AlertProxyModelImpl; @@ -16,7 +12,7 @@ class AlertProxyModelImpl; class AlertProxyModel : public QSortFilterProxyModel { private: - Q_DISABLE_COPY(AlertProxyModel) + Q_DISABLE_COPY_MOVE(AlertProxyModel) public: explicit AlertProxyModel(QObject* parent = nullptr); @@ -24,15 +20,13 @@ public: void SetAlertActiveFilter(bool enabled); - bool filterAcceptsRow(int sourceRow, - const QModelIndex& sourceParent) const override; + [[nodiscard]] bool + filterAcceptsRow(int sourceRow, + const QModelIndex& sourceParent) const override; private: - std::unique_ptr p; - - friend class AlertProxyModelImpl; + class Impl; + std::unique_ptr p; }; -} // namespace model -} // namespace qt -} // namespace scwx +} // namespace scwx::qt::model diff --git a/scwx-qt/source/scwx/qt/model/layer_model.cpp b/scwx-qt/source/scwx/qt/model/layer_model.cpp index 2f1b8a9d..6be8eb9d 100644 --- a/scwx-qt/source/scwx/qt/model/layer_model.cpp +++ b/scwx-qt/source/scwx/qt/model/layer_model.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include #include diff --git a/scwx-qt/source/scwx/qt/model/radar_site_model.cpp b/scwx-qt/source/scwx/qt/model/radar_site_model.cpp index e1a593d7..f131a1cd 100644 --- a/scwx-qt/source/scwx/qt/model/radar_site_model.cpp +++ b/scwx-qt/source/scwx/qt/model/radar_site_model.cpp @@ -4,8 +4,8 @@ #include #include #include -#include #include +#include #include #include @@ -117,7 +117,7 @@ void RadarSiteModelImpl::ReadPresets() // Determine if presets exists if (std::filesystem::exists(presetsPath_)) { - presetsJson = util::json::ReadJsonFile(presetsPath_); + presetsJson = scwx::util::json::ReadJsonFile(presetsPath_); } // If presets was successfully read @@ -160,7 +160,7 @@ void RadarSiteModelImpl::WritePresets() logger_->info("Saving presets"); auto presetsJson = boost::json::value_from(presets_); - util::json::WriteJsonFile(presetsPath_, presetsJson); + scwx::util::json::WriteJsonFile(presetsPath_, presetsJson); } int RadarSiteModel::rowCount(const QModelIndex& parent) const diff --git a/scwx-qt/source/scwx/qt/types/text_event_key.cpp b/scwx-qt/source/scwx/qt/types/text_event_key.cpp index bebf6f63..be5d0443 100644 --- a/scwx-qt/source/scwx/qt/types/text_event_key.cpp +++ b/scwx-qt/source/scwx/qt/types/text_event_key.cpp @@ -14,26 +14,29 @@ static const std::string logPrefix_ = "scwx::qt::types::text_event_key"; std::string TextEventKey::ToFullString() const { - return fmt::format("{} {} {} {:04}", + return fmt::format("{} {} {} {:04} ({:04})", officeId_, awips::GetPhenomenonText(phenomenon_), awips::GetSignificanceText(significance_), - etn_); + etn_, + static_cast(year_)); } std::string TextEventKey::ToString() const { - return fmt::format("{}.{}.{}.{:04}", + return fmt::format("{}.{}.{}.{:04}.{:04}", officeId_, awips::GetPhenomenonCode(phenomenon_), awips::GetSignificanceCode(significance_), - etn_); + etn_, + static_cast(year_)); } bool TextEventKey::operator==(const TextEventKey& o) const { return (officeId_ == o.officeId_ && phenomenon_ == o.phenomenon_ && - significance_ == o.significance_ && etn_ == o.etn_); + significance_ == o.significance_ && etn_ == o.etn_ && + year_ == o.year_); } size_t TextEventHash::operator()(const TextEventKey& x) const @@ -43,6 +46,7 @@ size_t TextEventHash::operator()(const TextEventKey& x) const boost::hash_combine(seed, x.phenomenon_); boost::hash_combine(seed, x.significance_); boost::hash_combine(seed, x.etn_); + boost::hash_combine(seed, static_cast(x.year_)); return seed; } diff --git a/scwx-qt/source/scwx/qt/types/text_event_key.hpp b/scwx-qt/source/scwx/qt/types/text_event_key.hpp index f962bcdf..15eec31c 100644 --- a/scwx-qt/source/scwx/qt/types/text_event_key.hpp +++ b/scwx-qt/source/scwx/qt/types/text_event_key.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include namespace scwx { @@ -12,12 +13,34 @@ namespace types struct TextEventKey { TextEventKey() : TextEventKey(awips::PVtec {}) {} - TextEventKey(const awips::PVtec& pvtec) : + TextEventKey(const awips::PVtec& pvtec, std::chrono::year yearHint = {}) : officeId_ {pvtec.office_id()}, phenomenon_ {pvtec.phenomenon()}, significance_ {pvtec.significance()}, etn_ {pvtec.event_tracking_number()} { + using namespace std::chrono_literals; + + static constexpr std::chrono::year kMinYear_ = 1970y; + + std::chrono::year_month_day ymd = + std::chrono::floor(pvtec.event_begin()); + if (ymd.year() > kMinYear_) + { + // Prefer the year from the event begin + year_ = ymd.year(); + } + else if (yearHint > kMinYear_) + { + // Otherwise, use the year hint + year_ = yearHint; + } + else + { + // If there was no year hint, use the event end + ymd = std::chrono::floor(pvtec.event_end()); + year_ = ymd.year(); + } } std::string ToFullString() const; @@ -27,7 +50,8 @@ struct TextEventKey std::string officeId_; awips::Phenomenon phenomenon_; awips::Significance significance_; - int16_t etn_; + std::int16_t etn_; + std::chrono::year year_ {}; }; template diff --git a/scwx-qt/source/scwx/qt/ui/alert_dock_widget.cpp b/scwx-qt/source/scwx/qt/ui/alert_dock_widget.cpp index 61fd160a..5e22071a 100644 --- a/scwx-qt/source/scwx/qt/ui/alert_dock_widget.cpp +++ b/scwx-qt/source/scwx/qt/ui/alert_dock_widget.cpp @@ -131,6 +131,11 @@ void AlertDockWidgetImpl::ConnectSignals() &QAction::toggled, proxyModel_.get(), &model::AlertProxyModel::SetAlertActiveFilter); + connect(textEventManager_.get(), + &manager::TextEventManager::AlertsRemoved, + alertModel_.get(), + &model::AlertModel::HandleAlertsRemoved, + Qt::QueuedConnection); connect(textEventManager_.get(), &manager::TextEventManager::AlertUpdated, alertModel_.get(), diff --git a/scwx-qt/source/scwx/qt/util/json.cpp b/scwx-qt/source/scwx/qt/util/json.cpp index 7bf0d23a..d508a224 100644 --- a/scwx-qt/source/scwx/qt/util/json.cpp +++ b/scwx-qt/source/scwx/qt/util/json.cpp @@ -1,41 +1,19 @@ #include +#include #include -#include - -#include -#include #include #include -namespace scwx -{ -namespace qt -{ -namespace util -{ -namespace json +namespace scwx::qt::util::json { static const std::string logPrefix_ = "scwx::qt::util::json"; static const auto logger_ = scwx::util::Logger::Create(logPrefix_); -/* Adapted from: - * https://www.boost.org/doc/libs/1_77_0/libs/json/doc/html/json/examples.html#json.examples.pretty - * - * Copyright (c) 2019, 2020 Vinnie Falco - * Copyright (c) 2020 Krystian Stasiowski - * Distributed under the Boost Software License, Version 1.0. (See - * http://www.boost.org/LICENSE_1_0.txt) - */ -static void PrettyPrintJson(std::ostream& os, - boost::json::value const& jv, - std::string* indent = nullptr); - static boost::json::value ReadJsonFile(QFile& file); -static boost::json::value ReadJsonStream(std::istream& is); -boost::json::value ReadJsonFile(const std::string& path) +boost::json::value ReadJsonQFile(const std::string& path) { boost::json::value json; @@ -46,8 +24,7 @@ boost::json::value ReadJsonFile(const std::string& path) } else { - std::ifstream ifs {path}; - json = ReadJsonStream(ifs); + json = ::scwx::util::json::ReadJsonFile(path); } return json; @@ -65,7 +42,7 @@ static boost::json::value ReadJsonFile(QFile& file) std::string jsonSource = jsonStream.readAll().toStdString(); std::istringstream is {jsonSource}; - json = ReadJsonStream(is); + json = ::scwx::util::json::ReadJsonStream(is); file.close(); } @@ -78,147 +55,4 @@ static boost::json::value ReadJsonFile(QFile& file) return json; } -static boost::json::value ReadJsonStream(std::istream& is) -{ - std::string line; - - boost::json::stream_parser p; - boost::system::error_code ec; - - while (std::getline(is, line)) - { - p.write(line, ec); - if (ec) - { - logger_->warn("{}", ec.message()); - return nullptr; - } - } - - p.finish(ec); - if (ec) - { - logger_->warn("{}", ec.message()); - return nullptr; - } - - return p.release(); -} - -void WriteJsonFile(const std::string& path, - const boost::json::value& json, - bool prettyPrint) -{ - std::ofstream ofs {path}; - - if (!ofs.is_open()) - { - logger_->warn("Cannot write JSON file: \"{}\"", path); - } - else - { - if (prettyPrint) - { - PrettyPrintJson(ofs, json); - } - else - { - ofs << json; - } - ofs.close(); - } -} - -static void PrettyPrintJson(std::ostream& os, - boost::json::value const& jv, - std::string* indent) -{ - std::string indent_; - if (!indent) - indent = &indent_; - switch (jv.kind()) - { - case boost::json::kind::object: - { - os << "{\n"; - indent->append(4, ' '); - auto const& obj = jv.get_object(); - if (!obj.empty()) - { - auto it = obj.begin(); - for (;;) - { - os << *indent << boost::json::serialize(it->key()) << " : "; - PrettyPrintJson(os, it->value(), indent); - if (++it == obj.end()) - break; - os << ",\n"; - } - } - os << "\n"; - indent->resize(indent->size() - 4); - os << *indent << "}"; - break; - } - - case boost::json::kind::array: - { - os << "[\n"; - indent->append(4, ' '); - auto const& arr = jv.get_array(); - if (!arr.empty()) - { - auto it = arr.begin(); - for (;;) - { - os << *indent; - PrettyPrintJson(os, *it, indent); - if (++it == arr.end()) - break; - os << ",\n"; - } - } - os << "\n"; - indent->resize(indent->size() - 4); - os << *indent << "]"; - break; - } - - case boost::json::kind::string: - { - os << boost::json::serialize(jv.get_string()); - break; - } - - case boost::json::kind::uint64: - os << jv.get_uint64(); - break; - - case boost::json::kind::int64: - os << jv.get_int64(); - break; - - case boost::json::kind::double_: - os << jv.get_double(); - break; - - case boost::json::kind::bool_: - if (jv.get_bool()) - os << "true"; - else - os << "false"; - break; - - case boost::json::kind::null: - os << "null"; - break; - } - - if (indent->empty()) - os << "\n"; -} - -} // namespace json -} // namespace util -} // namespace qt -} // namespace scwx +} // namespace scwx::qt::util::json diff --git a/scwx-qt/source/scwx/qt/util/json.hpp b/scwx-qt/source/scwx/qt/util/json.hpp index bbf497f4..9dd09810 100644 --- a/scwx-qt/source/scwx/qt/util/json.hpp +++ b/scwx-qt/source/scwx/qt/util/json.hpp @@ -1,7 +1,5 @@ #pragma once -#include - #include namespace scwx @@ -13,10 +11,7 @@ namespace util namespace json { -boost::json::value ReadJsonFile(const std::string& path); -void WriteJsonFile(const std::string& path, - const boost::json::value& json, - bool prettyPrint = true); +boost::json::value ReadJsonQFile(const std::string& path); } // namespace json } // namespace util diff --git a/test/.clang-tidy b/test/.clang-tidy new file mode 100644 index 00000000..d5079a03 --- /dev/null +++ b/test/.clang-tidy @@ -0,0 +1,16 @@ +Checks: + - '-*' + - 'bugprone-*' + - 'clang-analyzer-*' + - 'cppcoreguidelines-*' + - 'misc-*' + - 'modernize-*' + - 'performance-*' + - '-bugprone-easily-swappable-parameters' + - '-cppcoreguidelines-avoid-magic-numbers' + - '-cppcoreguidelines-pro-type-reinterpret-cast' + - '-misc-include-cleaner' + - '-misc-non-private-member-variables-in-classes' + - '-modernize-return-braced-init-list' + - '-modernize-use-trailing-return-type' +FormatStyle: 'file' diff --git a/test/source/scwx/awips/wmo_header.test.cpp b/test/source/scwx/awips/wmo_header.test.cpp new file mode 100644 index 00000000..17cd9706 --- /dev/null +++ b/test/source/scwx/awips/wmo_header.test.cpp @@ -0,0 +1,130 @@ +#include + +#include + +namespace scwx::awips +{ + +static const std::string logPrefix_ = "scwx::awips::wmo_header.test"; + +static const std::string kWmoHeaderSample_ { + "887\n" + "WFUS54 KOUN 280044\n" + "TOROUN"}; + +TEST(WmoHeader, WmoFields) +{ + std::stringstream ss {kWmoHeaderSample_}; + WmoHeader header; + const bool valid = header.Parse(ss); + + EXPECT_EQ(valid, true); + EXPECT_EQ(header.sequence_number(), "887"); + EXPECT_EQ(header.data_type(), "WF"); + EXPECT_EQ(header.geographic_designator(), "US"); + EXPECT_EQ(header.bulletin_id(), "54"); + EXPECT_EQ(header.icao(), "KOUN"); + EXPECT_EQ(header.date_time(), "280044"); + EXPECT_EQ(header.bbb_indicator(), ""); + EXPECT_EQ(header.product_category(), "TOR"); + EXPECT_EQ(header.product_designator(), "OUN"); + EXPECT_EQ(header.GetDateTime(), + std::chrono::sys_time {}); +} + +TEST(WmoHeader, DateHintBeforeParse) +{ + using namespace std::chrono; + + std::stringstream ss {kWmoHeaderSample_}; + WmoHeader header; + + header.SetDateHint(2022y / October); + const bool valid = header.Parse(ss); + + EXPECT_EQ(valid, true); + EXPECT_EQ(header.GetDateTime(), + sys_days {2022y / October / 28d} + 0h + 44min); +} + +TEST(WmoHeader, DateHintAfterParse) +{ + using namespace std::chrono; + + std::stringstream ss {kWmoHeaderSample_}; + WmoHeader header; + + const bool valid = header.Parse(ss); + header.SetDateHint(2022y / October); + + EXPECT_EQ(valid, true); + EXPECT_EQ(header.GetDateTime(), + sys_days {2022y / October / 28d} + 0h + 44min); +} + +TEST(WmoHeader, EndTimeHintSameMonth) +{ + using namespace std::chrono; + + std::stringstream ss {kWmoHeaderSample_}; + WmoHeader header; + + const bool valid = header.Parse(ss); + + auto endTimeHint = sys_days {2022y / October / 29d} + 0h + 0min + 0s; + + EXPECT_EQ(valid, true); + EXPECT_EQ(header.GetDateTime(endTimeHint), + sys_days {2022y / October / 28d} + 0h + 44min); +} + +TEST(WmoHeader, EndTimeHintPreviousMonth) +{ + using namespace std::chrono; + + std::stringstream ss {kWmoHeaderSample_}; + WmoHeader header; + + const bool valid = header.Parse(ss); + + auto endTimeHint = sys_days {2022y / October / 27d} + 0h + 0min + 0s; + + EXPECT_EQ(valid, true); + EXPECT_EQ(header.GetDateTime(endTimeHint), + sys_days {2022y / September / 28d} + 0h + 44min); +} + +TEST(WmoHeader, EndTimeHintPreviousYear) +{ + using namespace std::chrono; + + std::stringstream ss {kWmoHeaderSample_}; + WmoHeader header; + + const bool valid = header.Parse(ss); + + auto endTimeHint = sys_days {2022y / January / 27d} + 0h + 0min + 0s; + + EXPECT_EQ(valid, true); + EXPECT_EQ(header.GetDateTime(endTimeHint), + sys_days {2021y / December / 28d} + 0h + 44min); +} + +TEST(WmoHeader, EndTimeHintIgnored) +{ + using namespace std::chrono; + + std::stringstream ss {kWmoHeaderSample_}; + WmoHeader header; + + header.SetDateHint(2022y / October); + const bool valid = header.Parse(ss); + + auto endTimeHint = sys_days {2020y / January / 1d} + 0h + 0min + 0s; + + EXPECT_EQ(valid, true); + EXPECT_EQ(header.GetDateTime(endTimeHint), + sys_days {2022y / October / 28d} + 0h + 44min); +} + +} // namespace scwx::awips diff --git a/test/source/scwx/provider/iem_api_provider.test.cpp b/test/source/scwx/provider/iem_api_provider.test.cpp new file mode 100644 index 00000000..e3e25669 --- /dev/null +++ b/test/source/scwx/provider/iem_api_provider.test.cpp @@ -0,0 +1,57 @@ +#include + +#include + +namespace scwx::provider +{ + +TEST(IemApiProviderTest, ListTextProducts) +{ + using namespace std::chrono; + using sys_days = time_point; + + auto date = sys_days {2023y / March / 25d}; + + auto torProducts = IemApiProvider::ListTextProducts(date, {}, "TOR"); + + ASSERT_EQ(torProducts.has_value(), true); + EXPECT_EQ(torProducts.value().size(), 35); + + if (torProducts.value().size() >= 1) + { + EXPECT_EQ(torProducts.value().at(0).productId_, + "202303250016-KMEG-WFUS54-TORMEG"); + } + if (torProducts.value().size() >= 35) + { + EXPECT_EQ(torProducts.value().at(34).productId_, + "202303252015-KFFC-WFUS52-TORFFC"); + } +} + +TEST(IemApiProviderTest, LoadTextProducts) +{ + static const std::vector productIds { + "202303250016-KMEG-WFUS54-TORMEG", + "202303252015-KFFC-WFUS52-TORFFC", + "202303311942-KLZK-WWUS54-SVSLZK"}; + + auto textProducts = IemApiProvider::LoadTextProducts(productIds); + + EXPECT_EQ(textProducts.size(), 3); + + if (textProducts.size() >= 1) + { + EXPECT_EQ(textProducts.at(0)->message_count(), 1); + } + if (textProducts.size() >= 2) + { + EXPECT_EQ(textProducts.at(1)->message_count(), 1); + } + if (textProducts.size() >= 3) + { + EXPECT_EQ(textProducts.at(2)->message_count(), 2); + } +} + +} // namespace scwx::provider diff --git a/test/source/scwx/provider/warnings_provider.test.cpp b/test/source/scwx/provider/warnings_provider.test.cpp index 78ef9b95..c1c824da 100644 --- a/test/source/scwx/provider/warnings_provider.test.cpp +++ b/test/source/scwx/provider/warnings_provider.test.cpp @@ -13,53 +13,27 @@ static const std::string& kAlternateUrl {"https://warnings.cod.edu"}; class WarningsProviderTest : public testing::TestWithParam { }; -TEST_P(WarningsProviderTest, ListFiles) -{ - WarningsProvider provider(GetParam()); - - auto [newObjects, totalObjects] = provider.ListFiles(); - - // No objects, skip test - if (totalObjects == 0) - { - GTEST_SKIP(); - } - - EXPECT_GT(newObjects, 0); - EXPECT_GT(totalObjects, 0); - EXPECT_EQ(newObjects, totalObjects); -} TEST_P(WarningsProviderTest, LoadUpdatedFiles) { WarningsProvider provider(GetParam()); - auto [newObjects, totalObjects] = provider.ListFiles(); - auto updatedFiles = provider.LoadUpdatedFiles(); + const std::chrono::sys_time now = + std::chrono::floor(std::chrono::system_clock::now()); + const std::chrono::sys_time startTime = + now - std::chrono::days {3}; + + auto updatedFiles = provider.LoadUpdatedFiles(startTime); // No objects, skip test - if (totalObjects == 0) + if (updatedFiles.empty()) { GTEST_SKIP(); } - EXPECT_GT(newObjects, 0); - EXPECT_GT(totalObjects, 0); - EXPECT_EQ(newObjects, totalObjects); - EXPECT_EQ(updatedFiles.size(), newObjects); + EXPECT_GT(updatedFiles.size(), 0); - auto [newObjects2, totalObjects2] = provider.ListFiles(); - auto updatedFiles2 = provider.LoadUpdatedFiles(); - - // There should be no more than 2 updated warnings files since the last query - // (assumption that the previous newest file was updated, and a new file was - // created on the hour) - EXPECT_LE(newObjects2, 2); - EXPECT_EQ(updatedFiles2.size(), newObjects2); - - // The total number of objects may have changed, since the oldest file could - // have dropped off the list - EXPECT_GT(totalObjects2, 0); + auto updatedFiles2 = provider.LoadUpdatedFiles(); } INSTANTIATE_TEST_SUITE_P(WarningsProvider, diff --git a/test/test.cmake b/test/test.cmake index 3ec6ef19..0ae26b53 100644 --- a/test/test.cmake +++ b/test/test.cmake @@ -12,13 +12,15 @@ set(SRC_AWIPS_TESTS source/scwx/awips/coded_location.test.cpp source/scwx/awips/coded_time_motion_location.test.cpp source/scwx/awips/pvtec.test.cpp source/scwx/awips/text_product_file.test.cpp - source/scwx/awips/ugc.test.cpp) + source/scwx/awips/ugc.test.cpp + source/scwx/awips/wmo_header.test.cpp) set(SRC_COMMON_TESTS source/scwx/common/color_table.test.cpp source/scwx/common/products.test.cpp) set(SRC_GR_TESTS source/scwx/gr/placefile.test.cpp) set(SRC_NETWORK_TESTS source/scwx/network/dir_list.test.cpp) set(SRC_PROVIDER_TESTS source/scwx/provider/aws_level2_data_provider.test.cpp source/scwx/provider/aws_level3_data_provider.test.cpp + source/scwx/provider/iem_api_provider.test.cpp source/scwx/provider/warnings_provider.test.cpp) set(SRC_QT_CONFIG_TESTS source/scwx/qt/config/county_database.test.cpp source/scwx/qt/config/radar_site.test.cpp) diff --git a/wxdata/include/scwx/awips/text_product_file.hpp b/wxdata/include/scwx/awips/text_product_file.hpp index 478a93b4..b0d1c965 100644 --- a/wxdata/include/scwx/awips/text_product_file.hpp +++ b/wxdata/include/scwx/awips/text_product_file.hpp @@ -5,9 +5,7 @@ #include #include -namespace scwx -{ -namespace awips +namespace scwx::awips { class TextProductFileImpl; @@ -24,16 +22,17 @@ public: TextProductFile(TextProductFile&&) noexcept; TextProductFile& operator=(TextProductFile&&) noexcept; - size_t message_count() const; - std::vector> messages() const; - std::shared_ptr message(size_t i) const; + [[nodiscard]] std::size_t message_count() const; + [[nodiscard]] std::vector> + messages() const; + [[nodiscard]] std::shared_ptr message(size_t i) const; bool LoadFile(const std::string& filename); - bool LoadData(std::istream& is); + bool LoadData(const std::string& filename, std::istream& is); private: - std::unique_ptr p; + class Impl; + std::unique_ptr p; }; -} // namespace awips -} // namespace scwx +} // namespace scwx::awips diff --git a/wxdata/include/scwx/awips/text_product_message.hpp b/wxdata/include/scwx/awips/text_product_message.hpp index b043494f..373dc223 100644 --- a/wxdata/include/scwx/awips/text_product_message.hpp +++ b/wxdata/include/scwx/awips/text_product_message.hpp @@ -13,6 +13,8 @@ #include #include +#include + namespace scwx { namespace awips @@ -94,6 +96,7 @@ public: TextProductMessage(TextProductMessage&&) noexcept; TextProductMessage& operator=(TextProductMessage&&) noexcept; + [[nodiscard]] boost::uuids::uuid uuid() const; std::string message_content() const; std::shared_ptr wmo_header() const; std::vector mnd_header() const; diff --git a/wxdata/include/scwx/awips/wmo_header.hpp b/wxdata/include/scwx/awips/wmo_header.hpp index f3487b6d..889c955e 100644 --- a/wxdata/include/scwx/awips/wmo_header.hpp +++ b/wxdata/include/scwx/awips/wmo_header.hpp @@ -1,11 +1,11 @@ #pragma once +#include #include +#include #include -namespace scwx -{ -namespace awips +namespace scwx::awips { class WmoHeaderImpl; @@ -27,7 +27,7 @@ public: explicit WmoHeader(); ~WmoHeader(); - WmoHeader(const WmoHeader&) = delete; + WmoHeader(const WmoHeader&) = delete; WmoHeader& operator=(const WmoHeader&) = delete; WmoHeader(WmoHeader&&) noexcept; @@ -35,21 +35,53 @@ public: bool operator==(const WmoHeader& o) const; - std::string sequence_number() const; - std::string data_type() const; - std::string geographic_designator() const; - std::string bulletin_id() const; - std::string icao() const; - std::string date_time() const; - std::string bbb_indicator() const; - std::string product_category() const; - std::string product_designator() const; + [[nodiscard]] std::string sequence_number() const; + [[nodiscard]] std::string data_type() const; + [[nodiscard]] std::string geographic_designator() const; + [[nodiscard]] std::string bulletin_id() const; + [[nodiscard]] std::string icao() const; + [[nodiscard]] std::string date_time() const; + [[nodiscard]] std::string bbb_indicator() const; + [[nodiscard]] std::string product_category() const; + [[nodiscard]] std::string product_designator() const; + /** + * @brief Get the WMO date/time + * + * Gets the WMO date/time. Uses the optional date hint provided via + * SetDateHint(std::chrono::year_month). If the date hint has not been + * provided, the endTimeHint parameter is required. + * + * @param [in] endTimeHint The optional end time bounds to provide. This is + * ignored if a date hint has been provided to determine an absolute date. + */ + [[nodiscard]] std::chrono::sys_time GetDateTime( + std::optional endTimeHint = + std::nullopt); + + /** + * @brief Parse a WMO header + * + * @param [in] is The input stream to parse + */ bool Parse(std::istream& is); + /** + * @brief Provide a date hint for the WMO parser + * + * The WMO header contains a date/time in the format DDHHMM. The year and + * month must be derived using another source. The date hint provides the + * additional context required to determine the absolute product time. + * + * This function will update any absolute date/time already calculated, or + * affect the calculation of a subsequent absolute date/time. + * + * @param [in] dateHint The date hint to provide the WMO header parser + */ + void SetDateHint(std::chrono::year_month dateHint); + private: std::unique_ptr p; }; -} // namespace awips -} // namespace scwx +} // namespace scwx::awips diff --git a/wxdata/include/scwx/provider/iem_api_provider.hpp b/wxdata/include/scwx/provider/iem_api_provider.hpp new file mode 100644 index 00000000..1aac31b2 --- /dev/null +++ b/wxdata/include/scwx/provider/iem_api_provider.hpp @@ -0,0 +1,80 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +#if defined(_MSC_VER) +# pragma warning(push) +# pragma warning(disable : 4702) +#endif + +#if defined(_MSC_VER) +# pragma warning(pop) +#endif + +namespace scwx::provider +{ + +/** + * @brief Warnings Provider + */ +class IemApiProvider +{ +public: + explicit IemApiProvider(); + ~IemApiProvider(); + + IemApiProvider(const IemApiProvider&) = delete; + IemApiProvider& operator=(const IemApiProvider&) = delete; + + IemApiProvider(IemApiProvider&&) noexcept; + IemApiProvider& operator=(IemApiProvider&&) noexcept; + + static boost::outcome_v2::result> + ListTextProducts(std::chrono::sys_days date, + std::optional cccc = {}, + std::optional pil = {}); + + template + requires std::same_as, + std::chrono::sys_days> && + std::same_as, + std::string_view> && + std::same_as, std::string_view> + static boost::outcome_v2::result> + ListTextProducts(DateRange dates, CcccRange ccccs, PilRange pils); + + template + requires std::same_as, std::string> + static std::vector> + LoadTextProducts(const Range& textProducts); + +private: + class Impl; + std::unique_ptr p; + + static boost::outcome_v2::result> + ProcessTextProductLists(std::vector& asyncResponses); + static std::vector> + ProcessTextProductFiles( + std::vector>& asyncResponses); + + static const std::shared_ptr logger_; + + static const std::string kBaseUrl_; + static const std::string kListNwsTextProductsEndpoint_; + static const std::string kNwsTextProductEndpoint_; +}; + +} // namespace scwx::provider diff --git a/wxdata/include/scwx/provider/iem_api_provider.ipp b/wxdata/include/scwx/provider/iem_api_provider.ipp new file mode 100644 index 00000000..25187376 --- /dev/null +++ b/wxdata/include/scwx/provider/iem_api_provider.ipp @@ -0,0 +1,111 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include + +#if (__cpp_lib_chrono < 201907L) +# include +#endif + +namespace scwx::provider +{ + +template + requires std::same_as, + std::chrono::sys_days> && + std::same_as, std::string_view> && + std::same_as, std::string_view> +boost::outcome_v2::result> +IemApiProvider::ListTextProducts(DateRange dates, + CcccRange ccccs, + PilRange pils) +{ + using namespace std::chrono; + +#if (__cpp_lib_chrono >= 201907L) + namespace df = std; + + static constexpr std::string_view kDateFormat {"{:%Y-%m-%d}"}; +#else + using namespace date; + namespace df = date; + +# define kDateFormat "%Y-%m-%d" +#endif + + auto formattedDates = dates | ranges::views::transform( + [](const std::chrono::sys_days& date) + { return df::format(kDateFormat, date); }); + + logger_->debug("Listing text products for: {}", + boost::algorithm::join(formattedDates, ", ")); + + std::vector asyncResponses {}; + + for (const auto& [date, cccc, pil] : + ranges::views::cartesian_product(dates, ccccs, pils)) + { + auto parameters = + cpr::Parameters {{"date", df::format(kDateFormat, date)}}; + + // WMO Source Code + if (!cccc.empty()) + { + parameters.Add({"cccc", std::string {cccc}}); + } + + // AFOS / AWIPS ID / 3-6 length identifier + if (!pil.empty()) + { + parameters.Add({"pil", std::string {pil}}); + } + + asyncResponses.emplace_back( + cpr::GetAsync(cpr::Url {kBaseUrl_ + kListNwsTextProductsEndpoint_}, + network::cpr::GetHeader(), + parameters)); + } + + return ProcessTextProductLists(asyncResponses); +} + +template + requires std::same_as, std::string> +std::vector> +IemApiProvider::LoadTextProducts(const Range& textProducts) +{ + auto parameters = cpr::Parameters {{"nolimit", "true"}}; + + logger_->debug("Loading {} text products", textProducts.size()); + + std::vector> asyncResponses {}; + asyncResponses.reserve(textProducts.size()); + + const std::string endpointUrl = kBaseUrl_ + kNwsTextProductEndpoint_; + + for (const auto& productId : textProducts) + { + asyncResponses.emplace_back( + productId, + cpr::GetAsync(cpr::Url {endpointUrl + productId}, + network::cpr::GetHeader(), + parameters)); + } + + return ProcessTextProductFiles(asyncResponses); +} + +#ifdef kDateFormat +# undef kDateFormat +#endif + +} // namespace scwx::provider diff --git a/wxdata/include/scwx/provider/warnings_provider.hpp b/wxdata/include/scwx/provider/warnings_provider.hpp index e519ec5d..0d14d258 100644 --- a/wxdata/include/scwx/provider/warnings_provider.hpp +++ b/wxdata/include/scwx/provider/warnings_provider.hpp @@ -2,9 +2,7 @@ #include -namespace scwx -{ -namespace provider +namespace scwx::provider { /** @@ -22,15 +20,12 @@ public: WarningsProvider(WarningsProvider&&) noexcept; WarningsProvider& operator=(WarningsProvider&&) noexcept; - std::pair - ListFiles(std::chrono::system_clock::time_point newerThan = {}); std::vector> - LoadUpdatedFiles(std::chrono::system_clock::time_point newerThan = {}); + LoadUpdatedFiles(std::chrono::sys_time newerThan = {}); private: class Impl; std::unique_ptr p; }; -} // namespace provider -} // namespace scwx +} // namespace scwx::provider diff --git a/wxdata/include/scwx/types/iem_types.hpp b/wxdata/include/scwx/types/iem_types.hpp new file mode 100644 index 00000000..ee461c36 --- /dev/null +++ b/wxdata/include/scwx/types/iem_types.hpp @@ -0,0 +1,73 @@ +#pragma once + +#include +#include + +#include + +namespace scwx::types::iem +{ + +/** + * @brief AFOS Entry object + * + * + */ +struct AfosEntry +{ + std::int64_t index_ {}; + std::string entered_ {}; + std::string pil_ {}; + std::string productId_ {}; + std::string cccc_ {}; + std::int64_t count_ {}; + std::string link_ {}; + std::string textLink_ {}; +}; + +/** + * @brief AFOS List object + * + * + */ +struct AfosList +{ + std::vector data_ {}; +}; + +/** + * @brief Bad Request (400) object + */ +struct BadRequest +{ + std::string detail_ {}; +}; + +/** + * @brief Validation Error (422) object + */ +struct ValidationError +{ + struct Detail + { + std::string type_ {}; + std::vector> loc_ {}; + std::string msg_ {}; + std::string input_ {}; + struct Context + { + std::string error_ {}; + } ctx_; + }; + + std::vector detail_ {}; +}; + +AfosList tag_invoke(boost::json::value_to_tag, + const boost::json::value& jv); +BadRequest tag_invoke(boost::json::value_to_tag, + const boost::json::value& jv); +ValidationError tag_invoke(boost::json::value_to_tag, + const boost::json::value& jv); + +} // namespace scwx::types::iem diff --git a/wxdata/include/scwx/util/json.hpp b/wxdata/include/scwx/util/json.hpp new file mode 100644 index 00000000..ab836edc --- /dev/null +++ b/wxdata/include/scwx/util/json.hpp @@ -0,0 +1,15 @@ +#pragma once + +#include + +namespace scwx::util::json +{ + +boost::json::value ReadJsonFile(const std::string& path); +boost::json::value ReadJsonStream(std::istream& is); +boost::json::value ReadJsonString(std::string_view sv); +void WriteJsonFile(const std::string& path, + const boost::json::value& json, + bool prettyPrint = true); + +} // namespace scwx::util::json diff --git a/wxdata/source/scwx/awips/text_product_file.cpp b/wxdata/source/scwx/awips/text_product_file.cpp index 3edc7b2d..96d5503f 100644 --- a/wxdata/source/scwx/awips/text_product_file.cpp +++ b/wxdata/source/scwx/awips/text_product_file.cpp @@ -3,24 +3,31 @@ #include -namespace scwx -{ -namespace awips +#include + +namespace scwx::awips { static const std::string logPrefix_ = "scwx::awips::text_product_file"; static const auto logger_ = util::Logger::Create(logPrefix_); -class TextProductFileImpl +class TextProductFile::Impl { public: - explicit TextProductFileImpl() : messages_ {} {}; - ~TextProductFileImpl() = default; + explicit Impl() : messages_ {} {}; + ~Impl() = default; + + Impl(const Impl&) = delete; + Impl& operator=(const Impl&) = delete; + + Impl(Impl&&) = delete; + Impl& operator=(Impl&&) = delete; std::vector> messages_; }; -TextProductFile::TextProductFile() : p(std::make_unique()) +TextProductFile::TextProductFile() : + p(std::make_unique()) { } TextProductFile::~TextProductFile() = default; @@ -59,16 +66,34 @@ bool TextProductFile::LoadFile(const std::string& filename) if (fileValid) { - fileValid = LoadData(f); + fileValid = LoadData(filename, f); } return fileValid; } -bool TextProductFile::LoadData(std::istream& is) +bool TextProductFile::LoadData(const std::string& filename, std::istream& is) { + static constexpr LazyRE2 kDateTimePattern_ = { + R"(((?:19|20)\d{2}))" // Year (YYYY) + R"((0[1-9]|1[0-2]))" // Month (MM) + R"((0[1-9]|[12]\d|3[01]))" // Day (DD) + R"(_?)" // Optional separator (not captured) + R"(([01]\d|2[0-3]))" // Hour (HH) + }; + logger_->trace("Loading Data"); + // Attempt to parse the date from the filename + std::optional yearMonth; + int year {}; + unsigned int month {}; + + if (RE2::PartialMatch(filename, *kDateTimePattern_, &year, &month)) + { + yearMonth = std::chrono::year {year} / std::chrono::month {month}; + } + while (!is.eof()) { std::shared_ptr message = @@ -77,7 +102,7 @@ bool TextProductFile::LoadData(std::istream& is) if (message != nullptr) { - for (auto m : p->messages_) + for (const auto& m : p->messages_) { if (*m->wmo_header().get() == *message->wmo_header().get()) { @@ -88,6 +113,11 @@ bool TextProductFile::LoadData(std::istream& is) if (!duplicate) { + if (yearMonth.has_value()) + { + message->wmo_header()->SetDateHint(yearMonth.value()); + } + p->messages_.push_back(message); } } @@ -100,5 +130,4 @@ bool TextProductFile::LoadData(std::istream& is) return !p->messages_.empty(); } -} // namespace awips -} // namespace scwx +} // namespace scwx::awips diff --git a/wxdata/source/scwx/awips/text_product_message.cpp b/wxdata/source/scwx/awips/text_product_message.cpp index 54ce7e25..53ec502d 100644 --- a/wxdata/source/scwx/awips/text_product_message.cpp +++ b/wxdata/source/scwx/awips/text_product_message.cpp @@ -9,11 +9,10 @@ #include #include +#include #include -namespace scwx -{ -namespace awips +namespace scwx::awips { static const std::string logPrefix_ = "scwx::awips::text_product_message"; @@ -27,8 +26,8 @@ static const auto logger_ = scwx::util::Logger::Create(logPrefix_); // Look for hhmm (xM|UTC) to key the date/time string static constexpr LazyRE2 reDateTimeString = {"^[0-9]{3,4} ([AP]M|UTC)"}; -static void ParseCodedInformation(std::shared_ptr segment, - const std::string& wfo); +static void ParseCodedInformation(const std::shared_ptr& segment, + const std::string& wfo); static std::vector ParseProductContent(std::istream& is); static void SkipBlankLines(std::istream& is); static bool TryParseEndOfProduct(std::istream& is); @@ -50,6 +49,13 @@ public: } ~TextProductMessageImpl() = default; + TextProductMessageImpl(const TextProductMessageImpl&) = delete; + TextProductMessageImpl& operator=(const TextProductMessageImpl&) = delete; + TextProductMessageImpl(const TextProductMessageImpl&&) = delete; + TextProductMessageImpl& operator=(const TextProductMessageImpl&&) = delete; + + boost::uuids::uuid uuid_ {boost::uuids::random_generator()()}; + std::string messageContent_; std::shared_ptr wmoHeader_; std::vector mndHeader_; @@ -67,6 +73,11 @@ TextProductMessage::TextProductMessage(TextProductMessage&&) noexcept = default; TextProductMessage& TextProductMessage::operator=(TextProductMessage&&) noexcept = default; +boost::uuids::uuid TextProductMessage::uuid() const +{ + return p->uuid_; +} + std::string TextProductMessage::message_content() const { return p->messageContent_; @@ -116,71 +127,11 @@ std::chrono::system_clock::time_point Segment::event_begin() const // If event begin is 000000T0000Z if (eventBegin == std::chrono::system_clock::time_point {}) { - using namespace std::chrono; - // Determine event end from P-VTEC string - system_clock::time_point eventEnd = + std::chrono::system_clock::time_point eventEnd = header_->vtecString_[0].pVtec_.event_end(); - auto endDays = floor(eventEnd); - year_month_day endDate {endDays}; - - // Determine WMO date/time - std::string wmoDateTime = wmoHeader_->date_time(); - - bool wmoDateTimeValid = false; - unsigned int dayOfMonth = 0; - unsigned long beginHour = 0; - unsigned long beginMinute = 0; - - try - { - // WMO date time is in the format DDHHMM - dayOfMonth = - static_cast(std::stoul(wmoDateTime.substr(0, 2))); - beginHour = std::stoul(wmoDateTime.substr(2, 2)); - beginMinute = std::stoul(wmoDateTime.substr(4, 2)); - wmoDateTimeValid = true; - } - catch (const std::exception&) - { - logger_->warn("Malformed WMO date/time: {}", wmoDateTime); - } - - if (wmoDateTimeValid) - { - // Combine end date year and month with WMO date time - eventBegin = - sys_days {endDate.year() / endDate.month() / day {dayOfMonth}} + - hours {beginHour} + minutes {beginMinute}; - - // If the begin date is after the end date, assume the start time - // was the previous month (give a 1 day grace period for expiring - // events in the past) - if (eventBegin > eventEnd + 24h) - { - // If the current end month is January - if (endDate.month() == January) - { - // The begin month must be December of last year - eventBegin = - sys_days { - year {static_cast((endDate.year() - 1y).count())} / - December / day {dayOfMonth}} + - hours {beginHour} + minutes {beginMinute}; - } - else - { - // Back up one month - eventBegin = - sys_days {endDate.year() / - month {static_cast( - (endDate.month() - month {1}).count())} / - day {dayOfMonth}} + - hours {beginHour} + minutes {beginMinute}; - } - } - } + eventBegin = wmoHeader_->GetDateTime(eventEnd); } } @@ -232,7 +183,7 @@ bool TextProductMessage::Parse(std::istream& is) if (i == 0) { - if (is.peek() != '\r') + if (is.peek() != '\r' && is.peek() != '\n') { segment->header_ = TryParseSegmentHeader(is); } @@ -318,8 +269,8 @@ bool TextProductMessage::Parse(std::istream& is) return dataValid; } -void ParseCodedInformation(std::shared_ptr segment, - const std::string& wfo) +void ParseCodedInformation(const std::shared_ptr& segment, + const std::string& wfo) { typedef std::vector::const_iterator StringIterator; @@ -352,8 +303,8 @@ void ParseCodedInformation(std::shared_ptr segment, codedLocationEnd = it; } - else if (codedMotionBegin == productContent.cend() && - it->starts_with("TIME...MOT...LOC")) + if (codedMotionBegin == productContent.cend() && + it->starts_with("TIME...MOT...LOC")) { codedMotionBegin = it; } @@ -366,8 +317,7 @@ void ParseCodedInformation(std::shared_ptr segment, codedMotionEnd = it; } - else if (!segment->observed_ && - it->find("...OBSERVED") != std::string::npos) + if (!segment->observed_ && it->find("...OBSERVED") != std::string::npos) { segment->observed_ = true; } @@ -378,6 +328,8 @@ void ParseCodedInformation(std::shared_ptr segment, segment->tornadoPossible_ = true; } + // Assignment of an iterator permitted + // NOLINTBEGIN(bugprone-assignment-in-if-condition) else if (segment->threatCategory_ == ibw::ThreatCategory::Base && (threatTagIt = std::find_if(kThreatCategoryTags.cbegin(), kThreatCategoryTags.cend(), @@ -385,6 +337,7 @@ void ParseCodedInformation(std::shared_ptr segment, return it->starts_with(tag); })) != kThreatCategoryTags.cend() && it->length() > threatTagIt->length()) + // NOLINTEND(bugprone-assignment-in-if-condition) { const std::string threatCategoryName = it->substr(threatTagIt->length()); @@ -458,7 +411,7 @@ void SkipBlankLines(std::istream& is) { std::string line; - while (is.peek() == '\r') + while (is.peek() == '\r' || is.peek() == '\n') { util::getline(is, line); } @@ -513,7 +466,7 @@ std::vector TryParseMndHeader(std::istream& is) std::string line; std::streampos isBegin = is.tellg(); - while (!is.eof() && is.peek() != '\r') + while (!is.eof() && is.peek() != '\r' && is.peek() != '\n') { util::getline(is, line); mndHeader.push_back(line); @@ -546,7 +499,7 @@ std::vector TryParseOverviewBlock(std::istream& is) if (is.peek() == '.') { - while (!is.eof() && is.peek() != '\r') + while (!is.eof() && is.peek() != '\r' && is.peek() != '\n') { util::getline(is, line); overviewBlock.push_back(line); @@ -576,7 +529,7 @@ std::optional TryParseSegmentHeader(std::istream& is) header->ugcString_.push_back(line); // If UGC is multi-line, continue parsing - while (!is.eof() && is.peek() != '\r' && + while (!is.eof() && is.peek() != '\r' && is.peek() != '\n' && !RE2::PartialMatch(line, *reUgcExpiration)) { util::getline(is, line); @@ -595,7 +548,7 @@ std::optional TryParseSegmentHeader(std::istream& is) header->vtecString_.push_back(std::move(*vtec)); } - while (!is.eof() && is.peek() != '\r') + while (!is.eof() && is.peek() != '\r' && is.peek() != '\n') { util::getline(is, line); if (!RE2::PartialMatch(line, *reDateTimeString)) @@ -640,10 +593,8 @@ std::optional TryParseVtecString(std::istream& is) if (RE2::PartialMatch(line, *rePVtecString)) { - bool vtecValid; - - vtec = Vtec(); - vtecValid = vtec->pVtec_.Parse(line); + vtec = Vtec(); + const bool vtecValid = vtec->pVtec_.Parse(line); isBegin = is.tellg(); @@ -687,5 +638,4 @@ std::shared_ptr TextProductMessage::Create(std::istream& is) return message; } -} // namespace awips -} // namespace scwx +} // namespace scwx::awips diff --git a/wxdata/source/scwx/awips/wmo_header.cpp b/wxdata/source/scwx/awips/wmo_header.cpp index eb4501e7..f89db609 100644 --- a/wxdata/source/scwx/awips/wmo_header.cpp +++ b/wxdata/source/scwx/awips/wmo_header.cpp @@ -12,14 +12,19 @@ # include #endif -namespace scwx -{ -namespace awips +namespace scwx::awips { static const std::string logPrefix_ = "scwx::awips::wmo_header"; static const auto logger_ = util::Logger::Create(logPrefix_); +static constexpr std::size_t kWmoHeaderMinLineLength_ = 18; +static constexpr std::size_t kWmoIdentifierLengthMin_ = 5; +static constexpr std::size_t kWmoIdentifierLengthMax_ = 6; +static constexpr std::size_t kIcaoLength_ = 4; +static constexpr std::size_t kDateTimeLength_ = 6; +static constexpr std::size_t kAwipsIdentifierLineLength_ = 6; + class WmoHeaderImpl { public: @@ -37,17 +42,31 @@ public: } ~WmoHeaderImpl() = default; + WmoHeaderImpl(const WmoHeaderImpl&) = delete; + WmoHeaderImpl& operator=(const WmoHeaderImpl&) = delete; + WmoHeaderImpl(const WmoHeaderImpl&&) = delete; + WmoHeaderImpl& operator=(const WmoHeaderImpl&&) = delete; + + void CalculateAbsoluteDateTime(); + bool ParseDateTime(unsigned int& dayOfMonth, + unsigned long& hour, + unsigned long& minute); + bool operator==(const WmoHeaderImpl& o) const; - std::string sequenceNumber_; - std::string dataType_; - std::string geographicDesignator_; - std::string bulletinId_; - std::string icao_; - std::string dateTime_; - std::string bbbIndicator_; - std::string productCategory_; - std::string productDesignator_; + std::string sequenceNumber_ {}; + std::string dataType_ {}; + std::string geographicDesignator_ {}; + std::string bulletinId_ {}; + std::string icao_ {}; + std::string dateTime_ {}; + std::string bbbIndicator_ {}; + std::string productCategory_ {}; + std::string productDesignator_ {}; + + std::optional dateHint_ {}; + std::optional> + absoluteDateTime_ {}; }; WmoHeader::WmoHeader() : p(std::make_unique()) {} @@ -119,6 +138,71 @@ std::string WmoHeader::product_designator() const return p->productDesignator_; } +std::chrono::sys_time WmoHeader::GetDateTime( + std::optional endTimeHint) +{ + std::chrono::sys_time wmoDateTime {}; + + const auto absoluteDateTime = p->absoluteDateTime_; + + if (absoluteDateTime.has_value()) + { + wmoDateTime = absoluteDateTime.value(); + } + else if (endTimeHint.has_value()) + { + bool dateTimeValid = false; + unsigned int dayOfMonth = 0; + unsigned long hour = 0; + unsigned long minute = 0; + + dateTimeValid = p->ParseDateTime(dayOfMonth, hour, minute); + + if (dateTimeValid) + { + using namespace std::chrono; + + const auto endDays = floor(endTimeHint.value()); + const year_month_day endDate {endDays}; + + // Combine end date year and month with WMO date time + wmoDateTime = + sys_days {endDate.year() / endDate.month() / day {dayOfMonth}} + + hours {hour} + minutes {minute}; + + // If the begin date is after the end date, assume the start time + // was the previous month (give a 1 day grace period for expiring + // events in the past) + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) + if (wmoDateTime > endTimeHint.value() + 24h) + { + // If the current end month is January + if (endDate.month() == January) + { + // The begin month must be December of last year + wmoDateTime = + sys_days { + year {static_cast((endDate.year() - 1y).count())} / + December / day {dayOfMonth}} + + hours {hour} + minutes {minute}; + } + else + { + // Back up one month + wmoDateTime = + sys_days {endDate.year() / + month {static_cast( + (endDate.month() - month {1}).count())} / + day {dayOfMonth}} + + hours {hour} + minutes {minute}; + } + } + } + } + + return wmoDateTime; +} + bool WmoHeader::Parse(std::istream& is) { bool headerValid = true; @@ -132,9 +216,21 @@ bool WmoHeader::Parse(std::istream& is) { util::getline(is, sohLine); util::getline(is, sequenceLine); + util::getline(is, wmoLine); + } + else + { + // The next line could be the WMO line or the sequence line + util::getline(is, wmoLine); + if (wmoLine.length() < kWmoHeaderMinLineLength_) + { + // This is likely the sequence line instead + sequenceLine.swap(wmoLine); + util::getline(is, wmoLine); + } } - util::getline(is, wmoLine); + auto awipsLinePos = is.tellg(); util::getline(is, awipsLine); if (is.eof()) @@ -179,17 +275,18 @@ bool WmoHeader::Parse(std::istream& is) logger_->warn("Invalid number of WMO tokens"); headerValid = false; } - else if (wmoTokenList[0].size() != 6) + else if (wmoTokenList[0].size() < kWmoIdentifierLengthMin_ || + wmoTokenList[0].size() > kWmoIdentifierLengthMax_) { logger_->warn("WMO identifier malformed"); headerValid = false; } - else if (wmoTokenList[1].size() != 4) + else if (wmoTokenList[1].size() != kIcaoLength_) { logger_->warn("ICAO malformed"); headerValid = false; } - else if (wmoTokenList[2].size() != 6) + else if (wmoTokenList[2].size() != kDateTimeLength_) { logger_->warn("Date/time malformed"); headerValid = false; @@ -204,9 +301,11 @@ bool WmoHeader::Parse(std::istream& is) { p->dataType_ = wmoTokenList[0].substr(0, 2); p->geographicDesignator_ = wmoTokenList[0].substr(2, 2); - p->bulletinId_ = wmoTokenList[0].substr(4, 2); - p->icao_ = wmoTokenList[1]; - p->dateTime_ = wmoTokenList[2]; + p->bulletinId_ = wmoTokenList[0].substr(4, wmoTokenList[0].size() - 4); + p->icao_ = wmoTokenList[1]; + p->dateTime_ = wmoTokenList[2]; + + p->CalculateAbsoluteDateTime(); if (wmoTokenList.size() == 4) { @@ -224,10 +323,14 @@ bool WmoHeader::Parse(std::istream& is) if (headerValid) { - if (awipsLine.size() != 6) + if (awipsLine.size() != kAwipsIdentifierLineLength_) { - logger_->warn("AWIPS Identifier Line bad size"); - headerValid = false; + // Older products may be missing an AWIPS Identifier Line + logger_->trace("AWIPS Identifier Line bad size"); + + is.seekg(awipsLinePos); + p->productCategory_ = ""; + p->productDesignator_ = ""; } else { @@ -239,5 +342,60 @@ bool WmoHeader::Parse(std::istream& is) return headerValid; } -} // namespace awips -} // namespace scwx +void WmoHeader::SetDateHint(std::chrono::year_month dateHint) +{ + p->dateHint_ = dateHint; + p->CalculateAbsoluteDateTime(); +} + +bool WmoHeaderImpl::ParseDateTime(unsigned int& dayOfMonth, + unsigned long& hour, + unsigned long& minute) +{ + bool dateTimeValid = false; + + try + { + // WMO date time is in the format DDHHMM + dayOfMonth = + static_cast(std::stoul(dateTime_.substr(0, 2))); + hour = std::stoul(dateTime_.substr(2, 2)); + minute = std::stoul(dateTime_.substr(4, 2)); + dateTimeValid = true; + } + catch (const std::exception&) + { + logger_->warn("Malformed WMO date/time: {}", dateTime_); + } + + return dateTimeValid; +} + +void WmoHeaderImpl::CalculateAbsoluteDateTime() +{ + bool dateTimeValid = false; + + if (dateHint_.has_value() && !dateTime_.empty()) + { + unsigned int dayOfMonth = 0; + unsigned long hour = 0; + unsigned long minute = 0; + + dateTimeValid = ParseDateTime(dayOfMonth, hour, minute); + + if (dateTimeValid) + { + using namespace std::chrono; + absoluteDateTime_ = sys_days {dateHint_->year() / dateHint_->month() / + day {dayOfMonth}} + + hours {hour} + minutes {minute}; + } + } + + if (!dateTimeValid) + { + absoluteDateTime_.reset(); + } +} + +} // namespace scwx::awips diff --git a/wxdata/source/scwx/provider/iem_api_provider.cpp b/wxdata/source/scwx/provider/iem_api_provider.cpp new file mode 100644 index 00000000..ef0576e7 --- /dev/null +++ b/wxdata/source/scwx/provider/iem_api_provider.cpp @@ -0,0 +1,185 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#if (__cpp_lib_chrono < 201907L) +# include +#endif + +namespace scwx::provider +{ + +static const std::string logPrefix_ = "scwx::provider::iem_api_provider"; + +const std::shared_ptr IemApiProvider::logger_ = + util::Logger::Create(logPrefix_); + +const std::string IemApiProvider::kBaseUrl_ = + "https://mesonet.agron.iastate.edu/api/1"; + +const std::string IemApiProvider::kListNwsTextProductsEndpoint_ = + "/nws/afos/list.json"; +const std::string IemApiProvider::kNwsTextProductEndpoint_ = "/nwstext/"; + +class IemApiProvider::Impl +{ +public: + explicit Impl() = default; + ~Impl() = default; + Impl(const Impl&) = delete; + Impl& operator=(const Impl&) = delete; + Impl(const Impl&&) = delete; + Impl& operator=(const Impl&&) = delete; +}; + +IemApiProvider::IemApiProvider() : p(std::make_unique()) {} +IemApiProvider::~IemApiProvider() = default; + +IemApiProvider::IemApiProvider(IemApiProvider&&) noexcept = default; +IemApiProvider& IemApiProvider::operator=(IemApiProvider&&) noexcept = default; + +boost::outcome_v2::result> +IemApiProvider::ListTextProducts(std::chrono::sys_days date, + std::optional optionalCccc, + std::optional optionalPil) +{ + const std::string_view cccc = + optionalCccc.has_value() ? optionalCccc.value() : std::string_view {}; + const std::string_view pil = + optionalPil.has_value() ? optionalPil.value() : std::string_view {}; + + const auto dateArray = std::array {date}; + const auto ccccArray = std::array {cccc}; + const auto pilArray = std::array {pil}; + + return ListTextProducts(dateArray, ccccArray, pilArray); +} + +boost::outcome_v2::result> +IemApiProvider::ProcessTextProductLists( + std::vector& asyncResponses) +{ + std::vector textProducts {}; + + for (auto& asyncResponse : asyncResponses) + { + auto response = asyncResponse.get(); + + const boost::json::value json = util::json::ReadJsonString(response.text); + + if (response.status_code == cpr::status::HTTP_OK) + { + try + { + // Get AFOS list from response + auto entries = boost::json::value_to(json); + textProducts.insert(textProducts.end(), + std::make_move_iterator(entries.data_.begin()), + std::make_move_iterator(entries.data_.end())); + } + catch (const std::exception& ex) + { + // Unexpected bad response + logger_->warn("Error parsing JSON: {}", ex.what()); + return boost::system::errc::make_error_code( + boost::system::errc::bad_message); + } + } + else if (response.status_code == cpr::status::HTTP_BAD_REQUEST && + json != nullptr) + { + try + { + // Log bad request details + auto badRequest = + boost::json::value_to(json); + logger_->warn("ListTextProducts bad request: {}", + badRequest.detail_); + } + catch (const std::exception& ex) + { + // Unexpected bad response + logger_->warn("Error parsing bad response: {}", ex.what()); + } + + return boost::system::errc::make_error_code( + boost::system::errc::invalid_argument); + } + else if (response.status_code == cpr::status::HTTP_UNPROCESSABLE_ENTITY && + json != nullptr) + { + try + { + // Log validation error details + auto error = + boost::json::value_to(json); + logger_->warn("ListTextProducts validation error: {}", + error.detail_.at(0).msg_); + } + catch (const std::exception& ex) + { + // Unexpected bad response + logger_->warn("Error parsing validation error: {}", ex.what()); + } + + return boost::system::errc::make_error_code( + boost::system::errc::no_message_available); + } + else + { + logger_->warn("Could not list text products: {}", + response.status_line); + + return boost::system::errc::make_error_code( + boost::system::errc::no_message); + } + } + + logger_->debug("Found {} products", textProducts.size()); + + return textProducts; +} + +std::vector> +IemApiProvider::ProcessTextProductFiles( + std::vector>& asyncResponses) +{ + std::vector> textProductFiles; + + for (auto& asyncResponse : asyncResponses) + { + auto response = asyncResponse.second.get(); + + if (response.status_code == cpr::status::HTTP_OK) + { + // Load file + auto& productId = asyncResponse.first; + const std::shared_ptr textProductFile { + std::make_shared()}; + std::istringstream responseBody {response.text}; + if (textProductFile->LoadData(productId, responseBody)) + { + textProductFiles.push_back(textProductFile); + } + } + else + { + logger_->warn("Could not load text product: {} ({})", + asyncResponse.first, + response.status_line); + } + } + + logger_->debug("Loaded {} text products", textProductFiles.size()); + + return textProductFiles; +} + +} // namespace scwx::provider diff --git a/wxdata/source/scwx/provider/warnings_provider.cpp b/wxdata/source/scwx/provider/warnings_provider.cpp index 8cfe9b77..78eb687c 100644 --- a/wxdata/source/scwx/provider/warnings_provider.cpp +++ b/wxdata/source/scwx/provider/warnings_provider.cpp @@ -1,9 +1,19 @@ -#include -#include -#include +// Prevent redefinition of __cpp_lib_format +#if defined(_MSC_VER) +# include +#endif -#include -#include +// Enable chrono formatters +#ifndef __cpp_lib_format +// NOLINTNEXTLINE(bugprone-reserved-identifier, cppcoreguidelines-macro-usage) +# define __cpp_lib_format 202110L +#endif + +#include +#include +#include + +#include #if defined(_MSC_VER) # pragma warning(push, 0) @@ -11,8 +21,6 @@ #define LIBXML_HTML_ENABLED #include -#include -#include #if (__cpp_lib_chrono < 201907L) # include @@ -22,9 +30,7 @@ # pragma warning(pop) #endif -namespace scwx -{ -namespace provider +namespace scwx::provider { static const std::string logPrefix_ = "scwx::provider::warnings_provider"; @@ -35,25 +41,36 @@ class WarningsProvider::Impl public: struct FileInfoRecord { - std::chrono::system_clock::time_point startTime_ {}; - std::chrono::system_clock::time_point lastModified_ {}; - size_t size_ {}; - bool updated_ {}; + FileInfoRecord(std::string contentLength, std::string lastModified) : + contentLengthStr_ {std::move(contentLength)}, + lastModifiedStr_ {std::move(lastModified)} + { + } + + std::string contentLengthStr_ {}; + std::string lastModifiedStr_ {}; }; - typedef std::map WarningFileMap; + using WarningFileMap = std::map; - explicit Impl(const std::string& baseUrl) : - baseUrl_ {baseUrl}, files_ {}, filesMutex_ {} + explicit Impl(std::string baseUrl) : + baseUrl_ {std::move(baseUrl)}, files_ {}, filesMutex_ {} { } - ~Impl() {} + ~Impl() = default; + Impl(const Impl&) = delete; + Impl& operator=(const Impl&) = delete; + Impl(const Impl&&) = delete; + Impl& operator=(const Impl&&) = delete; + + bool UpdateFileRecord(const cpr::Response& response, + const std::string& filename); std::string baseUrl_; - WarningFileMap files_; - std::shared_mutex filesMutex_; + WarningFileMap files_; + std::mutex filesMutex_; }; WarningsProvider::WarningsProvider(const std::string& baseUrl) : @@ -66,145 +83,177 @@ WarningsProvider::WarningsProvider(WarningsProvider&&) noexcept = default; WarningsProvider& WarningsProvider::operator=(WarningsProvider&&) noexcept = default; -std::pair -WarningsProvider::ListFiles(std::chrono::system_clock::time_point newerThan) +std::vector> +WarningsProvider::LoadUpdatedFiles( + std::chrono::sys_time startTime) { using namespace std::chrono; -#if (__cpp_lib_chrono < 201907L) +#if (__cpp_lib_chrono >= 201907L) + namespace df = std; + + static constexpr std::string_view kDateTimeFormat { + "warnings_{:%Y%m%d_%H}.txt"}; +#else using namespace date; + namespace df = date; + +# define kDateTimeFormat "warnings_%Y%m%d_%H.txt" #endif - static constexpr LazyRE2 reWarningsFilename = { - "warnings_[0-9]{8}_[0-9]{2}.txt"}; - static const std::string dateTimeFormat {"warnings_%Y%m%d_%H.txt"}; - - logger_->trace("Listing files"); - - size_t updatedObjects = 0; - size_t totalObjects = 0; - - // Perform a directory listing - auto records = network::DirList(p->baseUrl_); - - // Sort records by filename - std::sort(records.begin(), - records.end(), - [](auto& a, auto& b) { return a.filename_ < b.filename_; }); - - // Filter warning records - auto warningRecords = - records | - std::views::filter( - [](auto& record) - { - return record.type_ == std::filesystem::file_type::regular && - RE2::FullMatch(record.filename_, *reWarningsFilename); - }); - - std::unique_lock lock(p->filesMutex_); - - Impl::WarningFileMap warningFileMap; - - // Store records - for (auto& record : warningRecords) - { - // Determine start time - std::chrono::sys_time startTime; - std::istringstream ssFilename {record.filename_}; - - ssFilename >> parse(dateTimeFormat, startTime); - - // If start time is valid - if (!ssFilename.fail()) - { - // Determine if the record should be marked updated - bool updated = true; - auto it = p->files_.find(record.filename_); - if (it != p->files_.cend()) - { - auto& existingRecord = it->second; - - updated = existingRecord.updated_ || - record.size_ != existingRecord.size_ || - record.mtime_ != existingRecord.lastModified_; - } - - // Update object counts, but only if newer than threshold - if (newerThan < startTime) - { - if (updated) - { - ++updatedObjects; - } - ++totalObjects; - } - - // Store record - warningFileMap.emplace( - std::piecewise_construct, - std::forward_as_tuple(record.filename_), - std::forward_as_tuple( - startTime, record.mtime_, record.size_, updated)); - } - } - - p->files_ = std::move(warningFileMap); - - return std::make_pair(updatedObjects, totalObjects); -} - -std::vector> -WarningsProvider::LoadUpdatedFiles( - std::chrono::system_clock::time_point newerThan) -{ - logger_->debug("Loading updated files"); - + std::vector< + std::pair, false>>> + asyncCallbacks; std::vector> updatedFiles; - std::vector> asyncResponses; + const std::chrono::sys_time now = + std::chrono::floor(std::chrono::system_clock::now()); + std::chrono::sys_time currentHour = + (startTime != std::chrono::sys_time {}) ? + startTime : + now - std::chrono::hours {1}; - std::unique_lock lock(p->filesMutex_); + logger_->trace("Querying files newer than: {}", util::TimeString(startTime)); - // For each warning file - for (auto& record : p->files_) + while (currentHour <= now) { - // If file is updated, and time is later than the threshold - if (record.second.updated_ && newerThan < record.second.startTime_) - { - // Retrieve warning file - asyncResponses.emplace_back( - record.first, - cpr::GetAsync(cpr::Url {p->baseUrl_ + "/" + record.first})); + const std::string filename = df::format(kDateTimeFormat, currentHour); + const std::string url = p->baseUrl_ + "/" + filename; - // Clear updated flag - record.second.updated_ = false; - } + logger_->trace("HEAD request for file: {}", filename); + + asyncCallbacks.emplace_back( + filename, + cpr::HeadCallback( + [url, filename, this]( + cpr::Response headResponse) -> std::optional + { + if (headResponse.status_code == cpr::status::HTTP_OK) + { + const bool updated = + p->UpdateFileRecord(headResponse, filename); + + if (updated) + { + logger_->trace("GET request for file: {}", filename); + return cpr::GetAsync(cpr::Url {url}); + } + } + else if (headResponse.status_code != cpr::status::HTTP_NOT_FOUND) + { + logger_->warn("HEAD request for file failed: {} ({})", + url, + headResponse.status_line); + } + + return std::nullopt; + }, + cpr::Url {url})); + + // Query the next hour + currentHour += 1h; } - lock.unlock(); - - // Wait for warning files to load - for (auto& asyncResponse : asyncResponses) + for (auto& asyncCallback : asyncCallbacks) { - cpr::Response response = asyncResponse.second.get(); - if (response.status_code == cpr::status::HTTP_OK) - { - logger_->debug("Loading file: {}", asyncResponse.first); + auto& filename = asyncCallback.first; + auto& callback = asyncCallback.second; - // Load file - std::shared_ptr textProductFile { - std::make_shared()}; - std::istringstream responseBody {response.text}; - if (textProductFile->LoadData(responseBody)) + if (callback.valid()) + { + // Wait for futures to complete + callback.wait(); + auto asyncResponse = callback.get(); + + if (asyncResponse.has_value()) { - updatedFiles.push_back(textProductFile); + auto response = asyncResponse.value().get(); + + if (response.status_code == cpr::status::HTTP_OK) + { + logger_->debug("Loading file: {}", filename); + + // Load file + const std::shared_ptr textProductFile { + std::make_shared()}; + std::istringstream responseBody {response.text}; + if (textProductFile->LoadData(filename, responseBody)) + { + updatedFiles.push_back(textProductFile); + } + } + else + { + logger_->warn("Could not load file: {} ({})", + filename, + response.status_line); + } } } + else + { + logger_->error("Invalid future state"); + } } return updatedFiles; } -} // namespace provider -} // namespace scwx +bool WarningsProvider::Impl::UpdateFileRecord(const cpr::Response& response, + const std::string& filename) +{ + bool updated = false; + + auto contentLengthIt = response.header.find("Content-Length"); + auto lastModifiedIt = response.header.find("Last-Modified"); + + std::string contentLength {}; + std::string lastModified {}; + + if (contentLengthIt != response.header.cend()) + { + contentLength = contentLengthIt->second; + } + if (lastModifiedIt != response.header.cend()) + { + lastModified = lastModifiedIt->second; + } + + const std::unique_lock lock(filesMutex_); + + auto it = files_.find(filename); + if (it != files_.cend()) + { + auto& existingRecord = it->second; + + // If the size or last modified changes, request an update + + if (!contentLength.empty() && + contentLength != existingRecord.contentLengthStr_) + { + // Size changed + existingRecord.contentLengthStr_ = contentLengthIt->second; + updated = true; + } + else if (!lastModified.empty() && + lastModified != existingRecord.lastModifiedStr_) + { + // Last modified changed + existingRecord.lastModifiedStr_ = lastModifiedIt->second; + updated = true; + } + } + else + { + // File not found + files_.emplace(std::piecewise_construct, + std::forward_as_tuple(filename), + std::forward_as_tuple(contentLength, lastModified)); + updated = true; + } + + return updated; +} + +} // namespace scwx::provider diff --git a/wxdata/source/scwx/types/iem_types.cpp b/wxdata/source/scwx/types/iem_types.cpp new file mode 100644 index 00000000..ed3884d3 --- /dev/null +++ b/wxdata/source/scwx/types/iem_types.cpp @@ -0,0 +1,111 @@ +#include + +#include + +namespace scwx::types::iem +{ + +AfosEntry tag_invoke(boost::json::value_to_tag, + const boost::json::value& jv) +{ + auto& jo = jv.as_object(); + + AfosEntry entry {}; + + // Required parameters + entry.index_ = jo.at("index").as_int64(); + entry.entered_ = jo.at("entered").as_string(); + entry.pil_ = jo.at("pil").as_string(); + entry.productId_ = jo.at("product_id").as_string(); + entry.cccc_ = jo.at("cccc").as_string(); + entry.count_ = jo.at("count").as_int64(); + entry.link_ = jo.at("link").as_string(); + entry.textLink_ = jo.at("text_link").as_string(); + + return entry; +} + +AfosList tag_invoke(boost::json::value_to_tag, + const boost::json::value& jv) +{ + auto& jo = jv.as_object(); + + AfosList list {}; + + // Required parameters + list.data_ = boost::json::value_to>(jo.at("data")); + + return list; +} + +BadRequest tag_invoke(boost::json::value_to_tag, + const boost::json::value& jv) +{ + auto& jo = jv.as_object(); + + BadRequest badRequest {}; + + // Required parameters + badRequest.detail_ = jo.at("detail").as_string(); + + return badRequest; +} + +ValidationError::Detail::Context +tag_invoke(boost::json::value_to_tag, + const boost::json::value& jv) +{ + auto& jo = jv.as_object(); + + ValidationError::Detail::Context ctx {}; + + // Required parameters + ctx.error_ = jo.at("error").as_string(); + + return ctx; +} + +ValidationError::Detail +tag_invoke(boost::json::value_to_tag, + const boost::json::value& jv) +{ + auto& jo = jv.as_object(); + + ValidationError::Detail detail {}; + + // Required parameters + detail.type_ = jo.at("type").as_string(); + detail.loc_ = boost::json::value_to< + std::vector>>(jo.at("loc")); + detail.msg_ = jo.at("msg").as_string(); + + // Optional parameters + if (jo.contains("input")) + { + detail.input_ = jo.at("input").as_string(); + } + + if (jo.contains("ctx")) + { + detail.ctx_ = + boost::json::value_to(jo.at("ctx")); + } + + return detail; +} + +ValidationError tag_invoke(boost::json::value_to_tag, + const boost::json::value& jv) +{ + auto& jo = jv.as_object(); + + ValidationError error {}; + + // Required parameters + error.detail_ = boost::json::value_to>( + jo.at("detail")); + + return error; +} + +} // namespace scwx::types::iem diff --git a/wxdata/source/scwx/util/json.cpp b/wxdata/source/scwx/util/json.cpp new file mode 100644 index 00000000..d5873758 --- /dev/null +++ b/wxdata/source/scwx/util/json.cpp @@ -0,0 +1,201 @@ +#include +#include + +#include + +#include +#include + +namespace scwx::util::json +{ + +static const std::string logPrefix_ = "scwx::util::json"; +static const auto logger_ = scwx::util::Logger::Create(logPrefix_); + +/* Adapted from: + * https://www.boost.org/doc/libs/1_77_0/libs/json/doc/html/json/examples.html#json.examples.pretty + * + * Copyright (c) 2019, 2020 Vinnie Falco + * Copyright (c) 2020 Krystian Stasiowski + * Distributed under the Boost Software License, Version 1.0. (See + * http://www.boost.org/LICENSE_1_0.txt) + */ +static void PrettyPrintJson(std::ostream& os, + boost::json::value const& jv, + std::string* indent = nullptr); + +boost::json::value ReadJsonFile(const std::string& path) +{ + boost::json::value json; + + std::ifstream ifs {path}; + json = ReadJsonStream(ifs); + + return json; +} + +boost::json::value ReadJsonStream(std::istream& is) +{ + std::string line; + + boost::json::stream_parser p; + boost::system::error_code ec; + + while (std::getline(is, line)) + { + p.write(line, ec); + if (ec) + { + logger_->warn("{}", ec.message()); + return nullptr; + } + } + + p.finish(ec); + if (ec) + { + logger_->warn("{}", ec.message()); + return nullptr; + } + + return p.release(); +} + +boost::json::value ReadJsonString(std::string_view sv) +{ + boost::json::stream_parser p; + boost::system::error_code ec; + + p.write(sv, ec); + if (ec) + { + logger_->warn("{}", ec.message()); + return nullptr; + } + + p.finish(ec); + if (ec) + { + logger_->warn("{}", ec.message()); + return nullptr; + } + + return p.release(); +} + +void WriteJsonFile(const std::string& path, + const boost::json::value& json, + bool prettyPrint) +{ + std::ofstream ofs {path}; + + if (!ofs.is_open()) + { + logger_->warn("Cannot write JSON file: \"{}\"", path); + } + else + { + if (prettyPrint) + { + PrettyPrintJson(ofs, json); + } + else + { + ofs << json; + } + ofs.close(); + } +} + +// Allow recursion within the pretty print function +// NOLINTNEXTLINE(misc-no-recursion) +static void PrettyPrintJson(std::ostream& os, + boost::json::value const& jv, + std::string* indent) +{ + std::string indent_; + if (!indent) + indent = &indent_; + switch (jv.kind()) + { + case boost::json::kind::object: + { + os << "{\n"; + indent->append(4, ' '); + auto const& obj = jv.get_object(); + if (!obj.empty()) + { + auto it = obj.begin(); + for (;;) + { + os << *indent << boost::json::serialize(it->key()) << " : "; + PrettyPrintJson(os, it->value(), indent); + if (++it == obj.end()) + break; + os << ",\n"; + } + } + os << "\n"; + indent->resize(indent->size() - 4); + os << *indent << "}"; + break; + } + + case boost::json::kind::array: + { + os << "[\n"; + indent->append(4, ' '); + auto const& arr = jv.get_array(); + if (!arr.empty()) + { + auto it = arr.begin(); + for (;;) + { + os << *indent; + PrettyPrintJson(os, *it, indent); + if (++it == arr.end()) + break; + os << ",\n"; + } + } + os << "\n"; + indent->resize(indent->size() - 4); + os << *indent << "]"; + break; + } + + case boost::json::kind::string: + { + os << boost::json::serialize(jv.get_string()); + break; + } + + case boost::json::kind::uint64: + os << jv.get_uint64(); + break; + + case boost::json::kind::int64: + os << jv.get_int64(); + break; + + case boost::json::kind::double_: + os << jv.get_double(); + break; + + case boost::json::kind::bool_: + if (jv.get_bool()) + os << "true"; + else + os << "false"; + break; + + case boost::json::kind::null: + os << "null"; + break; + } + + if (indent->empty()) + os << "\n"; +} + +} // namespace scwx::util::json diff --git a/wxdata/source/scwx/util/streams.cpp b/wxdata/source/scwx/util/streams.cpp index 9e094f9b..6374ed35 100644 --- a/wxdata/source/scwx/util/streams.cpp +++ b/wxdata/source/scwx/util/streams.cpp @@ -1,8 +1,7 @@ #include +#include -namespace scwx -{ -namespace util +namespace scwx::util { std::istream& getline(std::istream& is, std::string& t) @@ -17,7 +16,8 @@ std::istream& getline(std::istream& is, std::string& t) int c = sb->sbumpc(); switch (c) { - case '\n': return is; + case '\n': + return is; case '\r': while (sb->sgetc() == '\r') @@ -30,6 +30,10 @@ std::istream& getline(std::istream& is, std::string& t) } return is; + case common::Characters::ETX: + sb->sungetc(); + return is; + case std::streambuf::traits_type::eof(): if (t.empty()) { @@ -37,10 +41,10 @@ std::istream& getline(std::istream& is, std::string& t) } return is; - default: t += static_cast(c); + default: + t += static_cast(c); } } } -} // namespace util -} // namespace scwx +} // namespace scwx::util diff --git a/wxdata/source/scwx/util/time.cpp b/wxdata/source/scwx/util/time.cpp index 7a706224..563aea1b 100644 --- a/wxdata/source/scwx/util/time.cpp +++ b/wxdata/source/scwx/util/time.cpp @@ -21,9 +21,7 @@ # include #endif -namespace scwx -{ -namespace util +namespace scwx::util { static const std::string logPrefix_ = "scwx::util::time"; @@ -48,6 +46,7 @@ std::chrono::system_clock::time_point TimePoint(uint32_t modifiedJulianDate, using sys_days = time_point; constexpr auto epoch = sys_days {1969y / December / 31d}; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers): literals are used return epoch + (modifiedJulianDate * 24h) + std::chrono::milliseconds {milliseconds}; } @@ -60,15 +59,19 @@ std::string TimeString(std::chrono::system_clock::time_point time, using namespace std::chrono; #if (__cpp_lib_chrono >= 201907L) -# define FORMAT_STRING_24_HOUR "{:%Y-%m-%d %H:%M:%S %Z}" -# define FORMAT_STRING_12_HOUR "{:%Y-%m-%d %I:%M:%S %p %Z}" namespace date = std::chrono; namespace df = std; + + static constexpr std::string_view kFormatString24Hour = + "{:%Y-%m-%d %H:%M:%S %Z}"; + static constexpr std::string_view kFormatString12Hour = + "{:%Y-%m-%d %I:%M:%S %p %Z}"; #else -# define FORMAT_STRING_24_HOUR "%Y-%m-%d %H:%M:%S %Z" -# define FORMAT_STRING_12_HOUR "%Y-%m-%d %I:%M:%S %p %Z" using namespace date; namespace df = date; + +# define kFormatString24Hour "%Y-%m-%d %H:%M:%S %Z" +# define kFormatString12Hour "%Y-%m-%d %I:%M:%S %p %Z" #endif auto timeInSeconds = time_point_cast(time); @@ -84,11 +87,11 @@ std::string TimeString(std::chrono::system_clock::time_point time, if (clockFormat == ClockFormat::_24Hour) { - os << df::format(FORMAT_STRING_24_HOUR, zt); + os << df::format(kFormatString24Hour, zt); } else { - os << df::format(FORMAT_STRING_12_HOUR, zt); + os << df::format(kFormatString12Hour, zt); } } catch (const std::exception& ex) @@ -110,11 +113,11 @@ std::string TimeString(std::chrono::system_clock::time_point time, { if (clockFormat == ClockFormat::_24Hour) { - os << df::format(FORMAT_STRING_24_HOUR, timeInSeconds); + os << df::format(kFormatString24Hour, timeInSeconds); } else { - os << df::format(FORMAT_STRING_12_HOUR, timeInSeconds); + os << df::format(kFormatString12Hour, timeInSeconds); } } } @@ -150,5 +153,4 @@ template std::optional> TryParseDateTime(const std::string& dateTimeFormat, const std::string& str); -} // namespace util -} // namespace scwx +} // namespace scwx::util diff --git a/wxdata/wxdata.cmake b/wxdata/wxdata.cmake index 94b0e3a7..8d2e15b4 100644 --- a/wxdata/wxdata.cmake +++ b/wxdata/wxdata.cmake @@ -6,6 +6,7 @@ find_package(Boost) find_package(cpr) find_package(LibXml2) find_package(OpenSSL) +find_package(range-v3) find_package(re2) find_package(spdlog) @@ -61,21 +62,27 @@ set(SRC_NETWORK source/scwx/network/cpr.cpp set(HDR_PROVIDER include/scwx/provider/aws_level2_data_provider.hpp include/scwx/provider/aws_level3_data_provider.hpp include/scwx/provider/aws_nexrad_data_provider.hpp + include/scwx/provider/iem_api_provider.hpp + include/scwx/provider/iem_api_provider.ipp include/scwx/provider/nexrad_data_provider.hpp include/scwx/provider/nexrad_data_provider_factory.hpp include/scwx/provider/warnings_provider.hpp) set(SRC_PROVIDER source/scwx/provider/aws_level2_data_provider.cpp source/scwx/provider/aws_level3_data_provider.cpp source/scwx/provider/aws_nexrad_data_provider.cpp + source/scwx/provider/iem_api_provider.cpp source/scwx/provider/nexrad_data_provider.cpp source/scwx/provider/nexrad_data_provider_factory.cpp source/scwx/provider/warnings_provider.cpp) +set(HDR_TYPES include/scwx/types/iem_types.hpp) +set(SRC_TYPES source/scwx/types/iem_types.cpp) set(HDR_UTIL include/scwx/util/digest.hpp include/scwx/util/enum.hpp include/scwx/util/environment.hpp include/scwx/util/float.hpp include/scwx/util/hash.hpp include/scwx/util/iterator.hpp + include/scwx/util/json.hpp include/scwx/util/logger.hpp include/scwx/util/map.hpp include/scwx/util/rangebuf.hpp @@ -88,6 +95,7 @@ set(SRC_UTIL source/scwx/util/digest.cpp source/scwx/util/environment.cpp source/scwx/util/float.cpp source/scwx/util/hash.cpp + source/scwx/util/json.cpp source/scwx/util/logger.cpp source/scwx/util/rangebuf.cpp source/scwx/util/streams.cpp @@ -224,6 +232,8 @@ add_library(wxdata OBJECT ${HDR_AWIPS} ${SRC_NETWORK} ${HDR_PROVIDER} ${SRC_PROVIDER} + ${HDR_TYPES} + ${SRC_TYPES} ${HDR_UTIL} ${SRC_UTIL} ${HDR_WSR88D} @@ -244,6 +254,8 @@ source_group("Header Files\\network" FILES ${HDR_NETWORK}) source_group("Source Files\\network" FILES ${SRC_NETWORK}) source_group("Header Files\\provider" FILES ${HDR_PROVIDER}) source_group("Source Files\\provider" FILES ${SRC_PROVIDER}) +source_group("Header Files\\types" FILES ${HDR_TYPES}) +source_group("Source Files\\types" FILES ${SRC_TYPES}) source_group("Header Files\\util" FILES ${HDR_UTIL}) source_group("Source Files\\util" FILES ${SRC_UTIL}) source_group("Header Files\\wsr88d" FILES ${HDR_WSR88D}) @@ -293,6 +305,7 @@ target_link_libraries(wxdata PUBLIC aws-cpp-sdk-core cpr::cpr LibXml2::LibXml2 OpenSSL::Crypto + range-v3::range-v3 re2::re2 spdlog::spdlog units::units)