From f37a77a9f7222aab3e1bdb7c7d9826808145759f Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 3 May 2025 23:20:26 -0500 Subject: [PATCH] Add text event pruning - Still need to prune AlertLayer - Still need to test alerts reload after being pruned --- .../source/scwx/qt/manager/alert_manager.cpp | 6 +- .../scwx/qt/manager/text_event_manager.cpp | 114 +++++++++++++++++- .../scwx/qt/manager/text_event_manager.hpp | 5 + scwx-qt/source/scwx/qt/model/alert_model.cpp | 31 ++++- scwx-qt/source/scwx/qt/model/alert_model.hpp | 5 + .../source/scwx/qt/ui/alert_dock_widget.cpp | 5 + 6 files changed, 158 insertions(+), 8 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/alert_manager.cpp b/scwx-qt/source/scwx/qt/manager/alert_manager.cpp index 757754a9..748e0943 100644 --- a/scwx-qt/source/scwx/qt/manager/alert_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/alert_manager.cpp @@ -138,8 +138,10 @@ common::Coordinate AlertManager::Impl::CurrentCoordinate( void AlertManager::Impl::HandleAlert(const types::TextEventKey& key, size_t messageIndex) const { + auto messages = textEventManager_->message_list(key); + // Skip alert if there are more messages to be processed - if (messageIndex + 1 < textEventManager_->message_count(key)) + if (messages.empty() || messageIndex + 1 < messages.size()) { return; } @@ -153,7 +155,7 @@ void AlertManager::Impl::HandleAlert(const types::TextEventKey& key, audioSettings.alert_radius().GetValue()); std::string alertWFO = audioSettings.alert_wfo().GetValue(); - auto message = textEventManager_->message_list(key).at(messageIndex); + auto message = messages.at(messageIndex); for (auto& segment : message->segments()) { diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp index c6da90f7..70505d44 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.cpp @@ -116,13 +116,14 @@ public: Impl(const Impl&&) = delete; Impl& operator=(const Impl&&) = delete; - void - HandleMessage(const std::shared_ptr& message); + void HandleMessage(const std::shared_ptr& message, + bool archiveEvent = false); template requires std::same_as, std::chrono::sys_days> void ListArchives(DateRange dates); void LoadArchives(std::chrono::system_clock::time_point dateTime); + void PruneArchives(); void RefreshAsync(); void Refresh(); template @@ -155,6 +156,15 @@ public: std::mutex archiveMutex_ {}; std::list archiveDates_ {}; + std::mutex archiveEventKeyMutex_ {}; + std::map>> + archiveEventKeys_ {}; + std::unordered_set> + liveEventKeys_ {}; + std::mutex unloadedProductMapMutex_ {}; std::map> @@ -249,7 +259,7 @@ void TextEventManager::SelectTime( const auto today = std::chrono::floor(dateTime); const auto yesterday = today - std::chrono::days {1}; const auto tomorrow = today + std::chrono::days {1}; - const auto dateArray = std::array {today, yesterday, tomorrow}; + const auto dateArray = std::array {yesterday, today, tomorrow}; const auto dates = dateArray | @@ -265,6 +275,7 @@ void TextEventManager::SelectTime( p->UpdateArchiveDates(dates); p->ListArchives(dates); p->LoadArchives(dateTime); + p->PruneArchives(); } catch (const std::exception& ex) { @@ -274,7 +285,7 @@ void TextEventManager::SelectTime( } void TextEventManager::Impl::HandleMessage( - const std::shared_ptr& message) + const std::shared_ptr& message, bool archiveEvent) { using namespace std::chrono_literals; @@ -335,6 +346,12 @@ void TextEventManager::Impl::HandleMessage( textEventMap_.emplace(key, std::vector {message}); messageIndex = 0; updated = true; + + if (!archiveEvent) + { + // Add the Text Event Key to the list of live events to prevent pruning + liveEventKeys_.insert(key); + } } else if (std::find_if(it->second.cbegin(), it->second.cend(), @@ -368,6 +385,17 @@ void TextEventManager::Impl::HandleMessage( updated = true; }; + // If this is an archive event, and the key does not exist in the live events + // Assumption: A live event will always be loaded before a duplicate archive + // event + if (archiveEvent && !liveEventKeys_.contains(key)) + { + // Add the Text Event Key to the current date's archive + const std::unique_lock archiveEventKeyLock {archiveEventKeyMutex_}; + auto& archiveKeys = archiveEventKeys_[wmoDate]; + archiveKeys.insert(key); + } + lock.unlock(); if (updated) @@ -518,11 +546,87 @@ void TextEventManager::Impl::LoadArchives( for (auto& message : messages) { - HandleMessage(message); + HandleMessage(message, true); } } } +void TextEventManager::Impl::PruneArchives() +{ + static constexpr std::size_t kMaxArchiveDates_ = 5; + + std::unordered_set> + eventKeysToKeep {}; + std::unordered_set> + eventKeysToPrune {}; + + // Remove oldest dates from the archive + while (archiveDates_.size() > kMaxArchiveDates_) + { + archiveDates_.pop_front(); + } + + const std::unique_lock archiveEventKeyLock {archiveEventKeyMutex_}; + + // If there are the same number of dates in both archiveEventKeys_ and + // archiveDates_, there is nothing to prune + if (archiveEventKeys_.size() == archiveDates_.size()) + { + // Nothing to prune + return; + } + + const std::unique_lock unloadedProductMapLock {unloadedProductMapMutex_}; + + for (auto it = archiveEventKeys_.begin(); it != archiveEventKeys_.end();) + { + const auto& date = it->first; + const auto& eventKeys = it->second; + + // If date is not in recent days map + if (std::find(archiveDates_.cbegin(), archiveDates_.cend(), date) == + archiveDates_.cend()) + { + // Prune these keys (unless they are in the eventKeysToKeep set) + eventKeysToPrune.insert(eventKeys.begin(), eventKeys.end()); + + // The date is not in the list of recent dates, remove it + it = archiveEventKeys_.erase(it); + unloadedProductMap_.erase(date); + } + else + { + // Make sure these keys don't get pruned + eventKeysToKeep.insert(eventKeys.begin(), eventKeys.end()); + + // The date is recent, keep it + ++it; + } + } + + // Remove elements from eventKeysToPrune if they are in eventKeysToKeep + for (const auto& eventKey : eventKeysToKeep) + { + eventKeysToPrune.erase(eventKey); + } + + // Remove eventKeysToPrune from textEventMap + for (const auto& eventKey : eventKeysToPrune) + { + textEventMap_.erase(eventKey); + } + + // If event keys were pruned, emit a signal + if (!eventKeysToPrune.empty()) + { + logger_->debug("Pruned {} archive events", eventKeysToPrune.size()); + + Q_EMIT self_->AlertsRemoved(eventKeysToPrune); + } +} + void TextEventManager::Impl::RefreshAsync() { boost::asio::post(threadPool_, diff --git a/scwx-qt/source/scwx/qt/manager/text_event_manager.hpp b/scwx-qt/source/scwx/qt/manager/text_event_manager.hpp index 312dece0..61affe6c 100644 --- a/scwx-qt/source/scwx/qt/manager/text_event_manager.hpp +++ b/scwx-qt/source/scwx/qt/manager/text_event_manager.hpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -35,6 +36,10 @@ public: static std::shared_ptr Instance(); signals: + void AlertsRemoved( + const std::unordered_set>& + keys); void AlertUpdated(const types::TextEventKey& key, std::size_t messageIndex, boost::uuids::uuid uuid); diff --git a/scwx-qt/source/scwx/qt/model/alert_model.cpp b/scwx-qt/source/scwx/qt/model/alert_model.cpp index 516dbfae..9baaf211 100644 --- a/scwx-qt/source/scwx/qt/model/alert_model.cpp +++ b/scwx-qt/source/scwx/qt/model/alert_model.cpp @@ -338,7 +338,7 @@ void AlertModel::HandleAlert(const types::TextEventKey& alertKey, auto alertMessages = p->textEventManager_->message_list(alertKey); // Skip alert if this is not the most recent message - if (messageIndex + 1 < alertMessages.size()) + if (alertMessages.empty() || messageIndex + 1 < alertMessages.size()) { return; } @@ -393,6 +393,35 @@ void AlertModel::HandleAlert(const types::TextEventKey& alertKey, } } +void AlertModel::HandleAlertsRemoved( + const std::unordered_set>& + alertKeys) +{ + logger_->trace("Handle alerts removed"); + + for (const auto& alertKey : alertKeys) + { + // Remove from the list of text event keys + auto it = std::find( + p->textEventKeys_.begin(), p->textEventKeys_.end(), alertKey); + if (it != p->textEventKeys_.end()) + { + int row = std::distance(p->textEventKeys_.begin(), it); + beginRemoveRows(QModelIndex(), row, row); + p->textEventKeys_.erase(it); + endRemoveRows(); + } + + // Remove from internal maps + p->observedMap_.erase(alertKey); + p->threatCategoryMap_.erase(alertKey); + p->tornadoPossibleMap_.erase(alertKey); + p->centroidMap_.erase(alertKey); + p->distanceMap_.erase(alertKey); + } +} + void AlertModel::HandleMapUpdate(double latitude, double longitude) { logger_->trace("Handle map update: {}, {}", latitude, longitude); diff --git a/scwx-qt/source/scwx/qt/model/alert_model.hpp b/scwx-qt/source/scwx/qt/model/alert_model.hpp index df6d561e..02781b6b 100644 --- a/scwx-qt/source/scwx/qt/model/alert_model.hpp +++ b/scwx-qt/source/scwx/qt/model/alert_model.hpp @@ -4,6 +4,7 @@ #include #include +#include #include @@ -51,6 +52,10 @@ public: public slots: void HandleAlert(const types::TextEventKey& alertKey, size_t messageIndex); + void HandleAlertsRemoved( + const std::unordered_set>& + alertKeys); void HandleMapUpdate(double latitude, double longitude); private: diff --git a/scwx-qt/source/scwx/qt/ui/alert_dock_widget.cpp b/scwx-qt/source/scwx/qt/ui/alert_dock_widget.cpp index 61fd160a..5e22071a 100644 --- a/scwx-qt/source/scwx/qt/ui/alert_dock_widget.cpp +++ b/scwx-qt/source/scwx/qt/ui/alert_dock_widget.cpp @@ -131,6 +131,11 @@ void AlertDockWidgetImpl::ConnectSignals() &QAction::toggled, proxyModel_.get(), &model::AlertProxyModel::SetAlertActiveFilter); + connect(textEventManager_.get(), + &manager::TextEventManager::AlertsRemoved, + alertModel_.get(), + &model::AlertModel::HandleAlertsRemoved, + Qt::QueuedConnection); connect(textEventManager_.get(), &manager::TextEventManager::AlertUpdated, alertModel_.get(),