From 655e9d0634ba773f29b6bdb17bb9553c12519255 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Fri, 3 Jun 2022 22:47:40 -0500 Subject: [PATCH] Load level 3 data from AWS provider --- scwx-qt/source/scwx/qt/main/main_window.cpp | 13 +- .../scwx/qt/manager/radar_product_manager.cpp | 267 +++++++++++++----- .../scwx/qt/manager/radar_product_manager.hpp | 8 +- scwx-qt/source/scwx/qt/map/map_widget.cpp | 100 +++---- scwx-qt/source/scwx/qt/map/map_widget.hpp | 4 +- scwx-qt/ts/scwx_en_US.ts | 2 +- 6 files changed, 266 insertions(+), 128 deletions(-) diff --git a/scwx-qt/source/scwx/qt/main/main_window.cpp b/scwx-qt/source/scwx/qt/main/main_window.cpp index 0c990270..9f3fdc04 100644 --- a/scwx-qt/source/scwx/qt/main/main_window.cpp +++ b/scwx-qt/source/scwx/qt/main/main_window.cpp @@ -145,6 +145,16 @@ MainWindow::MainWindow(QWidget* parent) : { p->SelectRadarProduct(p->maps_.at(1), common::Level2Product::Velocity); } + if (p->maps_.size() > 2 && p->maps_.at(2) != nullptr) + { + p->maps_.at(2)->SelectRadarProduct( + common::RadarProductGroup::Level3, "N0B", 153); + } + if (p->maps_.size() > 3 && p->maps_.at(3) != nullptr) + { + p->maps_.at(3)->SelectRadarProduct( + common::RadarProductGroup::Level3, "N0G", 154); + } connect(qApp, &QApplication::focusChanged, @@ -415,7 +425,8 @@ void MainWindowImpl::SelectRadarProduct(map::MapWidget* mapWidget, UpdateRadarProductSettings(); } - mapWidget->SelectRadarProduct(product); + mapWidget->SelectRadarProduct( + common::RadarProductGroup::Level2, productName, 0); } void MainWindowImpl::SetActiveMap(map::MapWidget* mapWidget) diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp index b303418a..a6f47b4e 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp @@ -80,29 +80,9 @@ public: } ~ProviderManager() = default; - void Disable() - { - std::unique_lock lock(refreshTimerMutex_); - refreshEnabled_ = false; - refreshTimer_.cancel(); - } + std::string name() const; - std::string name() const - { - std::string name; - - if (group_ == common::RadarProductGroup::Level3) - { - name = std::format( - "{}, {}", common::GetRadarProductGroupName(group_), product_); - } - else - { - name = common::GetRadarProductGroupName(group_); - } - - return name; - } + void Disable(); const common::RadarProductGroup group_; const std::string product_; @@ -121,13 +101,15 @@ public: coordinates0_5Degree_ {}, coordinates1Degree_ {}, level2ProductRecords_ {}, - level3ProductRecords_ {}, + level3ProductRecordsMap_ {}, level2ProductRecordMutex_ {}, level3ProductRecordMutex_ {}, - level2Data_ { - std::make_shared(common::RadarProductGroup::Level2)}, + level2ProviderManager_ {common::RadarProductGroup::Level2}, + level3ProviderManagerMap_ {}, + level3ProviderManagerMutex_ {}, initializeMutex_ {}, - loadLevel2DataMutex_ {} + loadLevel2DataMutex_ {}, + loadLevel3DataMutex_ {} { if (radarSite_ == nullptr) { @@ -135,14 +117,29 @@ public: radarSite_ = std::make_shared(); } - level2Data_->provider_ = + level2ProviderManager_.provider_ = provider::NexradDataProviderFactory::CreateLevel2DataProvider(radarId); } - ~RadarProductManagerImpl() { level2Data_->Disable(); } + ~RadarProductManagerImpl() + { + level2ProviderManager_.Disable(); + + std::shared_lock lock(level3ProviderManagerMutex_); + std::for_each(std::execution::par_unseq, + level3ProviderManagerMap_.begin(), + level3ProviderManagerMap_.end(), + [](auto& p) + { + auto& [key, providerManager] = p; + providerManager.Disable(); + }); + } RadarProductManager* self_; - void RefreshData(std::shared_ptr provider); + void EnableRefresh(RadarProductManagerImpl::ProviderManager& providerManager, + bool enabled); + void RefreshData(ProviderManager& providerManager); std::shared_ptr GetLevel2ProductRecord(std::chrono::system_clock::time_point time); @@ -152,6 +149,14 @@ public: std::shared_ptr StoreRadarProductRecord(std::shared_ptr record); + void + LoadProviderData(std::chrono::system_clock::time_point time, + RadarProductManagerImpl::ProviderManager& providerManager, + RadarProductRecordMap& recordMap, + std::shared_mutex& recordMutex, + std::mutex& loadDataMutex, + std::shared_ptr request); + static void LoadNexradFile(CreateNexradFileFunction load, std::shared_ptr request, @@ -165,16 +170,20 @@ public: std::vector coordinates0_5Degree_; std::vector coordinates1Degree_; - RadarProductRecordMap level2ProductRecords_; - std::unordered_map level3ProductRecords_; + RadarProductRecordMap level2ProductRecords_; + std::unordered_map + level3ProductRecordsMap_; std::shared_mutex level2ProductRecordMutex_; std::shared_mutex level3ProductRecordMutex_; - std::shared_ptr level2Data_; + ProviderManager level2ProviderManager_; + std::unordered_map level3ProviderManagerMap_; + std::shared_mutex level3ProviderManagerMutex_; std::mutex initializeMutex_; std::mutex loadLevel2DataMutex_; + std::mutex loadLevel3DataMutex_; }; RadarProductManager::RadarProductManager(const std::string& radarId) : @@ -183,6 +192,30 @@ RadarProductManager::RadarProductManager(const std::string& radarId) : } RadarProductManager::~RadarProductManager() = default; +std::string RadarProductManagerImpl::ProviderManager::name() const +{ + std::string name; + + if (group_ == common::RadarProductGroup::Level3) + { + name = std::format( + "{}, {}", common::GetRadarProductGroupName(group_), product_); + } + else + { + name = common::GetRadarProductGroupName(group_); + } + + return name; +} + +void RadarProductManagerImpl::ProviderManager::Disable() +{ + std::unique_lock lock(refreshTimerMutex_); + refreshEnabled_ = false; + refreshTimer_.cancel(); +} + void RadarProductManager::Cleanup() { { @@ -318,43 +351,75 @@ void RadarProductManager::Initialize() p->initialized_ = true; } -void RadarProductManager::EnableLevel2Refresh(bool enabled) +void RadarProductManager::EnableRefresh(common::RadarProductGroup group, + const std::string& product, + bool enabled) { - if (p->level2Data_->refreshEnabled_ != enabled) + if (group == common::RadarProductGroup::Level2) { - p->level2Data_->refreshEnabled_ = enabled; + p->EnableRefresh(p->level2ProviderManager_, enabled); + } + else + { + std::unique_lock lock(p->level3ProviderManagerMutex_); + + if (!p->level3ProviderManagerMap_.contains(product)) + { + p->level3ProviderManagerMap_.emplace( + std::piecewise_construct, + std::forward_as_tuple(product), + std::forward_as_tuple(common::RadarProductGroup::Level3, product)); + p->level3ProviderManagerMap_.at(product).provider_ = + provider::NexradDataProviderFactory::CreateLevel3DataProvider( + p->radarId_, product); + } + + RadarProductManagerImpl::ProviderManager& providerManager = + p->level3ProviderManagerMap_.at(product); + + lock.unlock(); + + p->EnableRefresh(providerManager, enabled); + } +} + +void RadarProductManagerImpl::EnableRefresh(ProviderManager& providerManager, + bool enabled) +{ + if (providerManager.refreshEnabled_ != enabled) + { + providerManager.refreshEnabled_ = enabled; if (enabled) { - p->RefreshData(p->level2Data_); + RefreshData(providerManager); } } } -void RadarProductManagerImpl::RefreshData( - std::shared_ptr provider) +void RadarProductManagerImpl::RefreshData(ProviderManager& providerManager) { - logger_->debug("RefreshData: {}", provider->name()); + logger_->debug("RefreshData: {}", providerManager.name()); { - std::unique_lock lock(provider->refreshTimerMutex_); - provider->refreshTimer_.cancel(); + std::unique_lock lock(providerManager.refreshTimerMutex_); + providerManager.refreshTimer_.cancel(); } util::async( - [=]() + [&]() { - auto [newObjects, totalObjects] = provider->provider_->Refresh(); + auto [newObjects, totalObjects] = providerManager.provider_->Refresh(); std::chrono::milliseconds interval = kRetryInterval_; if (newObjects > 0) { - std::string key = provider->provider_->FindLatestKey(); - auto latestTime = provider->provider_->GetTimePointByKey(key); + std::string key = providerManager.provider_->FindLatestKey(); + auto latestTime = providerManager.provider_->GetTimePointByKey(key); - auto updatePeriod = provider->provider_->update_period(); - auto lastModified = provider->provider_->last_modified(); + auto updatePeriod = providerManager.provider_->update_period(); + auto lastModified = providerManager.provider_->last_modified(); interval = std::chrono::duration_cast( updatePeriod - (std::chrono::system_clock::now() - lastModified)); @@ -364,43 +429,43 @@ void RadarProductManagerImpl::RefreshData( } emit self_->NewDataAvailable( - provider->group_, provider->product_, latestTime); + providerManager.group_, providerManager.product_, latestTime); } - else if (provider->refreshEnabled_ && totalObjects == 0) + else if (providerManager.refreshEnabled_ && totalObjects == 0) { logger_->info("[{}] No data found, disabling refresh", - provider->name()); + providerManager.name()); - provider->refreshEnabled_ = false; + providerManager.refreshEnabled_ = false; } - if (provider->refreshEnabled_) + if (providerManager.refreshEnabled_) { - std::unique_lock lock(provider->refreshTimerMutex_); + std::unique_lock lock(providerManager.refreshTimerMutex_); logger_->debug( "[{}] Scheduled refresh in {:%M:%S}", - provider->name(), + providerManager.name(), std::chrono::duration_cast(interval)); { - provider->refreshTimer_.expires_after(interval); - provider->refreshTimer_.async_wait( - [=](const boost::system::error_code& e) + providerManager.refreshTimer_.expires_after(interval); + providerManager.refreshTimer_.async_wait( + [&](const boost::system::error_code& e) { if (e == boost::system::errc::success) { - RefreshData(provider); + RefreshData(providerManager); } else if (e == boost::asio::error::operation_aborted) { logger_->debug("[{}] Data refresh timer cancelled", - provider->name()); + providerManager.name()); } else { logger_->warn("[{}] Data refresh timer error: {}", - provider->name(), + providerManager.name(), e.message()); } }); @@ -409,23 +474,30 @@ void RadarProductManagerImpl::RefreshData( }); } -void RadarProductManager::LoadLevel2Data( +void RadarProductManagerImpl::LoadProviderData( std::chrono::system_clock::time_point time, + RadarProductManagerImpl::ProviderManager& providerManager, + RadarProductRecordMap& recordMap, + std::shared_mutex& recordMutex, + std::mutex& loadDataMutex, std::shared_ptr request) { - logger_->debug("LoadLevel2Data: {}", util::TimeString(time)); + logger_->debug("LoadProviderData: {}, {}", + providerManager.name(), + util::TimeString(time)); RadarProductManagerImpl::LoadNexradFile( - [=]() -> std::shared_ptr + [=, &providerManager, &recordMap, &recordMutex, &loadDataMutex]() + -> std::shared_ptr { std::shared_ptr existingRecord = nullptr; std::shared_ptr nexradFile = nullptr; { - std::shared_lock sharedLock {p->level2ProductRecordMutex_}; + std::shared_lock sharedLock {recordMutex}; - auto it = p->level2ProductRecords_.find(time); - if (it != p->level2ProductRecords_.cend()) + auto it = recordMap.find(time); + if (it != recordMap.cend()) { logger_->debug( "Data previously loaded, loading from data cache"); @@ -436,8 +508,8 @@ void RadarProductManager::LoadLevel2Data( if (existingRecord == nullptr) { - std::string key = p->level2Data_->provider_->FindKey(time); - nexradFile = p->level2Data_->provider_->LoadObjectByKey(key); + std::string key = providerManager.provider_->FindKey(time); + nexradFile = providerManager.provider_->LoadObjectByKey(key); } else { @@ -447,7 +519,53 @@ void RadarProductManager::LoadLevel2Data( return nexradFile; }, request, - p->loadLevel2DataMutex_); + loadDataMutex); +} + +void RadarProductManager::LoadLevel2Data( + std::chrono::system_clock::time_point time, + std::shared_ptr request) +{ + logger_->debug("LoadLevel2Data: {}", util::TimeString(time)); + + p->LoadProviderData(time, + p->level2ProviderManager_, + p->level2ProductRecords_, + p->level2ProductRecordMutex_, + p->loadLevel2DataMutex_, + request); +} + +void RadarProductManager::LoadLevel3Data( + const std::string& product, + std::chrono::system_clock::time_point time, + std::shared_ptr request) +{ + logger_->debug("LoadLevel3Data: {}", util::TimeString(time)); + + // Look up provider manager + std::shared_lock providerManagerLock(p->level3ProviderManagerMutex_); + auto level3ProviderManager = p->level3ProviderManagerMap_.find(product); + if (level3ProviderManager == p->level3ProviderManagerMap_.cend()) + { + logger_->debug("No level 3 provider manager for product: {}", product); + return; + } + providerManagerLock.unlock(); + + // Look up product record + std::unique_lock productRecordLock(p->level3ProductRecordMutex_); + RadarProductRecordMap& level3ProductRecords = + p->level3ProductRecordsMap_[product]; + productRecordLock.unlock(); + + // Load provider data + p->LoadProviderData(time, + level3ProviderManager->second, + level3ProductRecords, + p->level3ProductRecordMutex_, + p->loadLevel3DataMutex_, + request); } void RadarProductManager::LoadData( @@ -564,8 +682,7 @@ RadarProductManagerImpl::GetLevel2ProductRecord( record = util::GetBoundedElementValue(level2ProductRecords_, time); // Does the record contain the time we are looking for? - if (record != nullptr && (time < record->level2_file()->start_time() || - record->level2_file()->end_time() < time)) + if (record != nullptr && (time < record->level2_file()->start_time())) { record = nullptr; } @@ -580,9 +697,11 @@ RadarProductManagerImpl::GetLevel3ProductRecord( { std::shared_ptr record = nullptr; - auto it = level3ProductRecords_.find(product); + std::unique_lock lock {level3ProductRecordMutex_}; - if (it != level3ProductRecords_.cend()) + auto it = level3ProductRecordsMap_.find(product); + + if (it != level3ProductRecordsMap_.cend()) { record = util::GetBoundedElementValue(it->second, time); } @@ -623,7 +742,7 @@ RadarProductManagerImpl::StoreRadarProductRecord( { std::unique_lock lock {level3ProductRecordMutex_}; - auto& productMap = level3ProductRecords_[record->radar_product()]; + auto& productMap = level3ProductRecordsMap_[record->radar_product()]; auto it = productMap.find(timeInSeconds); if (it != productMap.cend()) diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.hpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.hpp index d796f2b3..8d5c034c 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.hpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.hpp @@ -36,7 +36,9 @@ public: std::shared_ptr radar_site() const; void Initialize(); - void EnableLevel2Refresh(bool enabled); + void EnableRefresh(common::RadarProductGroup group, + const std::string& product, + bool enabled); std::tuple, float, @@ -55,6 +57,10 @@ public: void LoadLevel2Data( std::chrono::system_clock::time_point time, std::shared_ptr request = nullptr); + void LoadLevel3Data( + const std::string& product, + std::chrono::system_clock::time_point time, + std::shared_ptr request = nullptr); static void LoadData(std::istream& is, diff --git a/scwx-qt/source/scwx/qt/map/map_widget.cpp b/scwx-qt/source/scwx/qt/map/map_widget.cpp index 1a027b50..a712dc7e 100644 --- a/scwx-qt/source/scwx/qt/map/map_widget.cpp +++ b/scwx-qt/source/scwx/qt/map/map_widget.cpp @@ -250,22 +250,33 @@ void MapWidget::SelectElevation(float elevation) } } -void MapWidget::SelectRadarProduct(common::Level2Product product) +void MapWidget::SelectRadarProduct(common::RadarProductGroup group, + const std::string& product, + int16_t productCode) { bool radarProductViewCreated = false; std::shared_ptr& radarProductView = p->context_->radarProductView_; + std::string productName {product}; + + // Validate level 2 product, set to default if invalid + if (group == common::RadarProductGroup::Level2) + { + common::Level2Product level2Product = + p->GetLevel2ProductOrDefault(productName); + productName = common::GetLevel2Name(level2Product); + } + if (radarProductView == nullptr || - radarProductView->GetRadarProductGroup() != - common::RadarProductGroup::Level2 || - p->selectedLevel2Product_ != product) + radarProductView->GetRadarProductGroup() != group || + radarProductView->GetRadarProductName() != productName) { p->RadarProductViewDisconnect(); radarProductView = view::RadarProductViewFactory::Create( - product, p->radarProductManager_); + group, productName, productCode, p->radarProductManager_); p->RadarProductViewConnect(); @@ -273,11 +284,18 @@ void MapWidget::SelectRadarProduct(common::Level2Product product) } radarProductView->SelectTime(p->selectedTime_); - p->selectedLevel2Product_ = product; + if (group == common::RadarProductGroup::Level2) + { + p->selectedLevel2Product_ = common::GetLevel2Product(productName); + } if (radarProductViewCreated) { - p->InitializeNewRadarProductView(common::GetLevel2Palette(product)); + const std::string palette = + (group == common::RadarProductGroup::Level2) ? + common::GetLevel2Palette(common::GetLevel2Product(productName)) : + common::GetLevel3Palette(productCode); + p->InitializeNewRadarProductView(palette); } else { @@ -286,7 +304,7 @@ void MapWidget::SelectRadarProduct(common::Level2Product product) if (p->autoRefreshEnabled_) { - p->radarProductManager_->EnableLevel2Refresh(true); + p->radarProductManager_->EnableRefresh(group, productName, true); } } @@ -312,42 +330,10 @@ void MapWidget::SelectRadarProduct( product, util::TimeString(time)); - if (group == common::RadarProductGroup::Level2) - { - common::Level2Product level2Product = - p->GetLevel2ProductOrDefault(product); + p->SetRadarSite(radarId); + p->selectedTime_ = time; - p->SetRadarSite(radarId); - p->selectedTime_ = time; - - SelectRadarProduct(level2Product); - } - else - { - // TODO: Combine this with the SelectRadarProduct(Level2Product) function - std::shared_ptr radarProductManager = - manager::RadarProductManager::Instance(radarId); - std::shared_ptr radarProductView = - view::RadarProductViewFactory::Create( - group, product, productCode, radarProductManager); - - if (radarProductView == nullptr) - { - logger_->debug("No view created for product"); - return; - } - - p->RadarProductViewDisconnect(); - - p->context_->radarProductView_ = radarProductView; - p->SetRadarSite(radarId); - p->selectedTime_ = time; - radarProductView->SelectTime(p->selectedTime_); - - p->RadarProductViewConnect(); - - p->InitializeNewRadarProductView(common::GetLevel3Palette(productCode)); - } + SelectRadarProduct(group, product, productCode); } void MapWidget::SetActive(bool isActive) @@ -362,9 +348,12 @@ void MapWidget::SetAutoRefresh(bool enabled) { p->autoRefreshEnabled_ = enabled; - if (p->autoRefreshEnabled_) + if (p->autoRefreshEnabled_ && p->context_->radarProductView_ != nullptr) { - p->radarProductManager_->EnableLevel2Refresh(true); + p->radarProductManager_->EnableRefresh( + p->context_->radarProductView_->GetRadarProductGroup(), + p->context_->radarProductView_->GetRadarProductName(), + true); } } } @@ -561,7 +550,7 @@ void MapWidget::initializeGL() std::shared_ptr radarSite = p->radarProductManager_->radar_site(); p->map_->setCoordinateZoom({radarSite->latitude(), radarSite->longitude()}, - 9); + 7); p->UpdateStoredMapParameters(); QString styleUrl = qgetenv("MAPBOX_STYLE_URL"); @@ -610,8 +599,9 @@ void MapWidgetImpl::AutoRefreshConnect() std::chrono::system_clock::time_point latestTime) { if (autoRefreshEnabled_ && context_->radarProductView_ != nullptr && - group == common::RadarProductGroup::Level2 && - context_->radarProductView_->GetRadarProductGroup() == group) + context_->radarProductView_->GetRadarProductGroup() == group && + (group == common::RadarProductGroup::Level2 || + context_->radarProductView_->GetRadarProductName() == product)) { // Create file request std::shared_ptr request = @@ -634,8 +624,18 @@ void MapWidgetImpl::AutoRefreshConnect() // Load file util::async( - [=]() { - radarProductManager_->LoadLevel2Data(latestTime, request); + [=]() + { + if (group == common::RadarProductGroup::Level2) + { + radarProductManager_->LoadLevel2Data(latestTime, + request); + } + else + { + radarProductManager_->LoadLevel3Data( + product, latestTime, request); + } }); } }, diff --git a/scwx-qt/source/scwx/qt/map/map_widget.hpp b/scwx-qt/source/scwx/qt/map/map_widget.hpp index 27981045..d15e62b7 100644 --- a/scwx-qt/source/scwx/qt/map/map_widget.hpp +++ b/scwx-qt/source/scwx/qt/map/map_widget.hpp @@ -42,7 +42,9 @@ public: uint16_t GetVcp() const; void SelectElevation(float elevation); - void SelectRadarProduct(common::Level2Product product); + void SelectRadarProduct(common::RadarProductGroup group, + const std::string& product, + int16_t productCode); void SelectRadarProduct(std::shared_ptr record); void SetActive(bool isActive); void SetAutoRefresh(bool enabled); diff --git a/scwx-qt/ts/scwx_en_US.ts b/scwx-qt/ts/scwx_en_US.ts index 78eb7c94..ce5a0a17 100644 --- a/scwx-qt/ts/scwx_en_US.ts +++ b/scwx-qt/ts/scwx_en_US.ts @@ -102,7 +102,7 @@ scwx::qt::main::MainWindow - + Unrecognized NEXRAD Product: