From a754d6684492a1e1e215d37ca3d73b9659d89c4d Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Sun, 30 Mar 2025 13:18:04 -0400 Subject: [PATCH] Setting up for merging last and current scan's, and having archive and chunks --- .../scwx/qt/manager/radar_product_manager.cpp | 129 +++++++++++++++--- .../aws_level2_chunks_data_provider.hpp | 1 + .../provider/aws_nexrad_data_provider.hpp | 1 + .../scwx/provider/nexrad_data_provider.hpp | 9 ++ .../provider/nexrad_data_provider_factory.hpp | 3 + .../aws_level2_chunks_data_provider.cpp | 8 +- .../provider/aws_nexrad_data_provider.cpp | 6 + .../provider/nexrad_data_provider_factory.cpp | 9 +- 8 files changed, 143 insertions(+), 23 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp index 9bd5dbd8..0c7f10ea 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp @@ -134,6 +134,8 @@ public: level3ProductsInitialized_ {false}, radarSite_ {config::RadarSite::Get(radarId)}, level2ProviderManager_ {std::make_shared( + self_, radarId_, common::RadarProductGroup::Level2)}, + level2ChunksProviderManager_ {std::make_shared( self_, radarId_, common::RadarProductGroup::Level2)} { if (radarSite_ == nullptr) @@ -144,10 +146,14 @@ public: level2ProviderManager_->provider_ = provider::NexradDataProviderFactory::CreateLevel2DataProvider(radarId); + level2ChunksProviderManager_->provider_ = + provider::NexradDataProviderFactory::CreateLevel2ChunksDataProvider( + radarId); } ~RadarProductManagerImpl() { level2ProviderManager_->Disable(); + level2ChunksProviderManager_->Disable(); std::shared_lock lock(level3ProviderManagerMutex_); std::for_each(std::execution::par_unseq, @@ -251,6 +257,7 @@ public: std::shared_mutex level3ProductRecordMutex_ {}; std::shared_ptr level2ProviderManager_; + std::shared_ptr level2ChunksProviderManager_; std::unordered_map> level3ProviderManagerMap_ {}; std::shared_mutex level3ProviderManagerMutex_ {}; @@ -639,6 +646,7 @@ void RadarProductManager::EnableRefresh(common::RadarProductGroup group, if (group == common::RadarProductGroup::Level2) { p->EnableRefresh(uuid, p->level2ProviderManager_, enabled); + p->EnableRefresh(uuid, p->level2ChunksProviderManager_, enabled); } else { @@ -986,6 +994,12 @@ void RadarProductManager::LoadLevel2Data( p->level2ProductRecordMutex_, p->loadLevel2DataMutex_, request); + p->LoadProviderData(time, + p->level2ChunksProviderManager_, + p->level2ProductRecords_, + p->level2ProductRecordMutex_, + p->loadLevel2DataMutex_, + request); } void RadarProductManager::LoadLevel3Data( @@ -1162,6 +1176,10 @@ void RadarProductManagerImpl::PopulateLevel2ProductTimes( level2ProductRecords_, level2ProductRecordMutex_, time); + PopulateProductTimes(level2ChunksProviderManager_, + level2ProductRecords_, + level2ProductRecordMutex_, + time); } void RadarProductManagerImpl::PopulateLevel3ProductTimes( @@ -1504,35 +1522,104 @@ RadarProductManager::GetLevel2Data(wsr88d::rda::DataBlockType dataBlockType, auto records = p->GetLevel2ProductRecords(time); - for (auto& recordPair : records) + //TODO decide when to use chunked vs archived data. + if (true) { - auto& record = recordPair.second; - - if (record != nullptr) + auto currentFile = std::dynamic_pointer_cast( + p->level2ChunksProviderManager_->provider_->LoadLatestObject()); + std::shared_ptr currentRadarData = nullptr; + float currentElevationCut = 0.0f; + std::vector currentElevationCuts; + if (currentFile != nullptr) { - std::shared_ptr recordRadarData = nullptr; - float recordElevationCut = 0.0f; - std::vector recordElevationCuts; + std::tie(currentRadarData, currentElevationCut, currentElevationCuts) = + currentFile->GetElevationScan(dataBlockType, elevation, time); + } - std::tie(recordRadarData, recordElevationCut, recordElevationCuts) = - record->level2_file()->GetElevationScan( - dataBlockType, elevation, time); + std::shared_ptr lastRadarData = nullptr; + float lastElevationCut = 0.0f; + std::vector lastElevationCuts; + auto lastFile = std::dynamic_pointer_cast( + p->level2ChunksProviderManager_->provider_->LoadSecondLatestObject()); + if (lastFile != nullptr) + { + std::tie(lastRadarData, lastElevationCut, lastElevationCuts) = + lastFile->GetElevationScan(dataBlockType, elevation, time); + } - if (recordRadarData != nullptr) + if (currentRadarData != nullptr) + { + if (lastRadarData != nullptr) { - auto& radarData0 = (*recordRadarData)[0]; + auto& radarData0 = (*currentRadarData)[0]; auto collectionTime = std::chrono::floor( - scwx::util::TimePoint(radarData0->modified_julian_date(), - radarData0->collection_time())); + scwx::util::TimePoint(radarData0->modified_julian_date(), + radarData0->collection_time())); - // Find the newest radar data, not newer than the selected time - if (radarData == nullptr || - (collectionTime <= time && foundTime < collectionTime)) + // TODO merge data + radarData = currentRadarData; + elevationCut = currentElevationCut; + elevationCuts = std::move(currentElevationCuts); + foundTime = collectionTime; + } + else + { + auto& radarData0 = (*currentRadarData)[0]; + auto collectionTime = std::chrono::floor( + scwx::util::TimePoint(radarData0->modified_julian_date(), + radarData0->collection_time())); + + radarData = currentRadarData; + elevationCut = currentElevationCut; + elevationCuts = std::move(currentElevationCuts); + foundTime = collectionTime; + } + } + else if (lastRadarData != nullptr) + { + auto& radarData0 = (*lastRadarData)[0]; + auto collectionTime = std::chrono::floor( + scwx::util::TimePoint(radarData0->modified_julian_date(), + radarData0->collection_time())); + + radarData = lastRadarData; + elevationCut = lastElevationCut; + elevationCuts = std::move(lastElevationCuts); + foundTime = collectionTime; + } + } + else + { + for (auto& recordPair : records) + { + auto& record = recordPair.second; + + if (record != nullptr) + { + std::shared_ptr recordRadarData = nullptr; + float recordElevationCut = 0.0f; + std::vector recordElevationCuts; + + std::tie(recordRadarData, recordElevationCut, recordElevationCuts) = + record->level2_file()->GetElevationScan( + dataBlockType, elevation, time); + + if (recordRadarData != nullptr) { - radarData = recordRadarData; - elevationCut = recordElevationCut; - elevationCuts = std::move(recordElevationCuts); - foundTime = collectionTime; + auto& radarData0 = (*recordRadarData)[0]; + auto collectionTime = std::chrono::floor( + scwx::util::TimePoint(radarData0->modified_julian_date(), + radarData0->collection_time())); + + // Find the newest radar data, not newer than the selected time + if (radarData == nullptr || + (collectionTime <= time && foundTime < collectionTime)) + { + radarData = recordRadarData; + elevationCut = recordElevationCut; + elevationCuts = std::move(recordElevationCuts); + foundTime = collectionTime; + } } } } diff --git a/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp b/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp index 57e8e301..106b6605 100644 --- a/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp +++ b/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp @@ -51,6 +51,7 @@ public: std::shared_ptr LoadObjectByTime(std::chrono::system_clock::time_point time) override; std::shared_ptr LoadLatestObject() override; + std::shared_ptr LoadSecondLatestObject() override; std::pair Refresh() override; void RequestAvailableProducts() override; diff --git a/wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp b/wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp index cb71f27b..b2946f38 100644 --- a/wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp +++ b/wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp @@ -49,6 +49,7 @@ public: std::shared_ptr LoadObjectByTime(std::chrono::system_clock::time_point time) override; std::shared_ptr LoadLatestObject() override; + std::shared_ptr LoadSecondLatestObject() override; std::pair Refresh() override; protected: diff --git a/wxdata/include/scwx/provider/nexrad_data_provider.hpp b/wxdata/include/scwx/provider/nexrad_data_provider.hpp index 81edc1eb..a0e09f04 100644 --- a/wxdata/include/scwx/provider/nexrad_data_provider.hpp +++ b/wxdata/include/scwx/provider/nexrad_data_provider.hpp @@ -106,6 +106,15 @@ public: virtual std::shared_ptr LoadLatestObject() = 0; + /** + * Loads the second NEXRAD file object + * + * @return NEXRAD data + */ + virtual std::shared_ptr + LoadSecondLatestObject() = 0; + + /** * Lists NEXRAD objects for the current date, and adds them to the cache. If * no objects have been added to the cache for the current date, the previous diff --git a/wxdata/include/scwx/provider/nexrad_data_provider_factory.hpp b/wxdata/include/scwx/provider/nexrad_data_provider_factory.hpp index bdd51b9e..510ee14c 100644 --- a/wxdata/include/scwx/provider/nexrad_data_provider_factory.hpp +++ b/wxdata/include/scwx/provider/nexrad_data_provider_factory.hpp @@ -27,6 +27,9 @@ public: static std::shared_ptr CreateLevel2DataProvider(const std::string& radarSite); + static std::shared_ptr + CreateLevel2ChunksDataProvider(const std::string& radarSite); + static std::shared_ptr CreateLevel3DataProvider(const std::string& radarSite, const std::string& product); diff --git a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp index 3536c16c..7c692f7f 100644 --- a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp @@ -603,7 +603,13 @@ AwsLevel2ChunksDataProvider::LoadObjectByTime( std::shared_ptr AwsLevel2ChunksDataProvider::LoadLatestObject() { - return LoadObjectByTime(FindLatestTime()); + return p->currentScan_.nexradFile_; +} + +std::shared_ptr +AwsLevel2ChunksDataProvider::LoadSecondLatestObject() +{ + return p->lastScan_.nexradFile_; } int AwsLevel2ChunksDataProvider::Impl::GetScanNumber(const std::string& prefix) diff --git a/wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp b/wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp index 74740be0..c0611804 100644 --- a/wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp @@ -351,6 +351,12 @@ std::shared_ptr AwsNexradDataProvider::LoadLatestObject() return LoadObjectByKey(FindLatestKey()); } +std::shared_ptr +AwsNexradDataProvider::LoadSecondLatestObject() +{ + return nullptr; +} + std::pair AwsNexradDataProvider::Refresh() { using namespace std::chrono; diff --git a/wxdata/source/scwx/provider/nexrad_data_provider_factory.cpp b/wxdata/source/scwx/provider/nexrad_data_provider_factory.cpp index 83ccb1a3..dcaf7c9f 100644 --- a/wxdata/source/scwx/provider/nexrad_data_provider_factory.cpp +++ b/wxdata/source/scwx/provider/nexrad_data_provider_factory.cpp @@ -1,5 +1,5 @@ #include -//#include +#include #include #include @@ -14,6 +14,13 @@ static const std::string logPrefix_ = std::shared_ptr NexradDataProviderFactory::CreateLevel2DataProvider( const std::string& radarSite) +{ + return std::make_unique(radarSite); +} + +std::shared_ptr +NexradDataProviderFactory::CreateLevel2ChunksDataProvider( + const std::string& radarSite) { return std::make_unique(radarSite); }