From 63585af26d8b399531708fb86287798e9959d79f Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Mon, 7 Apr 2025 18:23:29 -0400 Subject: [PATCH] Get level2 chunks and archive working together, reduce logging of level2 chunks --- .../scwx/qt/manager/radar_product_manager.cpp | 40 +++++++++++++------ .../scwx/qt/map/radar_product_layer.cpp | 5 +-- .../aws_level2_chunks_data_provider.cpp | 9 +++-- wxdata/source/scwx/wsr88d/ar2v_file.cpp | 14 +++---- 4 files changed, 43 insertions(+), 25 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp index 12cf9d61..ec54b4cd 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp @@ -68,6 +68,7 @@ static constexpr std::size_t kTimerPlaces_ {6u}; static constexpr std::chrono::seconds kFastRetryInterval_ {15}; static constexpr std::chrono::seconds kFastRetryIntervalChunks_ {3}; static constexpr std::chrono::seconds kSlowRetryInterval_ {120}; +static constexpr std::chrono::seconds kSlowRetryIntervalChunks_ {20}; static std::unordered_map> instanceMap_; @@ -774,11 +775,15 @@ void RadarProductManagerImpl::RefreshDataSync( auto [newObjects, totalObjects] = providerManager->provider_->Refresh(); - // Level2 chunked data is updated quickly and uses a fater interval + // Level2 chunked data is updated quickly and uses a faster interval const std::chrono::milliseconds fastRetryInterval = - providerManager->group_ == common::RadarProductGroup::Level2 ? + providerManager == level2ChunksProviderManager_ ? kFastRetryIntervalChunks_ : kFastRetryInterval_; + const std::chrono::milliseconds slowRetryInterval = + providerManager == level2ChunksProviderManager_ ? + kSlowRetryIntervalChunks_ : + kSlowRetryInterval_; std::chrono::milliseconds interval = fastRetryInterval; if (totalObjects > 0) @@ -798,7 +803,7 @@ void RadarProductManagerImpl::RefreshDataSync( { // If it has been at least 5 update periods since the file has // been last modified, slow the retry period - interval = kSlowRetryInterval_; + interval = slowRetryInterval; } else if (interval < std::chrono::milliseconds {fastRetryInterval}) { @@ -817,7 +822,7 @@ void RadarProductManagerImpl::RefreshDataSync( logger_->info("[{}] No data found", providerManager->name()); // If no data is found, retry at the slow retry interval - interval = kSlowRetryInterval_; + interval = slowRetryInterval; } std::unique_lock const lock(providerManager->refreshTimerMutex_); @@ -953,11 +958,13 @@ void RadarProductManagerImpl::LoadProviderData( { existingRecord = it->second.lock(); + /* if (existingRecord != nullptr) { logger_->debug( "Data previously loaded, loading from data cache"); } + */ } } @@ -1416,7 +1423,7 @@ std::shared_ptr RadarProductManagerImpl::StoreRadarProductRecord( std::shared_ptr record) { - logger_->debug("StoreRadarProductRecord()"); + //logger_->debug("StoreRadarProductRecord()"); std::shared_ptr storedRecord = nullptr; @@ -1433,11 +1440,12 @@ RadarProductManagerImpl::StoreRadarProductRecord( { storedRecord = it->second.lock(); + /* if (storedRecord != nullptr) { logger_->debug( "Level 2 product previously loaded, loading from cache"); - } + }*/ } if (storedRecord == nullptr) @@ -1520,15 +1528,23 @@ RadarProductManager::GetLevel2Data(wsr88d::rda::DataBlockType dataBlockType, std::vector elevationCuts {}; std::chrono::system_clock::time_point foundTime {}; - //TODO decide when to use chunked vs archived data. - if constexpr (true) + // See if we have this one in the chunk provider. + auto chunkFile = std::dynamic_pointer_cast( + p->level2ChunksProviderManager_->provider_->LoadObjectByTime(time)); + if (chunkFile != nullptr) { - auto currentFile = std::dynamic_pointer_cast( - p->level2ChunksProviderManager_->provider_->LoadLatestObject()); std::tie(radarData, elevationCut, elevationCuts) = - currentFile->GetElevationScan(dataBlockType, elevation, time); + chunkFile->GetElevationScan(dataBlockType, elevation, time); + + if (radarData != nullptr) + { + auto& radarData0 = (*radarData)[0]; + foundTime = std::chrono::floor( + scwx::util::TimePoint(radarData0->modified_julian_date(), + radarData0->collection_time())); + } } - else + else // It is not in the chunk provider, so get it from the archive { auto records = p->GetLevel2ProductRecords(time); for (auto& recordPair : records) diff --git a/scwx-qt/source/scwx/qt/map/radar_product_layer.cpp b/scwx-qt/source/scwx/qt/map/radar_product_layer.cpp index 0d2b7125..8640ba03 100644 --- a/scwx-qt/source/scwx/qt/map/radar_product_layer.cpp +++ b/scwx-qt/source/scwx/qt/map/radar_product_layer.cpp @@ -159,8 +159,6 @@ void RadarProductLayer::Initialize() void RadarProductLayer::UpdateSweep() { - logger_->debug("UpdateSweep()"); - gl::OpenGLFunctions& gl = context()->gl(); boost::timer::cpu_timer timer; @@ -172,9 +170,10 @@ void RadarProductLayer::UpdateSweep() std::try_to_lock); if (!sweepLock.owns_lock()) { - logger_->debug("Sweep locked, deferring update"); + //logger_->debug("Sweep locked, deferring update"); return; } + logger_->debug("UpdateSweep()"); p->sweepNeedsUpdate_ = false; diff --git a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp index 0b18113f..02fdbca8 100644 --- a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp @@ -618,10 +618,13 @@ AwsLevel2ChunksDataProvider::LoadObjectByTime( std::chrono::system_clock::time_point time) { std::unique_lock lock(p->scansMutex_); + static const std::chrono::system_clock::time_point epoch {}; - if (p->currentScan_.valid_ && time >= p->currentScan_.time_) + if (p->currentScan_.valid_ && + (time == epoch || time >= p->currentScan_.time_)) { - return p->currentScan_.nexradFile_; + return std::make_shared(p->currentScan_.nexradFile_, + p->lastScan_.nexradFile_); } else if (p->lastScan_.valid_ && time >= p->lastScan_.time_) { @@ -629,7 +632,6 @@ AwsLevel2ChunksDataProvider::LoadObjectByTime( } else { - logger_->warn("Could not find scan with time"); return nullptr; } } @@ -637,6 +639,7 @@ AwsLevel2ChunksDataProvider::LoadObjectByTime( std::shared_ptr AwsLevel2ChunksDataProvider::LoadLatestObject() { + std::unique_lock lock(p->scansMutex_); return std::make_shared(p->currentScan_.nexradFile_, p->lastScan_.nexradFile_); //return p->currentScan_.nexradFile_; diff --git a/wxdata/source/scwx/wsr88d/ar2v_file.cpp b/wxdata/source/scwx/wsr88d/ar2v_file.cpp index 7e7e14e9..250341a0 100644 --- a/wxdata/source/scwx/wsr88d/ar2v_file.cpp +++ b/wxdata/source/scwx/wsr88d/ar2v_file.cpp @@ -138,7 +138,7 @@ Ar2vFile::GetElevationScan(rda::DataBlockType dataBlockType, float elevation, std::chrono::system_clock::time_point time) const { - logger_->debug("GetElevationScan: {} degrees", elevation); + //logger_->debug("GetElevationScan: {} degrees", elevation); std::shared_ptr elevationScan = nullptr; float elevationCut = 0.0f; @@ -273,7 +273,7 @@ bool Ar2vFile::LoadData(std::istream& is) std::size_t Ar2vFileImpl::DecompressLDMRecords(std::istream& is) { - logger_->debug("Decompressing LDM Records"); + //logger_->debug("Decompressing LDM Records"); std::size_t numRecords = 0; @@ -321,22 +321,22 @@ std::size_t Ar2vFileImpl::DecompressLDMRecords(std::istream& is) ++numRecords; } - logger_->debug("Decompressed {} LDM Records", numRecords); + //logger_->debug("Decompressed {} LDM Records", numRecords); return numRecords; } void Ar2vFileImpl::ParseLDMRecords() { - logger_->debug("Parsing LDM Records"); + //logger_->debug("Parsing LDM Records"); - std::size_t count = 0; + //std::size_t count = 0; for (auto it = rawRecords_.begin(); it != rawRecords_.end(); it++) { std::stringstream& ss = *it; - logger_->trace("Record {}", count++); + //logger_->trace("Record {}", count++); ParseLDMRecord(ss); } @@ -445,7 +445,7 @@ void Ar2vFileImpl::ProcessRadarData( void Ar2vFileImpl::IndexFile() { - logger_->debug("Indexing file"); + //logger_->debug("Indexing file"); constexpr float scaleFactor = 8.0f / 0.043945f;