From f4226b487de5bbc86927c01ee49f560d7291a4c5 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Sat, 30 Aug 2025 16:27:55 -0500 Subject: [PATCH] Initial moving of product listing to the background for level 3 --- scwx-qt/scwx-qt.cmake | 1 + .../scwx/qt/manager/radar_product_manager.cpp | 254 ++++++++++++++---- .../scwx/qt/manager/radar_product_manager.hpp | 25 +- .../scwx/qt/types/radar_product_types.hpp | 16 ++ .../scwx/qt/view/level2_product_view.cpp | 18 +- .../scwx/qt/view/level3_product_view.cpp | 27 +- .../scwx/qt/view/level3_radial_view.cpp | 32 +-- .../scwx/qt/view/level3_raster_view.cpp | 12 +- .../scwx/qt/view/overlay_product_view.cpp | 28 +- .../aws_level2_chunks_data_provider.hpp | 4 +- .../provider/aws_nexrad_data_provider.hpp | 27 +- .../scwx/provider/nexrad_data_provider.hpp | 29 +- .../aws_level2_chunks_data_provider.cpp | 9 +- .../provider/aws_nexrad_data_provider.cpp | 52 ++-- 14 files changed, 356 insertions(+), 178 deletions(-) create mode 100644 scwx-qt/source/scwx/qt/types/radar_product_types.hpp diff --git a/scwx-qt/scwx-qt.cmake b/scwx-qt/scwx-qt.cmake index ecd26ef9..1a093200 100644 --- a/scwx-qt/scwx-qt.cmake +++ b/scwx-qt/scwx-qt.cmake @@ -234,6 +234,7 @@ set(HDR_TYPES source/scwx/qt/types/alert_types.hpp source/scwx/qt/types/media_types.hpp source/scwx/qt/types/qt_types.hpp source/scwx/qt/types/radar_product_record.hpp + source/scwx/qt/types/radar_product_types.hpp source/scwx/qt/types/text_event_key.hpp source/scwx/qt/types/text_types.hpp source/scwx/qt/types/texture_types.hpp diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp index bda6e232..7995968a 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -36,11 +37,7 @@ # pragma warning(pop) #endif -namespace scwx -{ -namespace qt -{ -namespace manager +namespace scwx::qt::manager { static const std::string logPrefix_ = @@ -210,7 +207,8 @@ public: std::shared_ptr> GetLevel2ProductRecords(std::chrono::system_clock::time_point time); std::tuple, - std::chrono::system_clock::time_point> + std::chrono::system_clock::time_point, + types::RadarProductLoadStatus> GetLevel3ProductRecord(const std::string& product, std::chrono::system_clock::time_point time); std::shared_ptr @@ -224,15 +222,24 @@ public: std::mutex& mutex, std::chrono::system_clock::time_point time); void - LoadProviderData(std::chrono::system_clock::time_point time, - std::shared_ptr providerManager, - RadarProductRecordMap& recordMap, - std::shared_mutex& recordMutex, - std::mutex& loadDataMutex, - const std::shared_ptr& request); - void PopulateLevel2ProductTimes(std::chrono::system_clock::time_point time); + LoadProviderData(std::chrono::system_clock::time_point time, + std::shared_ptr providerManager, + RadarProductRecordMap& recordMap, + std::shared_mutex& recordMutex, + std::mutex& loadDataMutex, + const std::shared_ptr& request); + + bool AreLevel2ProductTimesPopulated( + std::chrono::system_clock::time_point time) const; + bool + AreLevel3ProductTimesPopulated(const std::string& product, + std::chrono::system_clock::time_point time); + + void PopulateLevel2ProductTimes(std::chrono::system_clock::time_point time, + bool update = true); void PopulateLevel3ProductTimes(const std::string& product, - std::chrono::system_clock::time_point time); + std::chrono::system_clock::time_point time, + bool update = true); void UpdateAvailableProductsSync(); @@ -243,11 +250,16 @@ public: const float gateRangeOffset, std::vector& outputCoordinates); + static bool AreProductTimesPopulated( + const std::shared_ptr& providerManager, + std::chrono::system_clock::time_point time); + static void PopulateProductTimes(std::shared_ptr providerManager, RadarProductRecordMap& productRecordMap, std::shared_mutex& productRecordMutex, - std::chrono::system_clock::time_point time); + std::chrono::system_clock::time_point time, + bool update); static void LoadNexradFile(CreateNexradFileFunction load, @@ -934,32 +946,32 @@ RadarProductManager::GetActiveVolumeTimes( [&](const std::shared_ptr& provider) { // For yesterday, today and tomorrow (in parallel) - std::for_each(std::execution::par, - dates.begin(), - dates.end(), - [&](const auto& date) - { - // Don't query for a time point in the future - if (date > scwx::util::time::now()) - { - return; - } + std::for_each( + std::execution::par, + dates.begin(), + dates.end(), + [&](const auto& date) + { + // Don't query for a time point in the future + if (date > scwx::util::time::now()) + { + return; + } - // Query the provider for volume time points - auto timePoints = provider->GetTimePointsByDate(date); + // Query the provider for volume time points + auto timePoints = provider->GetTimePointsByDate(date, true); - // TODO: Note, this will miss volume times present in - // Level 2 products with a second scan + // TODO: Note, this will miss volume times present in Level 2 + // products with a second scan - // Lock the merged volume time list - std::unique_lock volumeTimesLock {volumeTimesMutex}; + // Lock the merged volume time list + const std::unique_lock volumeTimesLock {volumeTimesMutex}; - // Copy time points to the merged list - std::copy( - timePoints.begin(), - timePoints.end(), - std::inserter(volumeTimes, volumeTimes.end())); - }); + // Copy time points to the merged list + std::copy(timePoints.begin(), + timePoints.end(), + std::inserter(volumeTimes, volumeTimes.end())); + }); }); // Return merged volume times list @@ -1202,21 +1214,70 @@ void RadarProductManagerImpl::LoadNexradFile( } } +bool RadarProductManagerImpl::AreLevel2ProductTimesPopulated( + std::chrono::system_clock::time_point time) const +{ + return AreProductTimesPopulated(level2ProviderManager_, time); +} + +bool RadarProductManagerImpl::AreLevel3ProductTimesPopulated( + const std::string& product, std::chrono::system_clock::time_point time) +{ + // Get provider manager + const auto level3ProviderManager = GetLevel3ProviderManager(product); + + return AreProductTimesPopulated(level3ProviderManager, time); +} + +bool RadarProductManagerImpl::AreProductTimesPopulated( + const std::shared_ptr& providerManager, + std::chrono::system_clock::time_point time) +{ + const auto today = std::chrono::floor(time); + + bool productTimesPopulated = true; + + // Don't query for the epoch, assume populated + if (today == std::chrono::system_clock::time_point {}) + { + return productTimesPopulated; + } + + const auto yesterday = today - std::chrono::days {1}; + const auto tomorrow = today + std::chrono::days {1}; + const auto dates = {yesterday, today, tomorrow}; + + for (auto& date : dates) + { + // Don't query for a time point in the future + if (date > scwx::util::time::now()) + { + continue; + } + + if (!providerManager->provider_->IsDateCached(date)) + { + productTimesPopulated = false; + } + } + + return productTimesPopulated; +} + void RadarProductManagerImpl::PopulateLevel2ProductTimes( - std::chrono::system_clock::time_point time) + std::chrono::system_clock::time_point time, bool update) { PopulateProductTimes(level2ProviderManager_, level2ProductRecords_, level2ProductRecordMutex_, - time); - PopulateProductTimes(level2ChunksProviderManager_, - level2ProductRecords_, - level2ProductRecordMutex_, - time); + time, + update); } void RadarProductManagerImpl::PopulateLevel3ProductTimes( - const std::string& product, std::chrono::system_clock::time_point time) + const std::string& product, + std::chrono::system_clock::time_point time, + bool update) { // Get provider manager auto level3ProviderManager = GetLevel3ProviderManager(product); @@ -1229,15 +1290,23 @@ void RadarProductManagerImpl::PopulateLevel3ProductTimes( PopulateProductTimes(level3ProviderManager, level3ProductRecords, level3ProductRecordMutex_, - time); + time, + update); } void RadarProductManagerImpl::PopulateProductTimes( std::shared_ptr providerManager, RadarProductRecordMap& productRecordMap, std::shared_mutex& productRecordMutex, - std::chrono::system_clock::time_point time) + std::chrono::system_clock::time_point time, + bool update) { + logger_->debug("Populating product times (Update: {}): {}, {}, {}", + update, + common::GetRadarProductGroupName(providerManager->group_), + providerManager->product_, + scwx::util::time::TimeString(time)); + const auto today = std::chrono::floor(time); // Don't query for the epoch @@ -1267,7 +1336,8 @@ void RadarProductManagerImpl::PopulateProductTimes( // Query the provider for volume time points auto timePoints = - providerManager->provider_->GetTimePointsByDate(date); + providerManager->provider_->GetTimePointsByDate(date, + update); // Lock the merged volume time list std::unique_lock volumeTimesLock {volumeTimesMutex}; @@ -1381,16 +1451,46 @@ RadarProductManagerImpl::GetLevel2ProductRecords( } std::tuple, - std::chrono::system_clock::time_point> + std::chrono::system_clock::time_point, + types::RadarProductLoadStatus> RadarProductManagerImpl::GetLevel3ProductRecord( const std::string& product, std::chrono::system_clock::time_point time) { std::shared_ptr record {nullptr}; RadarProductRecordMap::const_pointer recordPtr {nullptr}; std::chrono::system_clock::time_point recordTime {time}; + types::RadarProductLoadStatus status { + types::RadarProductLoadStatus::ListingProducts}; // Ensure Level 3 product records are updated - PopulateLevel3ProductTimes(product, time); + if (!AreLevel3ProductTimesPopulated(product, time)) + { + logger_->debug("Level 3 product times need populated: {}, {}", + product, + scwx::util::time::TimeString(time)); + + // Populate level 3 product times asynchronously + boost::asio::post(threadPool_, + [product, time, this]() + { + // Populate product times + PopulateLevel3ProductTimes(product, time); + + // Signal finished + Q_EMIT self_->ProductTimesPopulated( + common::RadarProductGroup::Level3, product, time); + }); + + // Return listing products status + return {record, recordTime, status}; + } + else + { + PopulateLevel3ProductTimes(product, time, false); + } + + // Advance to loading product + status = types::RadarProductLoadStatus::LoadingProduct; std::unique_lock lock {level3ProductRecordMutex_}; @@ -1415,9 +1515,27 @@ RadarProductManagerImpl::GetLevel3ProductRecord( if (recordPtr != nullptr) { + using namespace std::chrono_literals; + // Don't check for an exact time match for level 3 products recordTime = recordPtr->first; - record = recordPtr->second.lock(); + + if ( + // For latest data, ensure it is from the last 24 hours + (time == std::chrono::system_clock::time_point {} && + (recordTime > scwx::util::time::now() - 24h || recordTime == time)) || + // For time queries, ensure data is within 24 hours of the request + (time != std::chrono::system_clock::time_point {} && + std::chrono::abs(recordTime - time) < 24h)) + { + record = recordPtr->second.lock(); + } + else + { + // Reset the record + recordPtr = nullptr; + recordTime = time; + } } if (recordPtr != nullptr && record == nullptr && @@ -1440,9 +1558,22 @@ RadarProductManagerImpl::GetLevel3ProductRecord( }); self_->LoadLevel3Data(product, recordTime, request); + + // Status is already set to LoadingProduct } - return {record, recordTime}; + if (recordPtr == nullptr) + { + // If the record is empty, the product is not available + status = types::RadarProductLoadStatus::ProductNotAvailable; + } + else if (record != nullptr) + { + // If the record was populated, the product has been loaded + status = types::RadarProductLoadStatus::ProductLoaded; + } + + return {record, recordTime, status}; } std::shared_ptr @@ -1543,7 +1674,8 @@ void RadarProductManagerImpl::UpdateRecentRecords( std::tuple, float, std::vector, - std::chrono::system_clock::time_point> + std::chrono::system_clock::time_point, + types::RadarProductLoadStatus> RadarProductManager::GetLevel2Data(wsr88d::rda::DataBlockType dataBlockType, float elevation, std::chrono::system_clock::time_point time) @@ -1639,25 +1771,31 @@ RadarProductManager::GetLevel2Data(wsr88d::rda::DataBlockType dataBlockType, } } - return {radarData, elevationCut, elevationCuts, foundTime}; + return {radarData, + elevationCut, + elevationCuts, + foundTime, + types::RadarProductLoadStatus::ProductLoaded}; } std::tuple, - std::chrono::system_clock::time_point> + std::chrono::system_clock::time_point, + types::RadarProductLoadStatus> RadarProductManager::GetLevel3Data(const std::string& product, std::chrono::system_clock::time_point time) { std::shared_ptr message = nullptr; + types::RadarProductLoadStatus status {}; std::shared_ptr record; - std::tie(record, time) = p->GetLevel3ProductRecord(product, time); + std::tie(record, time, status) = p->GetLevel3ProductRecord(product, time); if (record != nullptr) { message = record->level3_file()->message(); } - return {message, time}; + return {message, time, status}; } common::Level3ProductCategoryMap @@ -1809,6 +1947,4 @@ RadarProductManager::Instance(const std::string& radarSite) #include "radar_product_manager.moc" -} // namespace manager -} // namespace qt -} // namespace scwx +} // namespace scwx::qt::manager diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.hpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.hpp index e8c72193..87bd0484 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.hpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.hpp @@ -5,23 +5,19 @@ #include #include #include +#include #include #include #include #include #include -#include #include #include #include -namespace scwx -{ -namespace qt -{ -namespace manager +namespace scwx::qt::manager { class RadarProductManagerImpl; @@ -89,12 +85,13 @@ public: * @param [in] time Radar product time * * @return Level 2 radar data, selected elevation cut, available elevation - * cuts and selected time + * cuts, selected time and product load status */ std::tuple, float, std::vector, - std::chrono::system_clock::time_point> + std::chrono::system_clock::time_point, + types::RadarProductLoadStatus> GetLevel2Data(wsr88d::rda::DataBlockType dataBlockType, float elevation, std::chrono::system_clock::time_point time = {}); @@ -105,10 +102,11 @@ public: * @param [in] product Radar product name * @param [in] time Radar product time * - * @return Level 3 message data and selected time + * @return Level 3 message data, selected time and product load status */ std::tuple, - std::chrono::system_clock::time_point> + std::chrono::system_clock::time_point, + types::RadarProductLoadStatus> GetLevel3Data(const std::string& product, std::chrono::system_clock::time_point time = {}); @@ -151,6 +149,9 @@ signals: bool isChunks, std::chrono::system_clock::time_point latestTime); void IncomingLevel2ElevationChanged(std::optional incomingElevation); + void ProductTimesPopulated(common::RadarProductGroup group, + const std::string& product, + std::chrono::system_clock::time_point queryTime); private: std::unique_ptr p; @@ -158,6 +159,4 @@ private: friend class RadarProductManagerImpl; }; -} // namespace manager -} // namespace qt -} // namespace scwx +} // namespace scwx::qt::manager diff --git a/scwx-qt/source/scwx/qt/types/radar_product_types.hpp b/scwx-qt/source/scwx/qt/types/radar_product_types.hpp new file mode 100644 index 00000000..2a1f568e --- /dev/null +++ b/scwx-qt/source/scwx/qt/types/radar_product_types.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include + +namespace scwx::qt::types +{ + +enum class RadarProductLoadStatus : std::uint8_t +{ + ProductLoaded, + ListingProducts, + LoadingProduct, + ProductNotAvailable +}; + +} diff --git a/scwx-qt/source/scwx/qt/view/level2_product_view.cpp b/scwx-qt/source/scwx/qt/view/level2_product_view.cpp index 9ead358b..eadf3615 100644 --- a/scwx-qt/source/scwx/qt/view/level2_product_view.cpp +++ b/scwx-qt/source/scwx/qt/view/level2_product_view.cpp @@ -11,11 +11,7 @@ #include #include -namespace scwx -{ -namespace qt -{ -namespace view +namespace scwx::qt::view { static const std::string logPrefix_ = "scwx::qt::view::level2_product_view"; @@ -552,7 +548,11 @@ void Level2ProductView::ComputeSweep() std::shared_ptr radarData; std::chrono::system_clock::time_point requestedTime {selected_time()}; - std::tie(radarData, p->elevationCut_, p->elevationCuts_, std::ignore) = + std::tie(radarData, + p->elevationCut_, + p->elevationCuts_, + std::ignore, + std::ignore) = radarProductManager->GetLevel2Data( p->dataBlockType_, p->selectedElevation_, requestedTime); @@ -1369,7 +1369,7 @@ Level2ProductView::GetBinLevel(const common::Coordinate& coordinate) const auto nextRadial = radarData->find((i + 1) % numRadials); if (nextRadial != radarData->cend()) { - nextAngle = nextRadial->second->azimuth_angle(); + nextAngle = nextRadial->second->azimuth_angle(); // Level 2 angles are the center of the bins. const units::degrees deltaAngle = @@ -1564,6 +1564,4 @@ std::shared_ptr Level2ProductView::Create( return std::make_shared(product, radarProductManager); } -} // namespace view -} // namespace qt -} // namespace scwx +} // namespace scwx::qt::view diff --git a/scwx-qt/source/scwx/qt/view/level3_product_view.cpp b/scwx-qt/source/scwx/qt/view/level3_product_view.cpp index 97985a39..6abe7556 100644 --- a/scwx-qt/source/scwx/qt/view/level3_product_view.cpp +++ b/scwx-qt/source/scwx/qt/view/level3_product_view.cpp @@ -11,17 +11,12 @@ #include #include -#include #include #include #include -namespace scwx -{ -namespace qt -{ -namespace view +namespace scwx::qt::view { static const std::string logPrefix_ = "scwx::qt::view::level3_product_view"; @@ -151,6 +146,22 @@ void Level3ProductView::ConnectRadarProductManager() Update(); } }); + + connect(radar_product_manager().get(), + &manager::RadarProductManager::ProductTimesPopulated, + this, + [this](common::RadarProductGroup group, + const std::string& product, + std::chrono::system_clock::time_point queryTime) + { + if (group == common::RadarProductGroup::Level3 && + product == p->product_ && queryTime == selected_time()) + { + // If the data associated with the currently selected time is + // reloaded, update the view + Update(); + } + }); } void Level3ProductView::DisconnectRadarProductManager() @@ -596,6 +607,4 @@ bool Level3ProductView::IgnoreUnits() const return false; } -} // namespace view -} // namespace qt -} // namespace scwx +} // namespace scwx::qt::view diff --git a/scwx-qt/source/scwx/qt/view/level3_radial_view.cpp b/scwx-qt/source/scwx/qt/view/level3_radial_view.cpp index c01e0cd4..60eb780e 100644 --- a/scwx-qt/source/scwx/qt/view/level3_radial_view.cpp +++ b/scwx-qt/source/scwx/qt/view/level3_radial_view.cpp @@ -10,11 +10,7 @@ #include #include -namespace scwx -{ -namespace qt -{ -namespace view +namespace scwx::qt::view { static const std::string logPrefix_ = "scwx::qt::view::level3_radial_view"; @@ -31,15 +27,7 @@ static constexpr std::uint32_t VALUES_PER_VERTEX = 2u; class Level3RadialView::Impl { public: - explicit Impl(Level3RadialView* self) : - self_ {self}, - latitude_ {}, - longitude_ {}, - range_ {}, - vcp_ {}, - sweepTime_ {} - { - } + explicit Impl(Level3RadialView* self) : self_ {self} {} ~Impl() { threadPool_.join(); }; void ComputeCoordinates( @@ -65,13 +53,13 @@ public: bool lastShowSmoothedRangeFolding_ {false}; bool lastSmoothingEnabled_ {false}; - float latitude_; - float longitude_; + float latitude_ {}; + float longitude_ {}; std::optional elevation_ {}; - float range_; - std::uint16_t vcp_; + float range_ {}; + std::uint16_t vcp_ {}; - std::chrono::system_clock::time_point sweepTime_; + std::chrono::system_clock::time_point sweepTime_ {}; }; Level3RadialView::Level3RadialView( @@ -148,7 +136,7 @@ void Level3RadialView::ComputeSweep() std::shared_ptr message; std::chrono::system_clock::time_point requestedTime {selected_time()}; std::chrono::system_clock::time_point foundTime; - std::tie(message, foundTime) = + std::tie(message, foundTime, std::ignore) = radarProductManager->GetLevel3Data(GetRadarProductName(), requestedTime); // If a different time was found than what was requested, update it @@ -752,6 +740,4 @@ std::shared_ptr Level3RadialView::Create( return std::make_shared(product, radarProductManager); } -} // namespace view -} // namespace qt -} // namespace scwx +} // namespace scwx::qt::view diff --git a/scwx-qt/source/scwx/qt/view/level3_raster_view.cpp b/scwx-qt/source/scwx/qt/view/level3_raster_view.cpp index 3056cc03..75aa1c0d 100644 --- a/scwx-qt/source/scwx/qt/view/level3_raster_view.cpp +++ b/scwx-qt/source/scwx/qt/view/level3_raster_view.cpp @@ -10,11 +10,7 @@ #include #include -namespace scwx -{ -namespace qt -{ -namespace view +namespace scwx::qt::view { static const std::string logPrefix_ = "scwx::qt::view::level3_raster_view"; @@ -125,7 +121,7 @@ void Level3RasterView::ComputeSweep() std::shared_ptr message; std::chrono::system_clock::time_point requestedTime {selected_time()}; std::chrono::system_clock::time_point foundTime; - std::tie(message, foundTime) = + std::tie(message, foundTime, std::ignore) = radarProductManager->GetLevel3Data(GetRadarProductName(), requestedTime); // If a different time was found than what was requested, update it @@ -538,6 +534,4 @@ std::shared_ptr Level3RasterView::Create( return std::make_shared(product, radarProductManager); } -} // namespace view -} // namespace qt -} // namespace scwx +} // namespace scwx::qt::view diff --git a/scwx-qt/source/scwx/qt/view/overlay_product_view.cpp b/scwx-qt/source/scwx/qt/view/overlay_product_view.cpp index ccc41b9d..95f1af09 100644 --- a/scwx-qt/source/scwx/qt/view/overlay_product_view.cpp +++ b/scwx-qt/source/scwx/qt/view/overlay_product_view.cpp @@ -8,11 +8,7 @@ #include #include -namespace scwx -{ -namespace qt -{ -namespace view +namespace scwx::qt::view { static const std::string logPrefix_ = "scwx::qt::view::overlay_product_view"; @@ -128,6 +124,22 @@ void OverlayProductView::Impl::ConnectRadarProductManager() } }, Qt::QueuedConnection); + + connect(radarProductManager_.get(), + &manager::RadarProductManager::ProductTimesPopulated, + self_, + [this](common::RadarProductGroup group, + const std::string& product, + std::chrono::system_clock::time_point queryTime) + { + if (group == common::RadarProductGroup::Level3 && + product == kNst_ && queryTime == selectedTime_) + { + // If the data associated with the currently selected time is + // reloaded, update the view + Update(product); + } + }); } void OverlayProductView::Impl::DisconnectRadarProductManager() @@ -286,7 +298,7 @@ void OverlayProductView::Impl::Update(const std::string& product) std::shared_ptr message; std::chrono::system_clock::time_point requestedTime {selectedTime_}; std::chrono::system_clock::time_point foundTime; - std::tie(message, foundTime) = + std::tie(message, foundTime, std::ignore) = radarProductManager_->GetLevel3Data(product, requestedTime); // If a different time was found than what was requested, update it @@ -329,6 +341,4 @@ void OverlayProductView::SetAutoUpdate(bool enabled) p->autoUpdateEnabled_ = enabled; } -} // namespace view -} // namespace qt -} // namespace scwx +} // namespace scwx::qt::view diff --git a/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp b/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp index abd70787..1c8a1ca9 100644 --- a/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp +++ b/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp @@ -46,7 +46,9 @@ public: std::string FindLatestKey() override; std::chrono::system_clock::time_point FindLatestTime() override; std::vector - GetTimePointsByDate(std::chrono::system_clock::time_point date) override; + GetTimePointsByDate(std::chrono::system_clock::time_point date, + bool update) override; + bool IsDateCached(std::chrono::system_clock::time_point date) override; std::tuple ListObjects(std::chrono::system_clock::time_point date) override; std::shared_ptr diff --git a/wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp b/wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp index d6ddcd7c..3db8c273 100644 --- a/wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp +++ b/wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp @@ -2,17 +2,12 @@ #include -namespace Aws -{ -namespace S3 +namespace Aws::S3 { class S3Client; -} // namespace S3 -} // namespace Aws +} // namespace Aws::S3 -namespace scwx -{ -namespace provider +namespace scwx::provider { /** @@ -32,20 +27,23 @@ public: AwsNexradDataProvider(AwsNexradDataProvider&&) noexcept; AwsNexradDataProvider& operator=(AwsNexradDataProvider&&) noexcept; - size_t cache_size() const override; + [[nodiscard]] std::size_t cache_size() const override; - std::chrono::system_clock::time_point last_modified() const override; - std::chrono::seconds update_period() const override; + [[nodiscard]] std::chrono::system_clock::time_point + last_modified() const override; + [[nodiscard]] std::chrono::seconds update_period() const override; std::string FindKey(std::chrono::system_clock::time_point time) override; std::string FindLatestKey() override; std::chrono::system_clock::time_point FindLatestTime() override; std::vector - GetTimePointsByDate(std::chrono::system_clock::time_point date) override; + GetTimePointsByDate(std::chrono::system_clock::time_point date, + bool update) override; + bool IsDateCached(std::chrono::system_clock::time_point date) override; std::tuple ListObjects(std::chrono::system_clock::time_point date) override; std::shared_ptr - LoadObjectByKey(const std::string& key) override; + LoadObjectByKey(const std::string& key) override; std::shared_ptr LoadObjectByTime(std::chrono::system_clock::time_point time) override; std::pair Refresh() override; @@ -61,5 +59,4 @@ private: std::unique_ptr p; }; -} // namespace provider -} // namespace scwx +} // namespace scwx::provider diff --git a/wxdata/include/scwx/provider/nexrad_data_provider.hpp b/wxdata/include/scwx/provider/nexrad_data_provider.hpp index 2a7320d2..2bc3703d 100644 --- a/wxdata/include/scwx/provider/nexrad_data_provider.hpp +++ b/wxdata/include/scwx/provider/nexrad_data_provider.hpp @@ -7,9 +7,7 @@ #include #include -namespace scwx -{ -namespace provider +namespace scwx::provider { class NexradDataProvider @@ -24,7 +22,7 @@ public: NexradDataProvider(NexradDataProvider&&) noexcept; NexradDataProvider& operator=(NexradDataProvider&&) noexcept; - virtual size_t cache_size() const = 0; + [[nodiscard]] virtual size_t cache_size() const = 0; /** * Gets the last modified time. This is equal to the most recent object's @@ -32,7 +30,8 @@ public: * * @return Last modified time */ - virtual std::chrono::system_clock::time_point last_modified() const = 0; + [[nodiscard]] virtual std::chrono::system_clock::time_point + last_modified() const = 0; /** * Gets the current update period. This is equal to the difference between @@ -41,7 +40,7 @@ public: * * @return Update period */ - virtual std::chrono::seconds update_period() const = 0; + [[nodiscard]] virtual std::chrono::seconds update_period() const = 0; /** * Finds the most recent key in the cache, no later than the time provided. @@ -116,7 +115,7 @@ public: * * @return NEXRAD data time point */ - virtual std::chrono::system_clock::time_point + [[nodiscard]] virtual std::chrono::system_clock::time_point GetTimePointByKey(const std::string& key) const = 0; /** @@ -124,11 +123,22 @@ public: * to the cache if required. * * @param date Date for which to get NEXRAD data time points + * @param update Whether or not to list and add data not present in the cache * * @return NEXRAD data time points */ virtual std::vector - GetTimePointsByDate(std::chrono::system_clock::time_point date) = 0; + GetTimePointsByDate(std::chrono::system_clock::time_point date, + bool update) = 0; + + /** + * Determines if time points for the requested date are cached. + * + * @param date Date for which to query the cache + * + * @return Whether or not the requested date is cached + */ + virtual bool IsDateCached(std::chrono::system_clock::time_point date) = 0; /** * Requests available NEXRAD products for the current radar site, and adds @@ -148,5 +158,4 @@ private: std::unique_ptr p; }; -} // namespace provider -} // namespace scwx +} // namespace scwx::provider diff --git a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp index f7de36ca..e5f99b99 100644 --- a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp @@ -289,11 +289,18 @@ AwsLevel2ChunksDataProvider::FindLatestTime() std::vector AwsLevel2ChunksDataProvider::GetTimePointsByDate( - std::chrono::system_clock::time_point /*date*/) + std::chrono::system_clock::time_point /* date */, bool /* update */) { return {}; } +bool AwsLevel2ChunksDataProvider::IsDateCached( + std::chrono::system_clock::time_point /* date */) +{ + // No cache, default to true + return true; +} + std::chrono::system_clock::time_point AwsLevel2ChunksDataProvider::Impl::GetScanTime(const std::string& prefix) { diff --git a/wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp b/wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp index 97528a9e..b7eab596 100644 --- a/wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp @@ -15,9 +15,7 @@ #include #include -namespace scwx -{ -namespace provider +namespace scwx::provider { static const std::string logPrefix_ = @@ -177,7 +175,7 @@ std::chrono::system_clock::time_point AwsNexradDataProvider::FindLatestTime() std::vector AwsNexradDataProvider::GetTimePointsByDate( - std::chrono::system_clock::time_point date) + std::chrono::system_clock::time_point date, bool update) { const auto day = std::chrono::floor(date); @@ -188,23 +186,26 @@ AwsNexradDataProvider::GetTimePointsByDate( std::shared_lock lock(p->objectsMutex_); // Is the date present in the date list? - bool currentDatePresent; + bool currentDatePresent = false; auto currentDateIterator = std::find(p->objectDates_.cbegin(), p->objectDates_.cend(), day); if (currentDateIterator == p->objectDates_.cend()) { - // Temporarily unlock mutex - lock.unlock(); - - // List objects, since the date is not present in the date list - auto [success, newObjects, totalObjects] = ListObjects(date); - if (success) + if (update) { - p->UpdateObjectDates(date); - } + // Temporarily unlock mutex + lock.unlock(); - // Re-lock mutex - lock.lock(); + // List objects, since the date is not present in the date list + const auto [success, newObjects, totalObjects] = ListObjects(date); + if (success) + { + p->UpdateObjectDates(date); + } + + // Re-lock mutex + lock.lock(); + } currentDatePresent = false; } @@ -214,8 +215,8 @@ AwsNexradDataProvider::GetTimePointsByDate( } // Determine objects to retrieve - auto objectsBegin = p->objects_.lower_bound(day); - auto objectsEnd = p->objects_.lower_bound(day + std::chrono::days {1}); + const auto objectsBegin = p->objects_.lower_bound(day); + const auto objectsEnd = p->objects_.lower_bound(day + std::chrono::days {1}); // Copy time points to destination vector std::transform(objectsBegin, @@ -236,6 +237,20 @@ AwsNexradDataProvider::GetTimePointsByDate( return timePoints; } +bool AwsNexradDataProvider::IsDateCached( + std::chrono::system_clock::time_point date) +{ + const auto day = std::chrono::floor(date); + + const std::shared_lock lock(p->objectsMutex_); + + // Is the date present in the date list? + const auto currentDateIterator = + std::find(p->objectDates_.cbegin(), p->objectDates_.cend(), day); + + return currentDateIterator != p->objectDates_.cend(); +} + std::tuple AwsNexradDataProvider::ListObjects(std::chrono::system_clock::time_point date) { @@ -446,5 +461,4 @@ void AwsNexradDataProvider::Impl::UpdateObjectDates( objectDates_.push_back(day); } -} // namespace provider -} // namespace scwx +} // namespace scwx::provider