From f0ef6b35dd60c3ef20385f626100f0d40ae8402a Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Thu, 27 Mar 2025 11:20:01 -0400 Subject: [PATCH 01/34] Slight rework to nexrad data provider interface --- .../scwx/qt/manager/radar_product_manager.cpp | 13 +++------- .../provider/aws_nexrad_data_provider.hpp | 4 +++ .../scwx/provider/nexrad_data_provider.hpp | 25 +++++++++++++++++++ .../provider/aws_nexrad_data_provider.cpp | 24 ++++++++++++++++++ 4 files changed, 56 insertions(+), 10 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp index 48584a2d..c93dd78a 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp @@ -769,9 +769,7 @@ void RadarProductManagerImpl::RefreshDataSync( if (totalObjects > 0) { - std::string key = providerManager->provider_->FindLatestKey(); - auto latestTime = providerManager->provider_->GetTimePointByKey(key); - + auto latestTime = providerManager->provider_->FindLatestTime(); auto updatePeriod = providerManager->provider_->update_period(); auto lastModified = providerManager->provider_->last_modified(); auto sinceLastModified = std::chrono::system_clock::now() - lastModified; @@ -951,13 +949,8 @@ void RadarProductManagerImpl::LoadProviderData( if (existingRecord == nullptr) { - std::string key = providerManager->provider_->FindKey(time); - - if (!key.empty()) - { - nexradFile = providerManager->provider_->LoadObjectByKey(key); - } - else + nexradFile = providerManager->provider_->LoadObjectByTime(time); + if (nexradFile == nullptr) { logger_->warn("Attempting to load object without key: {}", scwx::util::TimeString(time)); diff --git a/wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp b/wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp index 462d293d..cb71f27b 100644 --- a/wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp +++ b/wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp @@ -39,12 +39,16 @@ public: std::string FindKey(std::chrono::system_clock::time_point time) override; std::string FindLatestKey() override; + std::chrono::system_clock::time_point FindLatestTime() override; std::vector GetTimePointsByDate(std::chrono::system_clock::time_point date) override; std::tuple ListObjects(std::chrono::system_clock::time_point date) override; std::shared_ptr LoadObjectByKey(const std::string& key) override; + std::shared_ptr + LoadObjectByTime(std::chrono::system_clock::time_point time) override; + std::shared_ptr LoadLatestObject() override; std::pair Refresh() override; protected: diff --git a/wxdata/include/scwx/provider/nexrad_data_provider.hpp b/wxdata/include/scwx/provider/nexrad_data_provider.hpp index 14a75815..81edc1eb 100644 --- a/wxdata/include/scwx/provider/nexrad_data_provider.hpp +++ b/wxdata/include/scwx/provider/nexrad_data_provider.hpp @@ -59,6 +59,13 @@ public: */ virtual std::string FindLatestKey() = 0; + /** + * Finds the most recent time in the cache. + * + * @return NEXRAD data key + */ + virtual std::chrono::system_clock::time_point FindLatestTime() = 0; + /** * Lists NEXRAD objects for the date supplied, and adds them to the cache. * @@ -81,6 +88,24 @@ public: virtual std::shared_ptr LoadObjectByKey(const std::string& key) = 0; + /** + * Loads a NEXRAD file object at the given time + * + * @param time NEXRAD time + * + * @return NEXRAD data + */ + virtual std::shared_ptr + LoadObjectByTime(std::chrono::system_clock::time_point time) = 0; + + /** + * Loads the latest NEXRAD file object + * + * @return NEXRAD data + */ + virtual std::shared_ptr + LoadLatestObject() = 0; + /** * Lists NEXRAD objects for the current date, and adds them to the cache. If * no objects have been added to the cache for the current date, the previous diff --git a/wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp b/wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp index c4ac523b..74740be0 100644 --- a/wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp @@ -170,6 +170,11 @@ std::string AwsNexradDataProvider::FindLatestKey() return key; } +std::chrono::system_clock::time_point AwsNexradDataProvider::FindLatestTime() +{ + return GetTimePointByKey(FindLatestKey()); +} + std::vector AwsNexradDataProvider::GetTimePointsByDate( std::chrono::system_clock::time_point date) @@ -327,6 +332,25 @@ AwsNexradDataProvider::LoadObjectByKey(const std::string& key) return nexradFile; } +std::shared_ptr AwsNexradDataProvider::LoadObjectByTime( + std::chrono::system_clock::time_point time) +{ + const std::string key = FindKey(time); + if (key.empty()) + { + return nullptr; + } + else + { + return LoadObjectByKey(key); + } +} + +std::shared_ptr AwsNexradDataProvider::LoadLatestObject() +{ + return LoadObjectByKey(FindLatestKey()); +} + std::pair AwsNexradDataProvider::Refresh() { using namespace std::chrono; From 05335fad8473b342bf3ff313f6828c8e521ea67c Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Thu, 27 Mar 2025 11:21:08 -0400 Subject: [PATCH 02/34] Add ability to load new LDM records into preexisting Ar2vFile objects for L2 chunks --- wxdata/include/scwx/wsr88d/ar2v_file.hpp | 3 +++ wxdata/source/scwx/wsr88d/ar2v_file.cpp | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/wxdata/include/scwx/wsr88d/ar2v_file.hpp b/wxdata/include/scwx/wsr88d/ar2v_file.hpp index 1f3ab0cc..34d50b32 100644 --- a/wxdata/include/scwx/wsr88d/ar2v_file.hpp +++ b/wxdata/include/scwx/wsr88d/ar2v_file.hpp @@ -53,6 +53,9 @@ public: bool LoadFile(const std::string& filename); bool LoadData(std::istream& is); + bool LoadLDMRecords(std::istream& is); + bool IndexFile(); + private: std::unique_ptr p; }; diff --git a/wxdata/source/scwx/wsr88d/ar2v_file.cpp b/wxdata/source/scwx/wsr88d/ar2v_file.cpp index ed976c24..db04feba 100644 --- a/wxdata/source/scwx/wsr88d/ar2v_file.cpp +++ b/wxdata/source/scwx/wsr88d/ar2v_file.cpp @@ -519,5 +519,25 @@ void Ar2vFileImpl::IndexFile() } } +bool Ar2vFile::LoadLDMRecords(std::istream& is) { + size_t decompressedRecords = p->DecompressLDMRecords(is); + if (decompressedRecords == 0) + { + p->ParseLDMRecord(is); + } + else + { + p->ParseLDMRecords(); + } + + return true; +} + +bool Ar2vFile::IndexFile() +{ + p->IndexFile(); + return true; +} + } // namespace wsr88d } // namespace scwx From 9570dcf20e26bfd2db576ff871f9a2121684cd23 Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Thu, 27 Mar 2025 11:22:44 -0400 Subject: [PATCH 03/34] Initial working level2 chunks data provider --- .../aws_level2_chunks_data_provider.hpp | 62 +++ .../aws_level2_chunks_data_provider.cpp | 490 ++++++++++++++++++ wxdata/wxdata.cmake | 2 + 3 files changed, 554 insertions(+) create mode 100644 wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp create mode 100644 wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp diff --git a/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp b/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp new file mode 100644 index 00000000..247ff346 --- /dev/null +++ b/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp @@ -0,0 +1,62 @@ +#pragma once + +#include + +namespace Aws::S3 +{ +class S3Client; +} // namespace Aws::S3 + +namespace scwx::provider +{ + +/** + * @brief AWS Level 2 Data Provider + */ +class AwsLevel2ChunksDataProvider : public NexradDataProvider +{ +public: + explicit AwsLevel2ChunksDataProvider(const std::string& radarSite); + explicit AwsLevel2ChunksDataProvider(const std::string& radarSite, + const std::string& bucketName, + const std::string& region); + ~AwsLevel2ChunksDataProvider() override; + + AwsLevel2ChunksDataProvider(const AwsLevel2ChunksDataProvider&) = delete; + AwsLevel2ChunksDataProvider& operator=(const AwsLevel2ChunksDataProvider&) = delete; + + AwsLevel2ChunksDataProvider(AwsLevel2ChunksDataProvider&&) noexcept; + AwsLevel2ChunksDataProvider& operator=(AwsLevel2ChunksDataProvider&&) noexcept; + + [[nodiscard]] std::chrono::system_clock::time_point + GetTimePointByKey(const std::string& key) const override; + + [[nodiscard]] size_t cache_size() const override; + + [[nodiscard]] std::chrono::system_clock::time_point + last_modified() const override; + [[nodiscard]] std::chrono::seconds update_period() const override; + + std::string FindKey(std::chrono::system_clock::time_point time) override; + std::string FindLatestKey() override; + std::chrono::system_clock::time_point FindLatestTime() override; + std::vector + GetTimePointsByDate(std::chrono::system_clock::time_point date) override; + std::tuple + ListObjects(std::chrono::system_clock::time_point date) override; + std::shared_ptr + LoadObjectByKey(const std::string& key) override; + std::shared_ptr + LoadObjectByTime(std::chrono::system_clock::time_point time) override; + std::shared_ptr LoadLatestObject() override; + std::pair Refresh() override; + + void RequestAvailableProducts() override; + std::vector GetAvailableProducts() override; + +private: + class Impl; + std::unique_ptr p; +}; + +} // namespace scwx::provider diff --git a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp new file mode 100644 index 00000000..4d641d8a --- /dev/null +++ b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp @@ -0,0 +1,490 @@ +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +#if (__cpp_lib_chrono < 201907L) +# include +#endif + +namespace scwx::provider +{ + +static const std::string logPrefix_ = + "scwx::provider::aws_level2_chunks_data_provider"; +static const auto logger_ = util::Logger::Create(logPrefix_); + +static const std::string kDefaultBucketName_ = "unidata-nexrad-level2-chunks"; +static const std::string kDefaultRegion_ = "us-east-1"; + +class AwsLevel2ChunksDataProvider::Impl +{ +public: + struct ScanRecord + { + explicit ScanRecord(std::string prefix) : + prefix_ {std::move(prefix)}, + nexradFile_ {}, + lastModified_ {}, + secondLastModified_ {} + { + } + ~ScanRecord() = default; + ScanRecord(const ScanRecord&) = default; + ScanRecord(ScanRecord&&) = default; + ScanRecord& operator=(const ScanRecord&) = default; + ScanRecord& operator=(ScanRecord&&) = default; + + std::string prefix_; + std::shared_ptr nexradFile_; + std::chrono::system_clock::time_point lastModified_; + std::chrono::system_clock::time_point secondLastModified_; + int nextFile_{1}; + bool hasAllFiles_{false}; + }; + + explicit Impl(AwsLevel2ChunksDataProvider* self, + std::string radarSite, + std::string bucketName, + std::string region) : + radarSite_ {std::move(radarSite)}, + bucketName_ {std::move(bucketName)}, + region_ {std::move(region)}, + client_ {nullptr}, + scans_ {}, + scansMutex_ {}, + lastModified_ {}, + updatePeriod_ {}, + self_ {self} + { + // Disable HTTP request for region + util::SetEnvironment("AWS_EC2_METADATA_DISABLED", "true"); + + // Use anonymous credentials + Aws::Auth::AWSCredentials credentials {}; + + Aws::Client::ClientConfiguration config; + config.region = region_; + config.connectTimeoutMs = 10000; + + client_ = std::make_shared( + credentials, + Aws::MakeShared( + Aws::S3::S3Client::GetAllocationTag()), + config); + } + ~Impl() = default; + Impl(const Impl&) = delete; + Impl(Impl&&) = delete; + Impl& operator=(const Impl&) = delete; + Impl& operator=(Impl&&) = delete; + + std::chrono::system_clock::time_point GetScanTime(const std::string& prefix); + std::string GetScanKey(const std::string& prefix, + const std::chrono::system_clock::time_point& time, + int last); + std::shared_ptr LoadScan(Impl::ScanRecord& scanRecord); + + std::string radarSite_; + std::string bucketName_; + std::string region_; + std::shared_ptr client_; + + std::mutex refreshMutex_; + + std::map scans_; + std::shared_mutex scansMutex_; + + std::chrono::system_clock::time_point lastModified_; + std::chrono::seconds updatePeriod_; + + AwsLevel2ChunksDataProvider* self_; + }; + +AwsLevel2ChunksDataProvider::AwsLevel2ChunksDataProvider( + const std::string& radarSite) : + AwsLevel2ChunksDataProvider(radarSite, kDefaultBucketName_, kDefaultRegion_) +{ +} + +AwsLevel2ChunksDataProvider::AwsLevel2ChunksDataProvider( + const std::string& radarSite, + const std::string& bucketName, + const std::string& region) : + p(std::make_unique(this, radarSite, bucketName, region)) +{ +} + +AwsLevel2ChunksDataProvider::~AwsLevel2ChunksDataProvider() = default; + +std::chrono::system_clock::time_point +AwsLevel2ChunksDataProvider::GetTimePointByKey(const std::string& key) const +{ + std::chrono::system_clock::time_point time {}; + + const size_t lastSeparator = key.rfind('/'); + const size_t offset = + (lastSeparator == std::string::npos) ? 0 : lastSeparator + 1; + + // Filename format is YYYYMMDD-TTTTTT-AAA-B + static const size_t formatSize = std::string("YYYYMMDD-TTTTTT").size(); + + if (key.size() >= offset + formatSize) + { + using namespace std::chrono; + +#if (__cpp_lib_chrono < 201907L) + using namespace date; +#endif + + static const std::string timeFormat {"%Y%m%d-%H%M%S"}; + + std::string timeStr {key.substr(offset, formatSize)}; + std::istringstream in {timeStr}; + in >> parse(timeFormat, time); + + if (in.fail()) + { + logger_->warn("Invalid time: \"{}\"", timeStr); + } + } + else + { + logger_->warn("Time not parsable from key: \"{}\"", key); + } + + return time; +} + +size_t AwsLevel2ChunksDataProvider::cache_size() const +{ + return p->scans_.size(); +} + +std::chrono::system_clock::time_point +AwsLevel2ChunksDataProvider::last_modified() const +{ + return p->lastModified_; +} +std::chrono::seconds AwsLevel2ChunksDataProvider::update_period() const +{ + return p->updatePeriod_; +} + +std::string +AwsLevel2ChunksDataProvider::FindKey(std::chrono::system_clock::time_point time) +{ + logger_->debug("FindKey: {}", util::TimeString(time)); + + std::shared_lock lock(p->scansMutex_); + + auto element = util::GetBoundedElement(p->scans_, time); + + if (element.has_value()) + { + return element->prefix_; + } + + return {}; +} + +std::string AwsLevel2ChunksDataProvider::FindLatestKey() +{ + std::shared_lock lock(p->scansMutex_); + if (p->scans_.empty()) + { + return ""; + } + + return p->scans_.crbegin()->second.prefix_; +} + +std::chrono::system_clock::time_point +AwsLevel2ChunksDataProvider::FindLatestTime() +{ + std::shared_lock lock(p->scansMutex_); + if (p->scans_.empty()) + { + return {}; + } + + return p->scans_.crbegin()->first; +} + +std::vector +AwsLevel2ChunksDataProvider::GetTimePointsByDate( + std::chrono::system_clock::time_point /*date*/) +{ + return {}; +} + +std::chrono::system_clock::time_point +AwsLevel2ChunksDataProvider::Impl::GetScanTime(const std::string& prefix) +{ + Aws::S3::Model::ListObjectsV2Request request; + request.SetBucket(bucketName_); + request.SetPrefix(prefix); + request.SetDelimiter("/"); + request.SetMaxKeys(1); + + auto outcome = client_->ListObjectsV2(request); + if (outcome.IsSuccess()) + { + return self_->GetTimePointByKey( + outcome.GetResult().GetContents().at(0).GetKey()); + } + + return {}; +} + +std::string AwsLevel2ChunksDataProvider::Impl::GetScanKey( + const std::string& prefix, + const std::chrono::system_clock::time_point& time, + int last) +{ + + static const std::string timeFormat {"%Y%m%d-%H%M%S"}; + + //TODO + return fmt::format( + "{0}/{1:%Y%m%d-%H%M%S}-{2}", prefix, fmt::gmtime(time), last - 1); +} + +std::tuple +AwsLevel2ChunksDataProvider::ListObjects(std::chrono::system_clock::time_point) +{ + // TODO this is slow. It could probably be speed up by not reloading every + // scan every time. + const std::string prefix = p->radarSite_ + "/"; + + logger_->debug("ListObjects: {}", prefix); + + Aws::S3::Model::ListObjectsV2Request request; + request.SetBucket(p->bucketName_); + request.SetPrefix(prefix); + request.SetDelimiter("/"); + + auto outcome = p->client_->ListObjectsV2(request); + + size_t newObjects = 0; + size_t totalObjects = 0; + + if (outcome.IsSuccess()) + { + auto& scans = outcome.GetResult().GetCommonPrefixes(); + logger_->debug("Found {} scans", scans.size()); + + for (const auto& scan : scans) + { + const std::string& prefix = scan.GetPrefix(); + + auto time = p->GetScanTime(prefix); + + if (!p->scans_.contains(time)) + { + p->scans_.insert_or_assign(time, Impl::ScanRecord {prefix}); + newObjects++; + } + + totalObjects++; + } + } + + return {outcome.IsSuccess(), newObjects, totalObjects}; +} + +std::shared_ptr +AwsLevel2ChunksDataProvider::LoadObjectByKey(const std::string& /*prefix*/) +{ + return nullptr; +} + +std::shared_ptr +AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) +{ + if (scanRecord.hasAllFiles_) + { + return scanRecord.nexradFile_; + } + + // TODO can get only new records using scanRecords last + Aws::S3::Model::ListObjectsV2Request listRequest; + listRequest.SetBucket(bucketName_); + listRequest.SetPrefix(scanRecord.prefix_); + listRequest.SetDelimiter("/"); + + auto listOutcome = client_->ListObjectsV2(listRequest); + if (!listOutcome.IsSuccess()) + { + logger_->warn("Could not find scan at {}", scanRecord.prefix_); + return nullptr; + } + + auto& chunks = listOutcome.GetResult().GetContents(); + for (const auto& chunk : chunks) + { + const std::string& key = chunk.GetKey(); + + // We just want the number of this chunk for now + // KIND/585/20250324-134727-001-S + constexpr size_t startNumberPos = + std::string("KIND/585/20250324-134727-").size(); + const std::string& keyNumberStr = key.substr(startNumberPos, 3); + const int keyNumber = std::stoi(keyNumberStr); + if (keyNumber != scanRecord.nextFile_) + { + continue; + } + + // Now we want the ending char + // KIND/585/20250324-134727-001-S + constexpr size_t charPos = + std::string("KIND/585/20250324-134727-001-").size(); + const char keyChar = key[charPos]; + + Aws::S3::Model::GetObjectRequest objectRequest; + objectRequest.SetBucket(bucketName_); + objectRequest.SetKey(key); + + auto outcome = client_->GetObject(objectRequest); + + if (!outcome.IsSuccess()) + { + logger_->warn("Could not get object: {}", + outcome.GetError().GetMessage()); + return nullptr; + } + + auto& body = outcome.GetResultWithOwnership().GetBody(); + + switch (keyChar) { + case 'S': + { // First chunk + scanRecord.nexradFile_ = std::make_shared(); + if (!scanRecord.nexradFile_->LoadData(body)) + { + logger_->warn("Failed to load first chunk"); + return nullptr; + } + break; + } + case 'I': + { // Middle chunk + if (!scanRecord.nexradFile_->LoadLDMRecords(body)) + { + logger_->warn("Failed to load middle chunk"); + return nullptr; + } + break; + } + case 'E': + { // Last chunk + if (!scanRecord.nexradFile_->LoadLDMRecords(body)) + { + logger_->warn("Failed to load last chunk"); + return nullptr; + } + scanRecord.hasAllFiles_ = true; + break; + } + default: + return nullptr; + } + + std::chrono::seconds lastModifiedSeconds { + outcome.GetResult().GetLastModified().Seconds()}; + std::chrono::system_clock::time_point lastModified { + lastModifiedSeconds}; + + scanRecord.secondLastModified_ = scanRecord.lastModified_; + scanRecord.lastModified_ = lastModified; + + scanRecord.nextFile_ += 1; + } + scanRecord.nexradFile_->IndexFile(); + + if (!scans_.empty()) + { + auto& lastScan = scans_.crend()->second; + lastModified_ = lastScan.lastModified_; + if (lastScan.secondLastModified_ != + std::chrono::system_clock::time_point()) + { + auto delta = lastScan.lastModified_ - lastScan.secondLastModified_; + updatePeriod_ = + std::chrono::duration_cast(delta); + } + } + + return scanRecord.nexradFile_; +} + +std::shared_ptr +AwsLevel2ChunksDataProvider::LoadObjectByTime( + std::chrono::system_clock::time_point time) +{ + std::shared_lock lock(p->scansMutex_); + + logger_->error("LoadObjectByTime({})", time); + + auto scanRecord = util::GetBoundedElementPointer(p->scans_, time); + if (scanRecord == nullptr) + { + logger_->warn("Could not find object at time {}", time); + return nullptr; + } + + // The scanRecord must be a reference + return p->LoadScan(p->scans_.at(scanRecord->first)); +} + +std::shared_ptr +AwsLevel2ChunksDataProvider::LoadLatestObject() +{ + return LoadObjectByTime(FindLatestTime()); +} + +std::pair AwsLevel2ChunksDataProvider::Refresh() +{ + using namespace std::chrono; + + std::unique_lock lock(p->refreshMutex_); + + auto [success, newObjects, totalObjects] = ListObjects({}); + + for (auto& scanRecord : p->scans_) + { + if (scanRecord.second.nexradFile_ != nullptr) + { + p->LoadScan(scanRecord.second); + newObjects += 1; + } + } + + return std::make_pair(newObjects, totalObjects); +} + +void AwsLevel2ChunksDataProvider::RequestAvailableProducts() {} +std::vector AwsLevel2ChunksDataProvider::GetAvailableProducts() +{ + return {}; +} + +AwsLevel2ChunksDataProvider::AwsLevel2ChunksDataProvider( + AwsLevel2ChunksDataProvider&&) noexcept = default; +AwsLevel2ChunksDataProvider& AwsLevel2ChunksDataProvider::operator=( + AwsLevel2ChunksDataProvider&&) noexcept = default; + +} // namespace scwx::provider diff --git a/wxdata/wxdata.cmake b/wxdata/wxdata.cmake index 8d2e15b4..2c062f4b 100644 --- a/wxdata/wxdata.cmake +++ b/wxdata/wxdata.cmake @@ -60,6 +60,7 @@ set(HDR_NETWORK include/scwx/network/cpr.hpp set(SRC_NETWORK source/scwx/network/cpr.cpp source/scwx/network/dir_list.cpp) set(HDR_PROVIDER include/scwx/provider/aws_level2_data_provider.hpp + include/scwx/provider/aws_level2_chunks_data_provider.hpp include/scwx/provider/aws_level3_data_provider.hpp include/scwx/provider/aws_nexrad_data_provider.hpp include/scwx/provider/iem_api_provider.hpp @@ -68,6 +69,7 @@ set(HDR_PROVIDER include/scwx/provider/aws_level2_data_provider.hpp include/scwx/provider/nexrad_data_provider_factory.hpp include/scwx/provider/warnings_provider.hpp) set(SRC_PROVIDER source/scwx/provider/aws_level2_data_provider.cpp + source/scwx/provider/aws_level2_chunks_data_provider.cpp source/scwx/provider/aws_level3_data_provider.cpp source/scwx/provider/aws_nexrad_data_provider.cpp source/scwx/provider/iem_api_provider.cpp From 7fef5789de4e5343912eb447dd4e826d45d51cf8 Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Thu, 27 Mar 2025 11:23:25 -0400 Subject: [PATCH 04/34] Temporarly use only the L2 Chunks data provider for L2 data --- wxdata/source/scwx/provider/nexrad_data_provider_factory.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wxdata/source/scwx/provider/nexrad_data_provider_factory.cpp b/wxdata/source/scwx/provider/nexrad_data_provider_factory.cpp index 5e75fd96..83ccb1a3 100644 --- a/wxdata/source/scwx/provider/nexrad_data_provider_factory.cpp +++ b/wxdata/source/scwx/provider/nexrad_data_provider_factory.cpp @@ -1,5 +1,6 @@ #include -#include +//#include +#include #include namespace scwx @@ -14,7 +15,7 @@ std::shared_ptr NexradDataProviderFactory::CreateLevel2DataProvider( const std::string& radarSite) { - return std::make_unique(radarSite); + return std::make_unique(radarSite); } std::shared_ptr From ac12cce5f24d0aa1cf2294fb2ea291a3dbe08df7 Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Thu, 27 Mar 2025 12:07:49 -0400 Subject: [PATCH 05/34] fix compilation errors with level_2_chunks for not gcc-13/clang-17 --- .../aws_level2_chunks_data_provider.hpp | 12 +++--- .../aws_level2_chunks_data_provider.cpp | 43 +++++++++---------- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp b/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp index 247ff346..57e8e301 100644 --- a/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp +++ b/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp @@ -23,10 +23,12 @@ public: ~AwsLevel2ChunksDataProvider() override; AwsLevel2ChunksDataProvider(const AwsLevel2ChunksDataProvider&) = delete; - AwsLevel2ChunksDataProvider& operator=(const AwsLevel2ChunksDataProvider&) = delete; + AwsLevel2ChunksDataProvider& + operator=(const AwsLevel2ChunksDataProvider&) = delete; AwsLevel2ChunksDataProvider(AwsLevel2ChunksDataProvider&&) noexcept; - AwsLevel2ChunksDataProvider& operator=(AwsLevel2ChunksDataProvider&&) noexcept; + AwsLevel2ChunksDataProvider& + operator=(AwsLevel2ChunksDataProvider&&) noexcept; [[nodiscard]] std::chrono::system_clock::time_point GetTimePointByKey(const std::string& key) const override; @@ -45,13 +47,13 @@ public: std::tuple ListObjects(std::chrono::system_clock::time_point date) override; std::shared_ptr - LoadObjectByKey(const std::string& key) override; + LoadObjectByKey(const std::string& key) override; std::shared_ptr LoadObjectByTime(std::chrono::system_clock::time_point time) override; std::shared_ptr LoadLatestObject() override; - std::pair Refresh() override; + std::pair Refresh() override; - void RequestAvailableProducts() override; + void RequestAvailableProducts() override; std::vector GetAvailableProducts() override; private: diff --git a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp index 4d641d8a..5dccf149 100644 --- a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp @@ -21,7 +21,6 @@ namespace scwx::provider { - static const std::string logPrefix_ = "scwx::provider::aws_level2_chunks_data_provider"; static const auto logger_ = util::Logger::Create(logPrefix_); @@ -47,12 +46,12 @@ public: ScanRecord& operator=(const ScanRecord&) = default; ScanRecord& operator=(ScanRecord&&) = default; - std::string prefix_; - std::shared_ptr nexradFile_; + std::string prefix_; + std::shared_ptr nexradFile_; std::chrono::system_clock::time_point lastModified_; std::chrono::system_clock::time_point secondLastModified_; - int nextFile_{1}; - bool hasAllFiles_{false}; + int nextFile_ {1}; + bool hasAllFiles_ {false}; }; explicit Impl(AwsLevel2ChunksDataProvider* self, @@ -92,9 +91,9 @@ public: Impl& operator=(Impl&&) = delete; std::chrono::system_clock::time_point GetScanTime(const std::string& prefix); - std::string GetScanKey(const std::string& prefix, - const std::chrono::system_clock::time_point& time, - int last); + std::string GetScanKey(const std::string& prefix, + const std::chrono::system_clock::time_point& time, + int last); std::shared_ptr LoadScan(Impl::ScanRecord& scanRecord); std::string radarSite_; @@ -111,7 +110,7 @@ public: std::chrono::seconds updatePeriod_; AwsLevel2ChunksDataProvider* self_; - }; +}; AwsLevel2ChunksDataProvider::AwsLevel2ChunksDataProvider( const std::string& radarSite) : @@ -225,7 +224,7 @@ AwsLevel2ChunksDataProvider::FindLatestTime() std::vector AwsLevel2ChunksDataProvider::GetTimePointsByDate( - std::chrono::system_clock::time_point /*date*/) + std::chrono::system_clock::time_point /*date*/) { return {}; } @@ -254,10 +253,10 @@ std::string AwsLevel2ChunksDataProvider::Impl::GetScanKey( const std::chrono::system_clock::time_point& time, int last) { - + static const std::string timeFormat {"%Y%m%d-%H%M%S"}; - //TODO + // TODO return fmt::format( "{0}/{1:%Y%m%d-%H%M%S}-{2}", prefix, fmt::gmtime(time), last - 1); } @@ -288,13 +287,13 @@ AwsLevel2ChunksDataProvider::ListObjects(std::chrono::system_clock::time_point) for (const auto& scan : scans) { - const std::string& prefix = scan.GetPrefix(); + const std::string& prefixScan = scan.GetPrefix(); - auto time = p->GetScanTime(prefix); + auto time = p->GetScanTime(prefixScan); if (!p->scans_.contains(time)) { - p->scans_.insert_or_assign(time, Impl::ScanRecord {prefix}); + p->scans_.insert_or_assign(time, Impl::ScanRecord {prefixScan}); newObjects++; } @@ -306,7 +305,7 @@ AwsLevel2ChunksDataProvider::ListObjects(std::chrono::system_clock::time_point) } std::shared_ptr -AwsLevel2ChunksDataProvider::LoadObjectByKey(const std::string& /*prefix*/) +AwsLevel2ChunksDataProvider::LoadObjectByKey(const std::string& /*prefix*/) { return nullptr; } @@ -339,10 +338,10 @@ AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) // We just want the number of this chunk for now // KIND/585/20250324-134727-001-S - constexpr size_t startNumberPos = + static const size_t startNumberPos = std::string("KIND/585/20250324-134727-").size(); const std::string& keyNumberStr = key.substr(startNumberPos, 3); - const int keyNumber = std::stoi(keyNumberStr); + const int keyNumber = std::stoi(keyNumberStr); if (keyNumber != scanRecord.nextFile_) { continue; @@ -350,7 +349,7 @@ AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) // Now we want the ending char // KIND/585/20250324-134727-001-S - constexpr size_t charPos = + static const size_t charPos = std::string("KIND/585/20250324-134727-001-").size(); const char keyChar = key[charPos]; @@ -369,7 +368,8 @@ AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) auto& body = outcome.GetResultWithOwnership().GetBody(); - switch (keyChar) { + switch (keyChar) + { case 'S': { // First chunk scanRecord.nexradFile_ = std::make_shared(); @@ -405,8 +405,7 @@ AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) std::chrono::seconds lastModifiedSeconds { outcome.GetResult().GetLastModified().Seconds()}; - std::chrono::system_clock::time_point lastModified { - lastModifiedSeconds}; + std::chrono::system_clock::time_point lastModified {lastModifiedSeconds}; scanRecord.secondLastModified_ = scanRecord.lastModified_; scanRecord.lastModified_ = lastModified; From 7c99bbc185cf92a54dcc538151adc734de23694b Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Fri, 28 Mar 2025 11:06:11 -0400 Subject: [PATCH 06/34] Begin work on moving over to only storing last 2 scans in chunks --- .../aws_level2_chunks_data_provider.cpp | 255 ++++++++++++------ 1 file changed, 166 insertions(+), 89 deletions(-) diff --git a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp index 5dccf149..236b97ee 100644 --- a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -33,11 +34,12 @@ class AwsLevel2ChunksDataProvider::Impl public: struct ScanRecord { - explicit ScanRecord(std::string prefix) : + explicit ScanRecord(std::string prefix, bool valid = true) : + valid_ {valid}, prefix_ {std::move(prefix)}, nexradFile_ {}, lastModified_ {}, - secondLastModified_ {} + lastKey_ {""} { } ~ScanRecord() = default; @@ -46,10 +48,12 @@ public: ScanRecord& operator=(const ScanRecord&) = default; ScanRecord& operator=(ScanRecord&&) = default; + bool valid_; std::string prefix_; std::shared_ptr nexradFile_; + std::chrono::system_clock::time_point time_; std::chrono::system_clock::time_point lastModified_; - std::chrono::system_clock::time_point secondLastModified_; + std::string lastKey_; int nextFile_ {1}; bool hasAllFiles_ {false}; }; @@ -62,10 +66,11 @@ public: bucketName_ {std::move(bucketName)}, region_ {std::move(region)}, client_ {nullptr}, - scans_ {}, + scanTimes_ {}, + lastScan_ {"", false}, + currentScan_ {"", false}, scansMutex_ {}, - lastModified_ {}, - updatePeriod_ {}, + updatePeriod_ {15}, self_ {self} { // Disable HTTP request for region @@ -95,6 +100,7 @@ public: const std::chrono::system_clock::time_point& time, int last); std::shared_ptr LoadScan(Impl::ScanRecord& scanRecord); + int GetScanNumber(const std::string& prefix); std::string radarSite_; std::string bucketName_; @@ -103,11 +109,13 @@ public: std::mutex refreshMutex_; - std::map scans_; - std::shared_mutex scansMutex_; + std::unordered_map + scanTimes_; + ScanRecord lastScan_; + ScanRecord currentScan_; + std::shared_mutex scansMutex_; - std::chrono::system_clock::time_point lastModified_; - std::chrono::seconds updatePeriod_; + std::chrono::seconds updatePeriod_; AwsLevel2ChunksDataProvider* self_; }; @@ -169,13 +177,13 @@ AwsLevel2ChunksDataProvider::GetTimePointByKey(const std::string& key) const size_t AwsLevel2ChunksDataProvider::cache_size() const { - return p->scans_.size(); + return 2; } std::chrono::system_clock::time_point AwsLevel2ChunksDataProvider::last_modified() const { - return p->lastModified_; + return p->currentScan_.lastModified_; } std::chrono::seconds AwsLevel2ChunksDataProvider::update_period() const { @@ -188,12 +196,13 @@ AwsLevel2ChunksDataProvider::FindKey(std::chrono::system_clock::time_point time) logger_->debug("FindKey: {}", util::TimeString(time)); std::shared_lock lock(p->scansMutex_); - - auto element = util::GetBoundedElement(p->scans_, time); - - if (element.has_value()) + if (p->currentScan_.valid_ && time >= p->currentScan_.time_) { - return element->prefix_; + return p->currentScan_.prefix_; + } + else if (p->lastScan_.valid_ && time >= p->lastScan_.time_) + { + return p->lastScan_.prefix_; } return {}; @@ -202,24 +211,24 @@ AwsLevel2ChunksDataProvider::FindKey(std::chrono::system_clock::time_point time) std::string AwsLevel2ChunksDataProvider::FindLatestKey() { std::shared_lock lock(p->scansMutex_); - if (p->scans_.empty()) + if (!p->currentScan_.valid_) { return ""; } - return p->scans_.crbegin()->second.prefix_; + return p->currentScan_.prefix_; } std::chrono::system_clock::time_point AwsLevel2ChunksDataProvider::FindLatestTime() { std::shared_lock lock(p->scansMutex_); - if (p->scans_.empty()) + if (!p->currentScan_.valid_) { return {}; } - return p->scans_.crbegin()->first; + return p->currentScan_.time_; } std::vector @@ -232,6 +241,12 @@ AwsLevel2ChunksDataProvider::GetTimePointsByDate( std::chrono::system_clock::time_point AwsLevel2ChunksDataProvider::Impl::GetScanTime(const std::string& prefix) { + const auto& scanTimeIt = scanTimes_.find(prefix); // O(log(n)) + if (scanTimeIt != scanTimes_.cend()) + { + return scanTimeIt->second; + } + Aws::S3::Model::ListObjectsV2Request request; request.SetBucket(bucketName_); request.SetPrefix(prefix); @@ -241,8 +256,9 @@ AwsLevel2ChunksDataProvider::Impl::GetScanTime(const std::string& prefix) auto outcome = client_->ListObjectsV2(request); if (outcome.IsSuccess()) { - return self_->GetTimePointByKey( + auto timePoint = self_->GetTimePointByKey( outcome.GetResult().GetContents().at(0).GetKey()); + return timePoint; } return {}; @@ -264,44 +280,7 @@ std::string AwsLevel2ChunksDataProvider::Impl::GetScanKey( std::tuple AwsLevel2ChunksDataProvider::ListObjects(std::chrono::system_clock::time_point) { - // TODO this is slow. It could probably be speed up by not reloading every - // scan every time. - const std::string prefix = p->radarSite_ + "/"; - - logger_->debug("ListObjects: {}", prefix); - - Aws::S3::Model::ListObjectsV2Request request; - request.SetBucket(p->bucketName_); - request.SetPrefix(prefix); - request.SetDelimiter("/"); - - auto outcome = p->client_->ListObjectsV2(request); - - size_t newObjects = 0; - size_t totalObjects = 0; - - if (outcome.IsSuccess()) - { - auto& scans = outcome.GetResult().GetCommonPrefixes(); - logger_->debug("Found {} scans", scans.size()); - - for (const auto& scan : scans) - { - const std::string& prefixScan = scan.GetPrefix(); - - auto time = p->GetScanTime(prefixScan); - - if (!p->scans_.contains(time)) - { - p->scans_.insert_or_assign(time, Impl::ScanRecord {prefixScan}); - newObjects++; - } - - totalObjects++; - } - } - - return {outcome.IsSuccess(), newObjects, totalObjects}; + return {true, 0, 0}; } std::shared_ptr @@ -313,16 +292,23 @@ AwsLevel2ChunksDataProvider::LoadObjectByKey(const std::string& /*prefix*/) std::shared_ptr AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) { - if (scanRecord.hasAllFiles_) + if (!scanRecord.valid_) + { + return nullptr; + } + else if (scanRecord.hasAllFiles_) { return scanRecord.nexradFile_; } - // TODO can get only new records using scanRecords last Aws::S3::Model::ListObjectsV2Request listRequest; listRequest.SetBucket(bucketName_); listRequest.SetPrefix(scanRecord.prefix_); listRequest.SetDelimiter("/"); + if (!scanRecord.lastKey_.empty()) + { + listRequest.SetStartAfter(scanRecord.lastKey_); + } auto listOutcome = client_->ListObjectsV2(listRequest); if (!listOutcome.IsSuccess()) @@ -336,6 +322,7 @@ AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) { const std::string& key = chunk.GetKey(); + // TODO this is wrong, 1st number can be 1-3 digits // We just want the number of this chunk for now // KIND/585/20250324-134727-001-S static const size_t startNumberPos = @@ -347,6 +334,7 @@ AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) continue; } + // TODO this is wrong, 1st number can be 1-3 digits // Now we want the ending char // KIND/585/20250324-134727-001-S static const size_t charPos = @@ -407,24 +395,15 @@ AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) outcome.GetResult().GetLastModified().Seconds()}; std::chrono::system_clock::time_point lastModified {lastModifiedSeconds}; - scanRecord.secondLastModified_ = scanRecord.lastModified_; - scanRecord.lastModified_ = lastModified; + scanRecord.lastModified_ = lastModified; scanRecord.nextFile_ += 1; + scanRecord.lastKey_ = key; } - scanRecord.nexradFile_->IndexFile(); - if (!scans_.empty()) + if (scanRecord.nexradFile_ != nullptr) { - auto& lastScan = scans_.crend()->second; - lastModified_ = lastScan.lastModified_; - if (lastScan.secondLastModified_ != - std::chrono::system_clock::time_point()) - { - auto delta = lastScan.lastModified_ - lastScan.secondLastModified_; - updatePeriod_ = - std::chrono::duration_cast(delta); - } + scanRecord.nexradFile_->IndexFile(); } return scanRecord.nexradFile_; @@ -434,19 +413,20 @@ std::shared_ptr AwsLevel2ChunksDataProvider::LoadObjectByTime( std::chrono::system_clock::time_point time) { - std::shared_lock lock(p->scansMutex_); + std::unique_lock lock(p->scansMutex_); - logger_->error("LoadObjectByTime({})", time); - - auto scanRecord = util::GetBoundedElementPointer(p->scans_, time); - if (scanRecord == nullptr) + if (p->currentScan_.valid_ && time >= p->currentScan_.time_) + { + return p->LoadScan(p->currentScan_); + } + else if (p->lastScan_.valid_ && time >= p->lastScan_.time_) + { + return p->LoadScan(p->lastScan_); + } + else { - logger_->warn("Could not find object at time {}", time); return nullptr; } - - // The scanRecord must be a reference - return p->LoadScan(p->scans_.at(scanRecord->first)); } std::shared_ptr @@ -455,23 +435,120 @@ AwsLevel2ChunksDataProvider::LoadLatestObject() return LoadObjectByTime(FindLatestTime()); } +int AwsLevel2ChunksDataProvider::Impl::GetScanNumber(const std::string& prefix) +{ + + // We just want the number of this chunk for now + // KIND/585/20250324-134727-001-S + static const size_t startNumberPos = std::string("KIND/").size(); + const std::string& prefixNumberStr = prefix.substr(startNumberPos, 3); + return std::stoi(prefixNumberStr); +} + std::pair AwsLevel2ChunksDataProvider::Refresh() { using namespace std::chrono; std::unique_lock lock(p->refreshMutex_); + std::unique_lock scanLock(p->scansMutex_); - auto [success, newObjects, totalObjects] = ListObjects({}); - for (auto& scanRecord : p->scans_) + size_t newObjects = 0; + size_t totalObjects = 0; + + const std::string prefix = p->radarSite_ + "/"; + + Aws::S3::Model::ListObjectsV2Request request; + request.SetBucket(p->bucketName_); + request.SetPrefix(prefix); + request.SetDelimiter("/"); + + auto outcome = p->client_->ListObjectsV2(request); + + + if (outcome.IsSuccess()) { - if (scanRecord.second.nexradFile_ != nullptr) + auto& scans = outcome.GetResult().GetCommonPrefixes(); + logger_->debug("Found {} scans", scans.size()); + + boost::timer::cpu_timer timer {}; + timer.start(); + if (scans.size() > 0) { - p->LoadScan(scanRecord.second); - newObjects += 1; + + // TODO this cannot be done by getting things form the network. + // Use index number instead. + + // find latest scan + std::chrono::system_clock::time_point latestTime = {}; + std::chrono::system_clock::time_point secondLatestTime = {}; + size_t latestIndex = 0; + size_t secondLatestIndex = 0; + + + for (size_t i = 0; i < scans.size(); i++) // O(n log(n)) n <= 999 + { + auto time = p->GetScanTime(scans[i].GetPrefix()); + if (time > latestTime) + { + secondLatestTime = latestTime; + latestTime = time; + secondLatestIndex = latestIndex; + latestIndex = i; + } + } + + const auto& last = scans.at(secondLatestIndex).GetPrefix(); + if (secondLatestTime != std::chrono::system_clock::time_point {}) + { + p->lastScan_ = p->currentScan_; + } + else if (!p->lastScan_.valid_ || p->lastScan_.prefix_ != last) + { + p->lastScan_.valid_ = true; + p->lastScan_.prefix_ = last; + p->lastScan_.nexradFile_ = nullptr; + p->lastScan_.time_ = secondLatestTime; + p->lastScan_.lastModified_ = {}; + p->lastScan_.lastKey_ = ""; + p->lastScan_.nextFile_ = 1; + p->lastScan_.hasAllFiles_ = false; + newObjects += 1; + } + + const auto& current = scans.at(latestIndex).GetPrefix(); + if (!p->currentScan_.valid_ || p->currentScan_.prefix_ != current) + { + p->currentScan_.valid_ = true; + p->currentScan_.prefix_ = current; + p->currentScan_.nexradFile_ = nullptr; + p->currentScan_.time_ = latestTime; + p->currentScan_.lastModified_ = {}; + p->currentScan_.lastKey_ = ""; + p->currentScan_.nextFile_ = 1; + p->currentScan_.hasAllFiles_ = false; + newObjects += 1; + } } + + timer.stop(); + logger_->debug("Updated current scans in {}", timer.format(6, "%ws")); } + logger_->debug("Loading scans"); + + if (p->currentScan_.valid_) + { + p->LoadScan(p->currentScan_); + totalObjects += 1; + } + if (p->lastScan_.valid_) + { + p->LoadScan(p->lastScan_); + totalObjects += 1; + } + + return std::make_pair(newObjects, totalObjects); } From fc83a7a36f35862378a16384fb241f311cca052b Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Sun, 30 Mar 2025 11:24:42 -0400 Subject: [PATCH 07/34] working level2 chunks with auto rerendering --- .../scwx/qt/manager/radar_product_manager.cpp | 12 +- .../scwx/qt/view/level2_product_view.cpp | 3 +- .../aws_level2_chunks_data_provider.cpp | 342 +++++++++++------- 3 files changed, 232 insertions(+), 125 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp index c93dd78a..9bd5dbd8 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp @@ -66,6 +66,7 @@ static const std::string kDefaultLevel3Product_ {"N0B"}; static constexpr std::size_t kTimerPlaces_ {6u}; static constexpr std::chrono::seconds kFastRetryInterval_ {15}; +static constexpr std::chrono::seconds kFastRetryIntervalChunks_ {3}; static constexpr std::chrono::seconds kSlowRetryInterval_ {120}; static std::unordered_map> @@ -765,7 +766,12 @@ void RadarProductManagerImpl::RefreshDataSync( auto [newObjects, totalObjects] = providerManager->provider_->Refresh(); - std::chrono::milliseconds interval = kFastRetryInterval_; + // Level2 chunked data is updated quickly and uses a fater interval + const std::chrono::milliseconds fastRetryInterval = + providerManager->group_ == common::RadarProductGroup::Level2 ? + kFastRetryIntervalChunks_ : + kFastRetryInterval_; + std::chrono::milliseconds interval = fastRetryInterval; if (totalObjects > 0) { @@ -786,10 +792,10 @@ void RadarProductManagerImpl::RefreshDataSync( // been last modified, slow the retry period interval = kSlowRetryInterval_; } - else if (interval < std::chrono::milliseconds {kFastRetryInterval_}) + else if (interval < std::chrono::milliseconds {fastRetryInterval}) { // The interval should be no quicker than the fast retry interval - interval = kFastRetryInterval_; + interval = fastRetryInterval; } if (newObjects > 0) diff --git a/scwx-qt/source/scwx/qt/view/level2_product_view.cpp b/scwx-qt/source/scwx/qt/view/level2_product_view.cpp index 6344dff0..69f0dbf2 100644 --- a/scwx-qt/source/scwx/qt/view/level2_product_view.cpp +++ b/scwx-qt/source/scwx/qt/view/level2_product_view.cpp @@ -561,7 +561,8 @@ void Level2ProductView::ComputeSweep() Q_EMIT SweepNotComputed(types::NoUpdateReason::NotLoaded); return; } - if (radarData == p->elevationScan_ && + // TODO do not do this when updating from live data + if (false && (radarData == p->elevationScan_) && smoothingEnabled == p->lastSmoothingEnabled_ && (showSmoothedRangeFolding == p->lastShowSmoothedRangeFolding_ || !smoothingEnabled)) diff --git a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp index 236b97ee..3536c16c 100644 --- a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp @@ -53,6 +53,7 @@ public: std::shared_ptr nexradFile_; std::chrono::system_clock::time_point time_; std::chrono::system_clock::time_point lastModified_; + std::chrono::system_clock::time_point secondLastModified_; std::string lastKey_; int nextFile_ {1}; bool hasAllFiles_ {false}; @@ -70,7 +71,8 @@ public: lastScan_ {"", false}, currentScan_ {"", false}, scansMutex_ {}, - updatePeriod_ {15}, + lastTimeListed_ {}, + updatePeriod_ {7}, self_ {self} { // Disable HTTP request for region @@ -96,11 +98,14 @@ public: Impl& operator=(Impl&&) = delete; std::chrono::system_clock::time_point GetScanTime(const std::string& prefix); - std::string GetScanKey(const std::string& prefix, - const std::chrono::system_clock::time_point& time, - int last); - std::shared_ptr LoadScan(Impl::ScanRecord& scanRecord); - int GetScanNumber(const std::string& prefix); + int GetScanNumber(const std::string& prefix); + std::string GetScanKey(const std::string& prefix, + const std::chrono::system_clock::time_point& time, + int last); + + bool LoadScan(Impl::ScanRecord& scanRecord); + std::tuple ListObjects(); + std::string radarSite_; std::string bucketName_; @@ -110,10 +115,11 @@ public: std::mutex refreshMutex_; std::unordered_map - scanTimes_; - ScanRecord lastScan_; - ScanRecord currentScan_; - std::shared_mutex scansMutex_; + scanTimes_; + ScanRecord lastScan_; + ScanRecord currentScan_; + std::shared_mutex scansMutex_; + std::chrono::system_clock::time_point lastTimeListed_; std::chrono::seconds updatePeriod_; @@ -187,6 +193,24 @@ AwsLevel2ChunksDataProvider::last_modified() const } std::chrono::seconds AwsLevel2ChunksDataProvider::update_period() const { + std::shared_lock lock(p->scansMutex_); + // Add an extra second of delay + static const auto extra = std::chrono::seconds(1); + // get update period from time between chunks + if (p->currentScan_.valid_ && p->currentScan_.nextFile_ > 2) + { + auto delta = + p->currentScan_.lastModified_ - p->currentScan_.secondLastModified_; + return std::chrono::duration_cast(delta) + extra; + } + else if (p->lastScan_.valid_ && p->lastScan_.nextFile_ > 2) + { + auto delta = + p->lastScan_.lastModified_ - p->lastScan_.secondLastModified_; + return std::chrono::duration_cast(delta) + extra; + } + + // default to a set update period return p->updatePeriod_; } @@ -241,10 +265,16 @@ AwsLevel2ChunksDataProvider::GetTimePointsByDate( std::chrono::system_clock::time_point AwsLevel2ChunksDataProvider::Impl::GetScanTime(const std::string& prefix) { + using namespace std::chrono; const auto& scanTimeIt = scanTimes_.find(prefix); // O(log(n)) if (scanTimeIt != scanTimes_.cend()) { - return scanTimeIt->second; + // If the time is greater than 2 hours ago, it may be a new scan + auto replaceBy = system_clock::now() - hours {2}; + if (scanTimeIt->second > replaceBy) + { + return scanTimeIt->second; + } } Aws::S3::Model::ListObjectsV2Request request; @@ -258,6 +288,7 @@ AwsLevel2ChunksDataProvider::Impl::GetScanTime(const std::string& prefix) { auto timePoint = self_->GetTimePointByKey( outcome.GetResult().GetContents().at(0).GetKey()); + scanTimes_.insert_or_assign(prefix, timePoint); return timePoint; } @@ -277,6 +308,135 @@ std::string AwsLevel2ChunksDataProvider::Impl::GetScanKey( "{0}/{1:%Y%m%d-%H%M%S}-{2}", prefix, fmt::gmtime(time), last - 1); } +std::tuple +AwsLevel2ChunksDataProvider::Impl::ListObjects() +{ + size_t newObjects = 0; + size_t totalObjects = 0; + + std::chrono::system_clock::time_point now = std::chrono::system_clock::now(); + + if (currentScan_.valid_ && !currentScan_.hasAllFiles_ && + lastTimeListed_ + std::chrono::minutes(7) > now) + { + return {true, newObjects, totalObjects}; + } + logger_->debug("ListObjects"); + lastTimeListed_ = now; + + const std::string prefix = radarSite_ + "/"; + + Aws::S3::Model::ListObjectsV2Request request; + request.SetBucket(bucketName_); + request.SetPrefix(prefix); + request.SetDelimiter("/"); + + auto outcome = client_->ListObjectsV2(request); + + if (outcome.IsSuccess()) + { + auto& scans = outcome.GetResult().GetCommonPrefixes(); + logger_->debug("Found {} scans", scans.size()); + + if (scans.size() > 0) + { + // find latest scan + auto scanNumberMap = std::map(); + + for (auto& scan : scans) // O(n log(n)) n <= 999 + { + const std::string& scanPrefix = scan.GetPrefix(); + scanNumberMap.insert_or_assign(GetScanNumber(scanPrefix), + scanPrefix); + } + + // TODO ensure not out of range + int lastScanNumber = -1; + // Start with last scan + int previousScanNumber = scanNumberMap.crbegin()->first; + const int firstScanNumber = scanNumberMap.cbegin()->first; + + // This indicates that highest number scan is the last scan + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) + if (previousScanNumber != 999 || firstScanNumber != 1) + { + lastScanNumber = previousScanNumber; + } + else + { + // have already checked scan with highest number, so skip first + previousScanNumber = firstScanNumber; + bool first = true; + for (const auto& scan : scanNumberMap) + { + if (first) + { + first = false; + continue; + } + if (scan.first != previousScanNumber + 1) + { + lastScanNumber = previousScanNumber; + break; + } + previousScanNumber = scan.first; + } + } + + if (lastScanNumber == -1) + { + logger_->warn("Could not find last scan"); + // TODO make sure this makes sence + return {false, 0, 0}; + } + + std::string& lastScanPrefix = scanNumberMap.at(lastScanNumber); + int secondLastScanNumber = + lastScanNumber == 1 ? 999 : lastScanNumber - 1; + + const auto& secondLastScanPrefix = + scanNumberMap.find(secondLastScanNumber); + + if (!currentScan_.valid_ || + currentScan_.prefix_ != lastScanPrefix) + { + if (currentScan_.valid_ && + (secondLastScanPrefix == scanNumberMap.cend() || + currentScan_.prefix_ == secondLastScanPrefix->second)) + { + lastScan_ = currentScan_; + } + else if (secondLastScanPrefix != scanNumberMap.cend()) + { + lastScan_.valid_ = true; + lastScan_.prefix_ = secondLastScanPrefix->second; + lastScan_.nexradFile_ = nullptr; + lastScan_.time_ = GetScanTime(secondLastScanPrefix->second); + lastScan_.lastModified_ = {}; + lastScan_.secondLastModified_ = {}; + lastScan_.lastKey_ = ""; + lastScan_.nextFile_ = 1; + lastScan_.hasAllFiles_ = false; + newObjects += 1; + } + + currentScan_.valid_ = true; + currentScan_.prefix_ = lastScanPrefix; + currentScan_.nexradFile_ = nullptr; + currentScan_.time_ = GetScanTime(lastScanPrefix); + currentScan_.lastModified_ = {}; + currentScan_.secondLastModified_ = {}; + currentScan_.lastKey_ = ""; + currentScan_.nextFile_ = 1; + currentScan_.hasAllFiles_ = false; + newObjects += 1; + } + } + } + + return {true, newObjects, totalObjects}; +} + std::tuple AwsLevel2ChunksDataProvider::ListObjects(std::chrono::system_clock::time_point) { @@ -289,16 +449,16 @@ AwsLevel2ChunksDataProvider::LoadObjectByKey(const std::string& /*prefix*/) return nullptr; } -std::shared_ptr -AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) +bool AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) { if (!scanRecord.valid_) { - return nullptr; + logger_->warn("Tried to load scan which was not listed yet"); + return false; } else if (scanRecord.hasAllFiles_) { - return scanRecord.nexradFile_; + return false; } Aws::S3::Model::ListObjectsV2Request listRequest; @@ -314,19 +474,23 @@ AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) if (!listOutcome.IsSuccess()) { logger_->warn("Could not find scan at {}", scanRecord.prefix_); - return nullptr; + return false; } + bool hasNew = false; auto& chunks = listOutcome.GetResult().GetContents(); for (const auto& chunk : chunks) { const std::string& key = chunk.GetKey(); - // TODO this is wrong, 1st number can be 1-3 digits - // We just want the number of this chunk for now // KIND/585/20250324-134727-001-S - static const size_t startNumberPos = - std::string("KIND/585/20250324-134727-").size(); + // KIND/5/20250324-134727-001-S + static const size_t firstSlash = std::string("KIND/").size(); + const size_t secondSlash = key.find('/', firstSlash); + static const size_t startNumberPosOffset = + std::string("/20250324-134727-").size(); + const size_t startNumberPos = + secondSlash + startNumberPosOffset; const std::string& keyNumberStr = key.substr(startNumberPos, 3); const int keyNumber = std::stoi(keyNumberStr); if (keyNumber != scanRecord.nextFile_) @@ -334,12 +498,11 @@ AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) continue; } - // TODO this is wrong, 1st number can be 1-3 digits // Now we want the ending char // KIND/585/20250324-134727-001-S static const size_t charPos = - std::string("KIND/585/20250324-134727-001-").size(); - const char keyChar = key[charPos]; + std::string("/20250324-134727-001-").size(); + const char keyChar = key[secondSlash + charPos]; Aws::S3::Model::GetObjectRequest objectRequest; objectRequest.SetBucket(bucketName_); @@ -351,7 +514,7 @@ AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) { logger_->warn("Could not get object: {}", outcome.GetError().GetMessage()); - return nullptr; + return hasNew; } auto& body = outcome.GetResultWithOwnership().GetBody(); @@ -364,7 +527,7 @@ AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) if (!scanRecord.nexradFile_->LoadData(body)) { logger_->warn("Failed to load first chunk"); - return nullptr; + return hasNew; } break; } @@ -373,7 +536,7 @@ AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) if (!scanRecord.nexradFile_->LoadLDMRecords(body)) { logger_->warn("Failed to load middle chunk"); - return nullptr; + return hasNew; } break; } @@ -382,31 +545,38 @@ AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) if (!scanRecord.nexradFile_->LoadLDMRecords(body)) { logger_->warn("Failed to load last chunk"); - return nullptr; + return hasNew; } scanRecord.hasAllFiles_ = true; break; } default: - return nullptr; + logger_->warn("Could not load chunk with unknown char"); + return hasNew; } + hasNew = true; std::chrono::seconds lastModifiedSeconds { outcome.GetResult().GetLastModified().Seconds()}; std::chrono::system_clock::time_point lastModified {lastModifiedSeconds}; + scanRecord.secondLastModified_ = scanRecord.lastModified_; scanRecord.lastModified_ = lastModified; scanRecord.nextFile_ += 1; scanRecord.lastKey_ = key; } - if (scanRecord.nexradFile_ != nullptr) + if (scanRecord.nexradFile_ == nullptr) + { + logger_->warn("Could not load file"); + } + else { scanRecord.nexradFile_->IndexFile(); } - return scanRecord.nexradFile_; + return hasNew; } std::shared_ptr @@ -417,14 +587,15 @@ AwsLevel2ChunksDataProvider::LoadObjectByTime( if (p->currentScan_.valid_ && time >= p->currentScan_.time_) { - return p->LoadScan(p->currentScan_); + return p->currentScan_.nexradFile_; } else if (p->lastScan_.valid_ && time >= p->lastScan_.time_) { - return p->LoadScan(p->lastScan_); + return p->lastScan_.nexradFile_; } else { + logger_->warn("Could not find scan with time"); return nullptr; } } @@ -440,8 +611,8 @@ int AwsLevel2ChunksDataProvider::Impl::GetScanNumber(const std::string& prefix) // We just want the number of this chunk for now // KIND/585/20250324-134727-001-S - static const size_t startNumberPos = std::string("KIND/").size(); - const std::string& prefixNumberStr = prefix.substr(startNumberPos, 3); + static const size_t firstSlash = std::string("KIND/").size(); + const std::string& prefixNumberStr = prefix.substr(firstSlash, 3); return std::stoi(prefixNumberStr); } @@ -449,106 +620,35 @@ std::pair AwsLevel2ChunksDataProvider::Refresh() { using namespace std::chrono; + boost::timer::cpu_timer timer {}; + timer.start(); + std::unique_lock lock(p->refreshMutex_); std::unique_lock scanLock(p->scansMutex_); - - size_t newObjects = 0; - size_t totalObjects = 0; - - const std::string prefix = p->radarSite_ + "/"; - - Aws::S3::Model::ListObjectsV2Request request; - request.SetBucket(p->bucketName_); - request.SetPrefix(prefix); - request.SetDelimiter("/"); - - auto outcome = p->client_->ListObjectsV2(request); - - - if (outcome.IsSuccess()) - { - auto& scans = outcome.GetResult().GetCommonPrefixes(); - logger_->debug("Found {} scans", scans.size()); - - boost::timer::cpu_timer timer {}; - timer.start(); - if (scans.size() > 0) - { - - // TODO this cannot be done by getting things form the network. - // Use index number instead. - - // find latest scan - std::chrono::system_clock::time_point latestTime = {}; - std::chrono::system_clock::time_point secondLatestTime = {}; - size_t latestIndex = 0; - size_t secondLatestIndex = 0; - - - for (size_t i = 0; i < scans.size(); i++) // O(n log(n)) n <= 999 - { - auto time = p->GetScanTime(scans[i].GetPrefix()); - if (time > latestTime) - { - secondLatestTime = latestTime; - latestTime = time; - secondLatestIndex = latestIndex; - latestIndex = i; - } - } - - const auto& last = scans.at(secondLatestIndex).GetPrefix(); - if (secondLatestTime != std::chrono::system_clock::time_point {}) - { - p->lastScan_ = p->currentScan_; - } - else if (!p->lastScan_.valid_ || p->lastScan_.prefix_ != last) - { - p->lastScan_.valid_ = true; - p->lastScan_.prefix_ = last; - p->lastScan_.nexradFile_ = nullptr; - p->lastScan_.time_ = secondLatestTime; - p->lastScan_.lastModified_ = {}; - p->lastScan_.lastKey_ = ""; - p->lastScan_.nextFile_ = 1; - p->lastScan_.hasAllFiles_ = false; - newObjects += 1; - } - - const auto& current = scans.at(latestIndex).GetPrefix(); - if (!p->currentScan_.valid_ || p->currentScan_.prefix_ != current) - { - p->currentScan_.valid_ = true; - p->currentScan_.prefix_ = current; - p->currentScan_.nexradFile_ = nullptr; - p->currentScan_.time_ = latestTime; - p->currentScan_.lastModified_ = {}; - p->currentScan_.lastKey_ = ""; - p->currentScan_.nextFile_ = 1; - p->currentScan_.hasAllFiles_ = false; - newObjects += 1; - } - } - - timer.stop(); - logger_->debug("Updated current scans in {}", timer.format(6, "%ws")); - } - - logger_->debug("Loading scans"); + auto [success, newObjects, totalObjects] = p->ListObjects(); if (p->currentScan_.valid_) { - p->LoadScan(p->currentScan_); + if (p->LoadScan(p->currentScan_)) + { + newObjects += 1; + } totalObjects += 1; } if (p->lastScan_.valid_) { - p->LoadScan(p->lastScan_); + /* + if (p->LoadScan(p->lastScan_)) + { + newObjects += 1; + } + */ totalObjects += 1; } - + timer.stop(); + logger_->debug("Refresh() in {}", timer.format(6, "%ws")); return std::make_pair(newObjects, totalObjects); } From a754d6684492a1e1e215d37ca3d73b9659d89c4d Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Sun, 30 Mar 2025 13:18:04 -0400 Subject: [PATCH 08/34] Setting up for merging last and current scan's, and having archive and chunks --- .../scwx/qt/manager/radar_product_manager.cpp | 129 +++++++++++++++--- .../aws_level2_chunks_data_provider.hpp | 1 + .../provider/aws_nexrad_data_provider.hpp | 1 + .../scwx/provider/nexrad_data_provider.hpp | 9 ++ .../provider/nexrad_data_provider_factory.hpp | 3 + .../aws_level2_chunks_data_provider.cpp | 8 +- .../provider/aws_nexrad_data_provider.cpp | 6 + .../provider/nexrad_data_provider_factory.cpp | 9 +- 8 files changed, 143 insertions(+), 23 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp index 9bd5dbd8..0c7f10ea 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp @@ -134,6 +134,8 @@ public: level3ProductsInitialized_ {false}, radarSite_ {config::RadarSite::Get(radarId)}, level2ProviderManager_ {std::make_shared( + self_, radarId_, common::RadarProductGroup::Level2)}, + level2ChunksProviderManager_ {std::make_shared( self_, radarId_, common::RadarProductGroup::Level2)} { if (radarSite_ == nullptr) @@ -144,10 +146,14 @@ public: level2ProviderManager_->provider_ = provider::NexradDataProviderFactory::CreateLevel2DataProvider(radarId); + level2ChunksProviderManager_->provider_ = + provider::NexradDataProviderFactory::CreateLevel2ChunksDataProvider( + radarId); } ~RadarProductManagerImpl() { level2ProviderManager_->Disable(); + level2ChunksProviderManager_->Disable(); std::shared_lock lock(level3ProviderManagerMutex_); std::for_each(std::execution::par_unseq, @@ -251,6 +257,7 @@ public: std::shared_mutex level3ProductRecordMutex_ {}; std::shared_ptr level2ProviderManager_; + std::shared_ptr level2ChunksProviderManager_; std::unordered_map> level3ProviderManagerMap_ {}; std::shared_mutex level3ProviderManagerMutex_ {}; @@ -639,6 +646,7 @@ void RadarProductManager::EnableRefresh(common::RadarProductGroup group, if (group == common::RadarProductGroup::Level2) { p->EnableRefresh(uuid, p->level2ProviderManager_, enabled); + p->EnableRefresh(uuid, p->level2ChunksProviderManager_, enabled); } else { @@ -986,6 +994,12 @@ void RadarProductManager::LoadLevel2Data( p->level2ProductRecordMutex_, p->loadLevel2DataMutex_, request); + p->LoadProviderData(time, + p->level2ChunksProviderManager_, + p->level2ProductRecords_, + p->level2ProductRecordMutex_, + p->loadLevel2DataMutex_, + request); } void RadarProductManager::LoadLevel3Data( @@ -1162,6 +1176,10 @@ void RadarProductManagerImpl::PopulateLevel2ProductTimes( level2ProductRecords_, level2ProductRecordMutex_, time); + PopulateProductTimes(level2ChunksProviderManager_, + level2ProductRecords_, + level2ProductRecordMutex_, + time); } void RadarProductManagerImpl::PopulateLevel3ProductTimes( @@ -1504,35 +1522,104 @@ RadarProductManager::GetLevel2Data(wsr88d::rda::DataBlockType dataBlockType, auto records = p->GetLevel2ProductRecords(time); - for (auto& recordPair : records) + //TODO decide when to use chunked vs archived data. + if (true) { - auto& record = recordPair.second; - - if (record != nullptr) + auto currentFile = std::dynamic_pointer_cast( + p->level2ChunksProviderManager_->provider_->LoadLatestObject()); + std::shared_ptr currentRadarData = nullptr; + float currentElevationCut = 0.0f; + std::vector currentElevationCuts; + if (currentFile != nullptr) { - std::shared_ptr recordRadarData = nullptr; - float recordElevationCut = 0.0f; - std::vector recordElevationCuts; + std::tie(currentRadarData, currentElevationCut, currentElevationCuts) = + currentFile->GetElevationScan(dataBlockType, elevation, time); + } - std::tie(recordRadarData, recordElevationCut, recordElevationCuts) = - record->level2_file()->GetElevationScan( - dataBlockType, elevation, time); + std::shared_ptr lastRadarData = nullptr; + float lastElevationCut = 0.0f; + std::vector lastElevationCuts; + auto lastFile = std::dynamic_pointer_cast( + p->level2ChunksProviderManager_->provider_->LoadSecondLatestObject()); + if (lastFile != nullptr) + { + std::tie(lastRadarData, lastElevationCut, lastElevationCuts) = + lastFile->GetElevationScan(dataBlockType, elevation, time); + } - if (recordRadarData != nullptr) + if (currentRadarData != nullptr) + { + if (lastRadarData != nullptr) { - auto& radarData0 = (*recordRadarData)[0]; + auto& radarData0 = (*currentRadarData)[0]; auto collectionTime = std::chrono::floor( - scwx::util::TimePoint(radarData0->modified_julian_date(), - radarData0->collection_time())); + scwx::util::TimePoint(radarData0->modified_julian_date(), + radarData0->collection_time())); - // Find the newest radar data, not newer than the selected time - if (radarData == nullptr || - (collectionTime <= time && foundTime < collectionTime)) + // TODO merge data + radarData = currentRadarData; + elevationCut = currentElevationCut; + elevationCuts = std::move(currentElevationCuts); + foundTime = collectionTime; + } + else + { + auto& radarData0 = (*currentRadarData)[0]; + auto collectionTime = std::chrono::floor( + scwx::util::TimePoint(radarData0->modified_julian_date(), + radarData0->collection_time())); + + radarData = currentRadarData; + elevationCut = currentElevationCut; + elevationCuts = std::move(currentElevationCuts); + foundTime = collectionTime; + } + } + else if (lastRadarData != nullptr) + { + auto& radarData0 = (*lastRadarData)[0]; + auto collectionTime = std::chrono::floor( + scwx::util::TimePoint(radarData0->modified_julian_date(), + radarData0->collection_time())); + + radarData = lastRadarData; + elevationCut = lastElevationCut; + elevationCuts = std::move(lastElevationCuts); + foundTime = collectionTime; + } + } + else + { + for (auto& recordPair : records) + { + auto& record = recordPair.second; + + if (record != nullptr) + { + std::shared_ptr recordRadarData = nullptr; + float recordElevationCut = 0.0f; + std::vector recordElevationCuts; + + std::tie(recordRadarData, recordElevationCut, recordElevationCuts) = + record->level2_file()->GetElevationScan( + dataBlockType, elevation, time); + + if (recordRadarData != nullptr) { - radarData = recordRadarData; - elevationCut = recordElevationCut; - elevationCuts = std::move(recordElevationCuts); - foundTime = collectionTime; + auto& radarData0 = (*recordRadarData)[0]; + auto collectionTime = std::chrono::floor( + scwx::util::TimePoint(radarData0->modified_julian_date(), + radarData0->collection_time())); + + // Find the newest radar data, not newer than the selected time + if (radarData == nullptr || + (collectionTime <= time && foundTime < collectionTime)) + { + radarData = recordRadarData; + elevationCut = recordElevationCut; + elevationCuts = std::move(recordElevationCuts); + foundTime = collectionTime; + } } } } diff --git a/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp b/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp index 57e8e301..106b6605 100644 --- a/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp +++ b/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp @@ -51,6 +51,7 @@ public: std::shared_ptr LoadObjectByTime(std::chrono::system_clock::time_point time) override; std::shared_ptr LoadLatestObject() override; + std::shared_ptr LoadSecondLatestObject() override; std::pair Refresh() override; void RequestAvailableProducts() override; diff --git a/wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp b/wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp index cb71f27b..b2946f38 100644 --- a/wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp +++ b/wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp @@ -49,6 +49,7 @@ public: std::shared_ptr LoadObjectByTime(std::chrono::system_clock::time_point time) override; std::shared_ptr LoadLatestObject() override; + std::shared_ptr LoadSecondLatestObject() override; std::pair Refresh() override; protected: diff --git a/wxdata/include/scwx/provider/nexrad_data_provider.hpp b/wxdata/include/scwx/provider/nexrad_data_provider.hpp index 81edc1eb..a0e09f04 100644 --- a/wxdata/include/scwx/provider/nexrad_data_provider.hpp +++ b/wxdata/include/scwx/provider/nexrad_data_provider.hpp @@ -106,6 +106,15 @@ public: virtual std::shared_ptr LoadLatestObject() = 0; + /** + * Loads the second NEXRAD file object + * + * @return NEXRAD data + */ + virtual std::shared_ptr + LoadSecondLatestObject() = 0; + + /** * Lists NEXRAD objects for the current date, and adds them to the cache. If * no objects have been added to the cache for the current date, the previous diff --git a/wxdata/include/scwx/provider/nexrad_data_provider_factory.hpp b/wxdata/include/scwx/provider/nexrad_data_provider_factory.hpp index bdd51b9e..510ee14c 100644 --- a/wxdata/include/scwx/provider/nexrad_data_provider_factory.hpp +++ b/wxdata/include/scwx/provider/nexrad_data_provider_factory.hpp @@ -27,6 +27,9 @@ public: static std::shared_ptr CreateLevel2DataProvider(const std::string& radarSite); + static std::shared_ptr + CreateLevel2ChunksDataProvider(const std::string& radarSite); + static std::shared_ptr CreateLevel3DataProvider(const std::string& radarSite, const std::string& product); diff --git a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp index 3536c16c..7c692f7f 100644 --- a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp @@ -603,7 +603,13 @@ AwsLevel2ChunksDataProvider::LoadObjectByTime( std::shared_ptr AwsLevel2ChunksDataProvider::LoadLatestObject() { - return LoadObjectByTime(FindLatestTime()); + return p->currentScan_.nexradFile_; +} + +std::shared_ptr +AwsLevel2ChunksDataProvider::LoadSecondLatestObject() +{ + return p->lastScan_.nexradFile_; } int AwsLevel2ChunksDataProvider::Impl::GetScanNumber(const std::string& prefix) diff --git a/wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp b/wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp index 74740be0..c0611804 100644 --- a/wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp @@ -351,6 +351,12 @@ std::shared_ptr AwsNexradDataProvider::LoadLatestObject() return LoadObjectByKey(FindLatestKey()); } +std::shared_ptr +AwsNexradDataProvider::LoadSecondLatestObject() +{ + return nullptr; +} + std::pair AwsNexradDataProvider::Refresh() { using namespace std::chrono; diff --git a/wxdata/source/scwx/provider/nexrad_data_provider_factory.cpp b/wxdata/source/scwx/provider/nexrad_data_provider_factory.cpp index 83ccb1a3..dcaf7c9f 100644 --- a/wxdata/source/scwx/provider/nexrad_data_provider_factory.cpp +++ b/wxdata/source/scwx/provider/nexrad_data_provider_factory.cpp @@ -1,5 +1,5 @@ #include -//#include +#include #include #include @@ -14,6 +14,13 @@ static const std::string logPrefix_ = std::shared_ptr NexradDataProviderFactory::CreateLevel2DataProvider( const std::string& radarSite) +{ + return std::make_unique(radarSite); +} + +std::shared_ptr +NexradDataProviderFactory::CreateLevel2ChunksDataProvider( + const std::string& radarSite) { return std::make_unique(radarSite); } From add57ff26fb723e21cf1ec522ad07f55e3f1bef0 Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Fri, 4 Apr 2025 12:04:05 -0400 Subject: [PATCH 09/34] Minor updates to level2 chunks --- .../scwx/qt/manager/radar_product_manager.cpp | 2 +- .../aws_level2_chunks_data_provider.cpp | 32 +++++++++++++++---- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp index 0c7f10ea..bc29ed9a 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp @@ -645,7 +645,7 @@ void RadarProductManager::EnableRefresh(common::RadarProductGroup group, { if (group == common::RadarProductGroup::Level2) { - p->EnableRefresh(uuid, p->level2ProviderManager_, enabled); + //p->EnableRefresh(uuid, p->level2ProviderManager_, enabled); p->EnableRefresh(uuid, p->level2ChunksProviderManager_, enabled); } else diff --git a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp index 7c692f7f..19783965 100644 --- a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp @@ -189,13 +189,26 @@ size_t AwsLevel2ChunksDataProvider::cache_size() const std::chrono::system_clock::time_point AwsLevel2ChunksDataProvider::last_modified() const { - return p->currentScan_.lastModified_; + if (p->currentScan_.valid_ && p->currentScan_.lastModified_ != + std::chrono::system_clock::time_point {}) + { + return p->currentScan_.lastModified_; + } + else if (p->lastScan_.valid_ && p->lastScan_.lastModified_ != + std::chrono::system_clock::time_point {}) + { + return p->lastScan_.lastModified_; + } + else + { + return {}; + } } std::chrono::seconds AwsLevel2ChunksDataProvider::update_period() const { std::shared_lock lock(p->scansMutex_); // Add an extra second of delay - static const auto extra = std::chrono::seconds(1); + static const auto extra = std::chrono::seconds(2); // get update period from time between chunks if (p->currentScan_.valid_ && p->currentScan_.nextFile_ > 2) { @@ -317,7 +330,7 @@ AwsLevel2ChunksDataProvider::Impl::ListObjects() std::chrono::system_clock::time_point now = std::chrono::system_clock::now(); if (currentScan_.valid_ && !currentScan_.hasAllFiles_ && - lastTimeListed_ + std::chrono::minutes(7) > now) + lastTimeListed_ + std::chrono::minutes(2) > now) { return {true, newObjects, totalObjects}; } @@ -350,13 +363,13 @@ AwsLevel2ChunksDataProvider::Impl::ListObjects() scanPrefix); } - // TODO ensure not out of range int lastScanNumber = -1; // Start with last scan int previousScanNumber = scanNumberMap.crbegin()->first; const int firstScanNumber = scanNumberMap.cbegin()->first; // This indicates that highest number scan is the last scan + // (including if there is only 1 scan) // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) if (previousScanNumber != 999 || firstScanNumber != 1) { @@ -431,6 +444,7 @@ AwsLevel2ChunksDataProvider::Impl::ListObjects() currentScan_.hasAllFiles_ = false; newObjects += 1; } + logger_->error("{}", currentScan_.prefix_); } } @@ -479,6 +493,7 @@ bool AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) bool hasNew = false; auto& chunks = listOutcome.GetResult().GetContents(); + logger_->debug("Found {} new chunks.", chunks.size()); for (const auto& chunk : chunks) { const std::string& key = chunk.GetKey(); @@ -502,6 +517,11 @@ bool AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) // KIND/585/20250324-134727-001-S static const size_t charPos = std::string("/20250324-134727-001-").size(); + if (secondSlash + charPos >= key.size()) + { + logger_->warn("Chunk key was not long enough"); + continue; + } const char keyChar = key[secondSlash + charPos]; Aws::S3::Model::GetObjectRequest objectRequest; @@ -571,7 +591,7 @@ bool AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) { logger_->warn("Could not load file"); } - else + else if (hasNew) { scanRecord.nexradFile_->IndexFile(); } @@ -614,8 +634,6 @@ AwsLevel2ChunksDataProvider::LoadSecondLatestObject() int AwsLevel2ChunksDataProvider::Impl::GetScanNumber(const std::string& prefix) { - - // We just want the number of this chunk for now // KIND/585/20250324-134727-001-S static const size_t firstSlash = std::string("KIND/").size(); const std::string& prefixNumberStr = prefix.substr(firstSlash, 3); From ac6d6093ec5df81bac95789bc492065cd96cf6a7 Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Fri, 4 Apr 2025 19:35:12 -0400 Subject: [PATCH 10/34] updated how the most recent scan was determined to ensure correctness --- .../aws_level2_chunks_data_provider.cpp | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp index 19783965..09436bbe 100644 --- a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp @@ -368,16 +368,19 @@ AwsLevel2ChunksDataProvider::Impl::ListObjects() int previousScanNumber = scanNumberMap.crbegin()->first; const int firstScanNumber = scanNumberMap.cbegin()->first; + // Look for a gap in scan numbers. This indicates that is the latest + // scan. + // This indicates that highest number scan is the last scan // (including if there is only 1 scan) // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) - if (previousScanNumber != 999 || firstScanNumber != 1) + if (previousScanNumber != 999 || scans.size() == 1) { lastScanNumber = previousScanNumber; } else { - // have already checked scan with highest number, so skip first + // Have already checked scan with highest number, so skip first previousScanNumber = firstScanNumber; bool first = true; for (const auto& scan : scanNumberMap) @@ -398,9 +401,17 @@ AwsLevel2ChunksDataProvider::Impl::ListObjects() if (lastScanNumber == -1) { - logger_->warn("Could not find last scan"); - // TODO make sure this makes sence - return {false, 0, 0}; + // 999 is the last scan + if (firstScanNumber != 1) + { + lastScanNumber = previousScanNumber; + } + else + { + logger_->warn("Could not find last scan"); + // TODO make sure this makes sence + return {false, 0, 0}; + } } std::string& lastScanPrefix = scanNumberMap.at(lastScanNumber); @@ -444,7 +455,6 @@ AwsLevel2ChunksDataProvider::Impl::ListObjects() currentScan_.hasAllFiles_ = false; newObjects += 1; } - logger_->error("{}", currentScan_.prefix_); } } @@ -510,6 +520,10 @@ bool AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) const int keyNumber = std::stoi(keyNumberStr); if (keyNumber != scanRecord.nextFile_) { + logger_->warn("Chunk found that was not in order {} {} {}", + key, + scanRecord.nextFile_, + keyNumber); continue; } From 8b7a3e978126673fe326255f4605b24524c0c4c7 Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Sun, 6 Apr 2025 16:09:48 -0400 Subject: [PATCH 11/34] partiallaly complete merging of radar data --- .../scwx/qt/manager/radar_product_manager.cpp | 11 ++ wxdata/include/scwx/wsr88d/ar2v_file.hpp | 2 + .../aws_level2_chunks_data_provider.cpp | 2 - wxdata/source/scwx/wsr88d/ar2v_file.cpp | 144 ++++++++++++++++++ 4 files changed, 157 insertions(+), 2 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp index bc29ed9a..e6ede7b4 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp @@ -1525,6 +1525,16 @@ RadarProductManager::GetLevel2Data(wsr88d::rda::DataBlockType dataBlockType, //TODO decide when to use chunked vs archived data. if (true) { + auto currentFile = std::dynamic_pointer_cast( + p->level2ChunksProviderManager_->provider_->LoadLatestObject()); + auto lastFile = std::dynamic_pointer_cast( + p->level2ChunksProviderManager_->provider_->LoadSecondLatestObject()); + auto radarFile = + std::make_shared(currentFile, lastFile); + std::tie(radarData, elevationCut, elevationCuts) = + radarFile->GetElevationScan(dataBlockType, elevation, time); + + /* auto currentFile = std::dynamic_pointer_cast( p->level2ChunksProviderManager_->provider_->LoadLatestObject()); std::shared_ptr currentRadarData = nullptr; @@ -1587,6 +1597,7 @@ RadarProductManager::GetLevel2Data(wsr88d::rda::DataBlockType dataBlockType, elevationCuts = std::move(lastElevationCuts); foundTime = collectionTime; } + */ } else { diff --git a/wxdata/include/scwx/wsr88d/ar2v_file.hpp b/wxdata/include/scwx/wsr88d/ar2v_file.hpp index 34d50b32..9afca516 100644 --- a/wxdata/include/scwx/wsr88d/ar2v_file.hpp +++ b/wxdata/include/scwx/wsr88d/ar2v_file.hpp @@ -32,6 +32,8 @@ public: Ar2vFile(Ar2vFile&&) noexcept; Ar2vFile& operator=(Ar2vFile&&) noexcept; + Ar2vFile(std::shared_ptr current, std::shared_ptr last); + std::uint32_t julian_date() const; std::uint32_t milliseconds() const; std::string icao() const; diff --git a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp index 09436bbe..ef5c392e 100644 --- a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp @@ -676,12 +676,10 @@ std::pair AwsLevel2ChunksDataProvider::Refresh() } if (p->lastScan_.valid_) { - /* if (p->LoadScan(p->lastScan_)) { newObjects += 1; } - */ totalObjects += 1; } diff --git a/wxdata/source/scwx/wsr88d/ar2v_file.cpp b/wxdata/source/scwx/wsr88d/ar2v_file.cpp index db04feba..a5f46731 100644 --- a/wxdata/source/scwx/wsr88d/ar2v_file.cpp +++ b/wxdata/source/scwx/wsr88d/ar2v_file.cpp @@ -5,9 +5,11 @@ #include #include #include +#include #include #include +#include #if defined(_MSC_VER) # pragma warning(push) @@ -539,5 +541,147 @@ bool Ar2vFile::IndexFile() return true; } +// TODO not good +bool IsRadarDataIncomplete( + const std::shared_ptr& radarData) +{ + // Assume the data is incomplete when the delta between the first and last + // angles is greater than 2.5 degrees. + constexpr units::degrees kIncompleteDataAngleThreshold_ {2.5}; + + const units::degrees firstAngle = + radarData->cbegin()->second->azimuth_angle(); + const units::degrees lastAngle = + radarData->crbegin()->second->azimuth_angle(); + const units::degrees angleDelta = + common::GetAngleDelta(firstAngle, lastAngle); + + return angleDelta > kIncompleteDataAngleThreshold_; +} + +Ar2vFile::Ar2vFile(std::shared_ptr current, + std::shared_ptr last) : + Ar2vFile() +{ + /*p->vcpData_ = std::make_shared( + *current->vcp_data());*/ + p->vcpData_ = nullptr; // TODO + /* + use index_ to go through each block type, and elevation. + get the latest time. + if the latest time is not complete, get the previous time (possibly in + last), and merge + */ + + if (current == nullptr) + { + return; + } + + for (const auto& type : current->p->index_) + { + for (const auto& elevation : type.second) + { + const auto& mostRecent = elevation.second.crbegin(); + if (mostRecent == elevation.second.crend()) + { + continue; + } + + if (IsRadarDataIncomplete(mostRecent->second)) + { + std::shared_ptr secondMostRecent = + nullptr; + auto maybe = elevation.second.rbegin(); + ++maybe; + + if (maybe == elevation.second.rend()) + { + if (last == nullptr) + { + // Nothing to merge with + p->index_[type.first][elevation.first][mostRecent->first] = + mostRecent->second; + continue; + } + + auto elevationScan = + std::get>( + last->GetElevationScan(type.first, elevation.first, {})); + if (elevationScan == nullptr) + { + // Nothing to merge with + p->index_[type.first][elevation.first][mostRecent->first] = + mostRecent->second; + continue; + } + + secondMostRecent = elevationScan; + } + else + { + secondMostRecent = maybe->second; + } + + auto newScan = std::make_shared(); + + // Convert old into new coords + logger_->error( + "old {}, new {}", + secondMostRecent->cbegin()->second->azimuth_angle().value(), + mostRecent->second->cbegin()->second->azimuth_angle().value()); + // TODO Ordering these correctly + for (const auto& radial : *secondMostRecent) + { + (*newScan)[radial.first] = radial.second; + } + for (const auto& radial : *(mostRecent->second)) + { + (*newScan)[radial.first] = radial.second; + } + + p->index_[type.first][elevation.first][mostRecent->first] = + newScan; + } + else + { + p->index_[type.first][elevation.first][mostRecent->first] = + mostRecent->second; + } + } + } + + // Go though last, adding other elevations TODO + if (last != nullptr) + { + for (const auto& type : last->p->index_) + { + float highestCurrentElevation = -90; + const auto& maybe1 = p->index_.find(type.first); + if (maybe1 != p->index_.cend()) + { + const auto& maybe2 = maybe1->second.crbegin(); + if (maybe2 != maybe1->second.crend()) { + highestCurrentElevation = maybe2->first + 0.01; + } + } + for (const auto& elevation : type.second) + { + if (elevation.first > highestCurrentElevation) + { + const auto& mostRecent = elevation.second.crbegin(); + if (mostRecent == elevation.second.crend()) + { + continue; + } + p->index_[type.first][elevation.first][mostRecent->first] = + mostRecent->second; + } + } + } + } + +} + } // namespace wsr88d } // namespace scwx From 094d286b418afbf3cd85f11b18b527551e033adf Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Mon, 7 Apr 2025 12:36:06 -0400 Subject: [PATCH 12/34] fully working merging of data from last and current scan --- .../scwx/qt/manager/radar_product_manager.cpp | 76 +------ .../scwx/qt/view/level2_product_view.cpp | 3 +- wxdata/include/scwx/wsr88d/ar2v_file.hpp | 3 +- .../aws_level2_chunks_data_provider.cpp | 4 +- wxdata/source/scwx/wsr88d/ar2v_file.cpp | 207 +++++++++++------- 5 files changed, 140 insertions(+), 153 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp index e6ede7b4..12cf9d61 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp @@ -1520,87 +1520,17 @@ RadarProductManager::GetLevel2Data(wsr88d::rda::DataBlockType dataBlockType, std::vector elevationCuts {}; std::chrono::system_clock::time_point foundTime {}; - auto records = p->GetLevel2ProductRecords(time); - //TODO decide when to use chunked vs archived data. - if (true) + if constexpr (true) { auto currentFile = std::dynamic_pointer_cast( p->level2ChunksProviderManager_->provider_->LoadLatestObject()); - auto lastFile = std::dynamic_pointer_cast( - p->level2ChunksProviderManager_->provider_->LoadSecondLatestObject()); - auto radarFile = - std::make_shared(currentFile, lastFile); std::tie(radarData, elevationCut, elevationCuts) = - radarFile->GetElevationScan(dataBlockType, elevation, time); - - /* - auto currentFile = std::dynamic_pointer_cast( - p->level2ChunksProviderManager_->provider_->LoadLatestObject()); - std::shared_ptr currentRadarData = nullptr; - float currentElevationCut = 0.0f; - std::vector currentElevationCuts; - if (currentFile != nullptr) - { - std::tie(currentRadarData, currentElevationCut, currentElevationCuts) = - currentFile->GetElevationScan(dataBlockType, elevation, time); - } - - std::shared_ptr lastRadarData = nullptr; - float lastElevationCut = 0.0f; - std::vector lastElevationCuts; - auto lastFile = std::dynamic_pointer_cast( - p->level2ChunksProviderManager_->provider_->LoadSecondLatestObject()); - if (lastFile != nullptr) - { - std::tie(lastRadarData, lastElevationCut, lastElevationCuts) = - lastFile->GetElevationScan(dataBlockType, elevation, time); - } - - if (currentRadarData != nullptr) - { - if (lastRadarData != nullptr) - { - auto& radarData0 = (*currentRadarData)[0]; - auto collectionTime = std::chrono::floor( - scwx::util::TimePoint(radarData0->modified_julian_date(), - radarData0->collection_time())); - - // TODO merge data - radarData = currentRadarData; - elevationCut = currentElevationCut; - elevationCuts = std::move(currentElevationCuts); - foundTime = collectionTime; - } - else - { - auto& radarData0 = (*currentRadarData)[0]; - auto collectionTime = std::chrono::floor( - scwx::util::TimePoint(radarData0->modified_julian_date(), - radarData0->collection_time())); - - radarData = currentRadarData; - elevationCut = currentElevationCut; - elevationCuts = std::move(currentElevationCuts); - foundTime = collectionTime; - } - } - else if (lastRadarData != nullptr) - { - auto& radarData0 = (*lastRadarData)[0]; - auto collectionTime = std::chrono::floor( - scwx::util::TimePoint(radarData0->modified_julian_date(), - radarData0->collection_time())); - - radarData = lastRadarData; - elevationCut = lastElevationCut; - elevationCuts = std::move(lastElevationCuts); - foundTime = collectionTime; - } - */ + currentFile->GetElevationScan(dataBlockType, elevation, time); } else { + auto records = p->GetLevel2ProductRecords(time); for (auto& recordPair : records) { auto& record = recordPair.second; diff --git a/scwx-qt/source/scwx/qt/view/level2_product_view.cpp b/scwx-qt/source/scwx/qt/view/level2_product_view.cpp index 69f0dbf2..9ead358b 100644 --- a/scwx-qt/source/scwx/qt/view/level2_product_view.cpp +++ b/scwx-qt/source/scwx/qt/view/level2_product_view.cpp @@ -561,8 +561,7 @@ void Level2ProductView::ComputeSweep() Q_EMIT SweepNotComputed(types::NoUpdateReason::NotLoaded); return; } - // TODO do not do this when updating from live data - if (false && (radarData == p->elevationScan_) && + if ((radarData == p->elevationScan_) && smoothingEnabled == p->lastSmoothingEnabled_ && (showSmoothedRangeFolding == p->lastShowSmoothedRangeFolding_ || !smoothingEnabled)) diff --git a/wxdata/include/scwx/wsr88d/ar2v_file.hpp b/wxdata/include/scwx/wsr88d/ar2v_file.hpp index 9afca516..64319283 100644 --- a/wxdata/include/scwx/wsr88d/ar2v_file.hpp +++ b/wxdata/include/scwx/wsr88d/ar2v_file.hpp @@ -32,7 +32,8 @@ public: Ar2vFile(Ar2vFile&&) noexcept; Ar2vFile& operator=(Ar2vFile&&) noexcept; - Ar2vFile(std::shared_ptr current, std::shared_ptr last); + Ar2vFile(const std::shared_ptr& current, + const std::shared_ptr& last); std::uint32_t julian_date() const; std::uint32_t milliseconds() const; diff --git a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp index ef5c392e..0b18113f 100644 --- a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp @@ -637,7 +637,9 @@ AwsLevel2ChunksDataProvider::LoadObjectByTime( std::shared_ptr AwsLevel2ChunksDataProvider::LoadLatestObject() { - return p->currentScan_.nexradFile_; + return std::make_shared(p->currentScan_.nexradFile_, + p->lastScan_.nexradFile_); + //return p->currentScan_.nexradFile_; } std::shared_ptr diff --git a/wxdata/source/scwx/wsr88d/ar2v_file.cpp b/wxdata/source/scwx/wsr88d/ar2v_file.cpp index a5f46731..7e7e14e9 100644 --- a/wxdata/source/scwx/wsr88d/ar2v_file.cpp +++ b/wxdata/source/scwx/wsr88d/ar2v_file.cpp @@ -9,7 +9,6 @@ #include #include -#include #if defined(_MSC_VER) # pragma warning(push) @@ -559,114 +558,171 @@ bool IsRadarDataIncomplete( return angleDelta > kIncompleteDataAngleThreshold_; } -Ar2vFile::Ar2vFile(std::shared_ptr current, - std::shared_ptr last) : - Ar2vFile() +Ar2vFile::Ar2vFile(const std::shared_ptr& current, + const std::shared_ptr& last) : + Ar2vFile() { - /*p->vcpData_ = std::make_shared( - *current->vcp_data());*/ - p->vcpData_ = nullptr; // TODO - /* - use index_ to go through each block type, and elevation. - get the latest time. - if the latest time is not complete, get the previous time (possibly in - last), and merge - */ + // This is only used to index right now, so not a huge deal + p->vcpData_ = nullptr; - if (current == nullptr) + // Reconstruct index from the other's indexes + if (current != nullptr) { - return; - } - - for (const auto& type : current->p->index_) - { - for (const auto& elevation : type.second) + for (const auto& type : current->p->index_) { - const auto& mostRecent = elevation.second.crbegin(); - if (mostRecent == elevation.second.crend()) + for (const auto& elevation : type.second) { - continue; - } - - if (IsRadarDataIncomplete(mostRecent->second)) - { - std::shared_ptr secondMostRecent = - nullptr; - auto maybe = elevation.second.rbegin(); - ++maybe; - - if (maybe == elevation.second.rend()) + // Get the most recent scan + const auto& mostRecent = elevation.second.crbegin(); + if (mostRecent == elevation.second.crend()) { - if (last == nullptr) + continue; + } + + // Merge this scan with the last one if it is incomplete + if (IsRadarDataIncomplete(mostRecent->second)) + { + std::shared_ptr secondMostRecent = nullptr; + + // check if this volume scan has an earlier elevation scan + auto maybe = elevation.second.rbegin(); // TODO name + ++maybe; + + if (maybe == elevation.second.rend()) { - // Nothing to merge with - p->index_[type.first][elevation.first][mostRecent->first] = - mostRecent->second; - continue; + if (last == nullptr) + { + // Nothing to merge with + p->index_[type.first][elevation.first][mostRecent->first] = + mostRecent->second; + continue; + } + + // get the scan from the last scan + auto elevationScan = + std::get>( + last->GetElevationScan( + type.first, elevation.first, {})); + if (elevationScan == nullptr) + { + // Nothing to merge with + p->index_[type.first][elevation.first][mostRecent->first] = + mostRecent->second; + continue; + } + + secondMostRecent = elevationScan; + } + else + { + secondMostRecent = maybe->second; } - auto elevationScan = - std::get>( - last->GetElevationScan(type.first, elevation.first, {})); - if (elevationScan == nullptr) + // Make the new scan + auto newScan = std::make_shared(); + + // Copy over the new radials + for (const auto& radial : *(mostRecent->second)) { - // Nothing to merge with - p->index_[type.first][elevation.first][mostRecent->first] = - mostRecent->second; - continue; + (*newScan)[radial.first] = radial.second; } - secondMostRecent = elevationScan; + /* Correctly order the old radials. The radials need to be in + * order for the rendering to work, and the index needs to start + * at 0 and increase by one from there. Since the new radial + * should have index 0, the old radial needs to be reshaped to + * match the new radials indexing. + */ + + const double lowestAzm = + mostRecent->second->cbegin()->second->azimuth_angle().value(); + const double heighestAzm = mostRecent->second->crbegin() + ->second->azimuth_angle() + .value(); + std::uint16_t index = mostRecent->second->crbegin()->first + 1; + + // Sort by the azimuth. Makes the rest of this way easier + auto secondMostRecentAzmMap = + std::map>(); + for (const auto& radial : *secondMostRecent) + { + secondMostRecentAzmMap[radial.second->azimuth_angle() + .value()] = radial.second; + } + + if (lowestAzm <= heighestAzm) // New scan does not contain 0/360 + { + // Get the radials following the new radials + for (const auto& radial : secondMostRecentAzmMap) + { + if (radial.first > heighestAzm) + { + (*newScan)[index] = radial.second; + ++index; + } + } + // Get the radials before the new radials + for (const auto& radial : secondMostRecentAzmMap) + { + if (radial.first < lowestAzm) + { + (*newScan)[index] = radial.second; + ++index; + } + else + { + break; + } + } + } + else // New scan includes 0/360 + { + // The radials will already be in the right order + for (const auto& radial : secondMostRecentAzmMap) + { + if (radial.first > heighestAzm && radial.first < lowestAzm) + { + (*newScan)[index] = radial.second; + ++index; + } + } + } + + p->index_[type.first][elevation.first][mostRecent->first] = + newScan; } else { - secondMostRecent = maybe->second; + p->index_[type.first][elevation.first][mostRecent->first] = + mostRecent->second; } - - auto newScan = std::make_shared(); - - // Convert old into new coords - logger_->error( - "old {}, new {}", - secondMostRecent->cbegin()->second->azimuth_angle().value(), - mostRecent->second->cbegin()->second->azimuth_angle().value()); - // TODO Ordering these correctly - for (const auto& radial : *secondMostRecent) - { - (*newScan)[radial.first] = radial.second; - } - for (const auto& radial : *(mostRecent->second)) - { - (*newScan)[radial.first] = radial.second; - } - - p->index_[type.first][elevation.first][mostRecent->first] = - newScan; - } - else - { - p->index_[type.first][elevation.first][mostRecent->first] = - mostRecent->second; } } } - // Go though last, adding other elevations TODO + // Go though last, adding other elevations if (last != nullptr) { for (const auto& type : last->p->index_) { + // Find the highest elevation this type has for the current scan + // Start below any reasonable elevation + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) float highestCurrentElevation = -90; const auto& maybe1 = p->index_.find(type.first); if (maybe1 != p->index_.cend()) { const auto& maybe2 = maybe1->second.crbegin(); if (maybe2 != maybe1->second.crend()) { - highestCurrentElevation = maybe2->first + 0.01; + // Add a slight offset to ensure good floating point compare. + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) + highestCurrentElevation = maybe2->first + 0.01f; } } + for (const auto& elevation : type.second) { + // Only add elevations above the current scan's elevation if (elevation.first > highestCurrentElevation) { const auto& mostRecent = elevation.second.crbegin(); @@ -680,7 +736,6 @@ Ar2vFile::Ar2vFile(std::shared_ptr current, } } } - } } // namespace wsr88d From 63585af26d8b399531708fb86287798e9959d79f Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Mon, 7 Apr 2025 18:23:29 -0400 Subject: [PATCH 13/34] Get level2 chunks and archive working together, reduce logging of level2 chunks --- .../scwx/qt/manager/radar_product_manager.cpp | 40 +++++++++++++------ .../scwx/qt/map/radar_product_layer.cpp | 5 +-- .../aws_level2_chunks_data_provider.cpp | 9 +++-- wxdata/source/scwx/wsr88d/ar2v_file.cpp | 14 +++---- 4 files changed, 43 insertions(+), 25 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp index 12cf9d61..ec54b4cd 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp @@ -68,6 +68,7 @@ static constexpr std::size_t kTimerPlaces_ {6u}; static constexpr std::chrono::seconds kFastRetryInterval_ {15}; static constexpr std::chrono::seconds kFastRetryIntervalChunks_ {3}; static constexpr std::chrono::seconds kSlowRetryInterval_ {120}; +static constexpr std::chrono::seconds kSlowRetryIntervalChunks_ {20}; static std::unordered_map> instanceMap_; @@ -774,11 +775,15 @@ void RadarProductManagerImpl::RefreshDataSync( auto [newObjects, totalObjects] = providerManager->provider_->Refresh(); - // Level2 chunked data is updated quickly and uses a fater interval + // Level2 chunked data is updated quickly and uses a faster interval const std::chrono::milliseconds fastRetryInterval = - providerManager->group_ == common::RadarProductGroup::Level2 ? + providerManager == level2ChunksProviderManager_ ? kFastRetryIntervalChunks_ : kFastRetryInterval_; + const std::chrono::milliseconds slowRetryInterval = + providerManager == level2ChunksProviderManager_ ? + kSlowRetryIntervalChunks_ : + kSlowRetryInterval_; std::chrono::milliseconds interval = fastRetryInterval; if (totalObjects > 0) @@ -798,7 +803,7 @@ void RadarProductManagerImpl::RefreshDataSync( { // If it has been at least 5 update periods since the file has // been last modified, slow the retry period - interval = kSlowRetryInterval_; + interval = slowRetryInterval; } else if (interval < std::chrono::milliseconds {fastRetryInterval}) { @@ -817,7 +822,7 @@ void RadarProductManagerImpl::RefreshDataSync( logger_->info("[{}] No data found", providerManager->name()); // If no data is found, retry at the slow retry interval - interval = kSlowRetryInterval_; + interval = slowRetryInterval; } std::unique_lock const lock(providerManager->refreshTimerMutex_); @@ -953,11 +958,13 @@ void RadarProductManagerImpl::LoadProviderData( { existingRecord = it->second.lock(); + /* if (existingRecord != nullptr) { logger_->debug( "Data previously loaded, loading from data cache"); } + */ } } @@ -1416,7 +1423,7 @@ std::shared_ptr RadarProductManagerImpl::StoreRadarProductRecord( std::shared_ptr record) { - logger_->debug("StoreRadarProductRecord()"); + //logger_->debug("StoreRadarProductRecord()"); std::shared_ptr storedRecord = nullptr; @@ -1433,11 +1440,12 @@ RadarProductManagerImpl::StoreRadarProductRecord( { storedRecord = it->second.lock(); + /* if (storedRecord != nullptr) { logger_->debug( "Level 2 product previously loaded, loading from cache"); - } + }*/ } if (storedRecord == nullptr) @@ -1520,15 +1528,23 @@ RadarProductManager::GetLevel2Data(wsr88d::rda::DataBlockType dataBlockType, std::vector elevationCuts {}; std::chrono::system_clock::time_point foundTime {}; - //TODO decide when to use chunked vs archived data. - if constexpr (true) + // See if we have this one in the chunk provider. + auto chunkFile = std::dynamic_pointer_cast( + p->level2ChunksProviderManager_->provider_->LoadObjectByTime(time)); + if (chunkFile != nullptr) { - auto currentFile = std::dynamic_pointer_cast( - p->level2ChunksProviderManager_->provider_->LoadLatestObject()); std::tie(radarData, elevationCut, elevationCuts) = - currentFile->GetElevationScan(dataBlockType, elevation, time); + chunkFile->GetElevationScan(dataBlockType, elevation, time); + + if (radarData != nullptr) + { + auto& radarData0 = (*radarData)[0]; + foundTime = std::chrono::floor( + scwx::util::TimePoint(radarData0->modified_julian_date(), + radarData0->collection_time())); + } } - else + else // It is not in the chunk provider, so get it from the archive { auto records = p->GetLevel2ProductRecords(time); for (auto& recordPair : records) diff --git a/scwx-qt/source/scwx/qt/map/radar_product_layer.cpp b/scwx-qt/source/scwx/qt/map/radar_product_layer.cpp index 0d2b7125..8640ba03 100644 --- a/scwx-qt/source/scwx/qt/map/radar_product_layer.cpp +++ b/scwx-qt/source/scwx/qt/map/radar_product_layer.cpp @@ -159,8 +159,6 @@ void RadarProductLayer::Initialize() void RadarProductLayer::UpdateSweep() { - logger_->debug("UpdateSweep()"); - gl::OpenGLFunctions& gl = context()->gl(); boost::timer::cpu_timer timer; @@ -172,9 +170,10 @@ void RadarProductLayer::UpdateSweep() std::try_to_lock); if (!sweepLock.owns_lock()) { - logger_->debug("Sweep locked, deferring update"); + //logger_->debug("Sweep locked, deferring update"); return; } + logger_->debug("UpdateSweep()"); p->sweepNeedsUpdate_ = false; diff --git a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp index 0b18113f..02fdbca8 100644 --- a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp @@ -618,10 +618,13 @@ AwsLevel2ChunksDataProvider::LoadObjectByTime( std::chrono::system_clock::time_point time) { std::unique_lock lock(p->scansMutex_); + static const std::chrono::system_clock::time_point epoch {}; - if (p->currentScan_.valid_ && time >= p->currentScan_.time_) + if (p->currentScan_.valid_ && + (time == epoch || time >= p->currentScan_.time_)) { - return p->currentScan_.nexradFile_; + return std::make_shared(p->currentScan_.nexradFile_, + p->lastScan_.nexradFile_); } else if (p->lastScan_.valid_ && time >= p->lastScan_.time_) { @@ -629,7 +632,6 @@ AwsLevel2ChunksDataProvider::LoadObjectByTime( } else { - logger_->warn("Could not find scan with time"); return nullptr; } } @@ -637,6 +639,7 @@ AwsLevel2ChunksDataProvider::LoadObjectByTime( std::shared_ptr AwsLevel2ChunksDataProvider::LoadLatestObject() { + std::unique_lock lock(p->scansMutex_); return std::make_shared(p->currentScan_.nexradFile_, p->lastScan_.nexradFile_); //return p->currentScan_.nexradFile_; diff --git a/wxdata/source/scwx/wsr88d/ar2v_file.cpp b/wxdata/source/scwx/wsr88d/ar2v_file.cpp index 7e7e14e9..250341a0 100644 --- a/wxdata/source/scwx/wsr88d/ar2v_file.cpp +++ b/wxdata/source/scwx/wsr88d/ar2v_file.cpp @@ -138,7 +138,7 @@ Ar2vFile::GetElevationScan(rda::DataBlockType dataBlockType, float elevation, std::chrono::system_clock::time_point time) const { - logger_->debug("GetElevationScan: {} degrees", elevation); + //logger_->debug("GetElevationScan: {} degrees", elevation); std::shared_ptr elevationScan = nullptr; float elevationCut = 0.0f; @@ -273,7 +273,7 @@ bool Ar2vFile::LoadData(std::istream& is) std::size_t Ar2vFileImpl::DecompressLDMRecords(std::istream& is) { - logger_->debug("Decompressing LDM Records"); + //logger_->debug("Decompressing LDM Records"); std::size_t numRecords = 0; @@ -321,22 +321,22 @@ std::size_t Ar2vFileImpl::DecompressLDMRecords(std::istream& is) ++numRecords; } - logger_->debug("Decompressed {} LDM Records", numRecords); + //logger_->debug("Decompressed {} LDM Records", numRecords); return numRecords; } void Ar2vFileImpl::ParseLDMRecords() { - logger_->debug("Parsing LDM Records"); + //logger_->debug("Parsing LDM Records"); - std::size_t count = 0; + //std::size_t count = 0; for (auto it = rawRecords_.begin(); it != rawRecords_.end(); it++) { std::stringstream& ss = *it; - logger_->trace("Record {}", count++); + //logger_->trace("Record {}", count++); ParseLDMRecord(ss); } @@ -445,7 +445,7 @@ void Ar2vFileImpl::ProcessRadarData( void Ar2vFileImpl::IndexFile() { - logger_->debug("Indexing file"); + //logger_->debug("Indexing file"); constexpr float scaleFactor = 8.0f / 0.043945f; From 6ca76b9ecaea5edfd8554ee7b04e2694ecbe8409 Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Tue, 8 Apr 2025 10:41:44 -0400 Subject: [PATCH 14/34] Move elevation conversion code into VCP and DRD code --- wxdata/source/scwx/wsr88d/ar2v_file.cpp | 21 ++++--------------- .../scwx/wsr88d/rda/digital_radar_data.cpp | 13 +++++++++++- .../rda/volume_coverage_pattern_data.cpp | 15 ++++++++++++- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/wxdata/source/scwx/wsr88d/ar2v_file.cpp b/wxdata/source/scwx/wsr88d/ar2v_file.cpp index 250341a0..f804685b 100644 --- a/wxdata/source/scwx/wsr88d/ar2v_file.cpp +++ b/wxdata/source/scwx/wsr88d/ar2v_file.cpp @@ -447,11 +447,9 @@ void Ar2vFileImpl::IndexFile() { //logger_->debug("Indexing file"); - constexpr float scaleFactor = 8.0f / 0.043945f; - for (auto& elevationCut : radarData_) { - std::uint16_t elevationAngle {}; + float elevationAngle {}; rda::WaveformType waveformType = rda::WaveformType::Unknown; std::shared_ptr& radial0 = @@ -467,14 +465,14 @@ void Ar2vFileImpl::IndexFile() if (vcpData_ != nullptr) { - elevationAngle = vcpData_->elevation_angle_raw(elevationCut.first); + elevationAngle = vcpData_->elevation_angle(elevationCut.first); waveformType = vcpData_->waveform_type(elevationCut.first); } else if ((digitalRadarData0 = std::dynamic_pointer_cast(radial0)) != nullptr) { - elevationAngle = digitalRadarData0->elevation_angle_raw(); + elevationAngle = digitalRadarData0->elevation_angle().value(); } else { @@ -502,18 +500,7 @@ void Ar2vFileImpl::IndexFile() auto time = util::TimePoint(radial0->modified_julian_date(), radial0->collection_time()); - // NOLINTNEXTLINE This conversion is accurate - float elevationAngleConverted = elevationAngle / scaleFactor; - // Any elevation above 90 degrees should be interpreted as a - // negative angle - // NOLINTBEGIN(cppcoreguidelines-avoid-magic-numbers) - if (elevationAngleConverted > 90) - { - elevationAngleConverted -= 360; - } - // NOLINTEND(cppcoreguidelines-avoid-magic-numbers) - - index_[dataBlockType][elevationAngleConverted][time] = + index_[dataBlockType][elevationAngle][time] = elevationCut.second; } } diff --git a/wxdata/source/scwx/wsr88d/rda/digital_radar_data.cpp b/wxdata/source/scwx/wsr88d/rda/digital_radar_data.cpp index ebadf4f5..8f2643af 100644 --- a/wxdata/source/scwx/wsr88d/rda/digital_radar_data.cpp +++ b/wxdata/source/scwx/wsr88d/rda/digital_radar_data.cpp @@ -154,7 +154,18 @@ std::uint16_t DigitalRadarData::elevation_angle_raw() const units::degrees DigitalRadarData::elevation_angle() const { - return units::degrees {p->elevationAngle_ * kAngleDataScale}; + // NOLINTNEXTLINE This conversion is accurate + float elevationAngleConverted = p->elevationAngle_ * kAngleDataScale; + // Any elevation above 90 degrees should be interpreted as a + // negative angle + // NOLINTBEGIN(cppcoreguidelines-avoid-magic-numbers) + if (elevationAngleConverted > 90) + { + elevationAngleConverted -= 360; + } + // NOLINTEND(cppcoreguidelines-avoid-magic-numbers) + + return units::degrees {elevationAngleConverted}; } std::uint16_t DigitalRadarData::elevation_number() const diff --git a/wxdata/source/scwx/wsr88d/rda/volume_coverage_pattern_data.cpp b/wxdata/source/scwx/wsr88d/rda/volume_coverage_pattern_data.cpp index d2f3dec2..42ed3685 100644 --- a/wxdata/source/scwx/wsr88d/rda/volume_coverage_pattern_data.cpp +++ b/wxdata/source/scwx/wsr88d/rda/volume_coverage_pattern_data.cpp @@ -220,7 +220,20 @@ uint16_t VolumeCoveragePatternData::number_of_base_tilts() const double VolumeCoveragePatternData::elevation_angle(uint16_t e) const { - return p->elevationCuts_[e].elevationAngle_ * ANGLE_DATA_SCALE; + + // NOLINTNEXTLINE This conversion is accurate + float elevationAngleConverted = + p->elevationCuts_[e].elevationAngle_ * ANGLE_DATA_SCALE; + // Any elevation above 90 degrees should be interpreted as a + // negative angle + // NOLINTBEGIN(cppcoreguidelines-avoid-magic-numbers) + if (elevationAngleConverted > 90) + { + elevationAngleConverted -= 360; + } + // NOLINTEND(cppcoreguidelines-avoid-magic-numbers) + + return elevationAngleConverted; } uint16_t VolumeCoveragePatternData::elevation_angle_raw(uint16_t e) const From 0bda6296c0982b1f65bf820477754d79717c61d8 Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Tue, 8 Apr 2025 12:22:47 -0400 Subject: [PATCH 15/34] Add indicator of what level is currently being updated with level 2 chunks. --- scwx-qt/source/scwx/qt/main/main_window.cpp | 7 ++++ .../scwx/qt/manager/radar_product_manager.cpp | 24 ++++++++++++ .../scwx/qt/manager/radar_product_manager.hpp | 10 +++-- scwx-qt/source/scwx/qt/map/map_widget.cpp | 18 +++++++++ scwx-qt/source/scwx/qt/map/map_widget.hpp | 2 + .../scwx/qt/ui/level2_settings_widget.cpp | 15 +++++++ .../scwx/qt/ui/level2_settings_widget.hpp | 1 + .../aws_level2_chunks_data_provider.hpp | 2 + .../aws_level2_chunks_data_provider.cpp | 39 +++++++++++++++++++ 9 files changed, 114 insertions(+), 4 deletions(-) diff --git a/scwx-qt/source/scwx/qt/main/main_window.cpp b/scwx-qt/source/scwx/qt/main/main_window.cpp index d213b4fd..089d6749 100644 --- a/scwx-qt/source/scwx/qt/main/main_window.cpp +++ b/scwx-qt/source/scwx/qt/main/main_window.cpp @@ -963,6 +963,13 @@ void MainWindowImpl::ConnectMapSignals() } }, Qt::QueuedConnection); + connect( + mapWidget, + &map::MapWidget::IncomingLevel2ElevationChanged, + this, + [this](float incomingElevation) + { level2SettingsWidget_->UpdateIncomingElevation(incomingElevation); }, + Qt::QueuedConnection); } } diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp index ec54b4cd..ed952ed7 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -271,6 +272,8 @@ public: common::Level3ProductCategoryMap availableCategoryMap_ {}; std::shared_mutex availableCategoryMutex_ {}; + float incomingLevel2Elevation_ {-90}; + std::unordered_map, boost::hash> @@ -450,6 +453,11 @@ float RadarProductManager::gate_size() const return (is_tdwr()) ? 150.0f : 250.0f; } +float RadarProductManager::incoming_level_2_elevation() const +{ + return p->incomingLevel2Elevation_; +} + std::string RadarProductManager::radar_id() const { return p->radarId_; @@ -1542,6 +1550,16 @@ RadarProductManager::GetLevel2Data(wsr88d::rda::DataBlockType dataBlockType, foundTime = std::chrono::floor( scwx::util::TimePoint(radarData0->modified_julian_date(), radarData0->collection_time())); + + const float incomingElevation = + std::dynamic_pointer_cast( + p->level2ChunksProviderManager_->provider_) + ->GetCurrentElevation(); + if (incomingElevation != p->incomingLevel2Elevation_) + { + p->incomingLevel2Elevation_ = incomingElevation; + Q_EMIT IncomingLevel2ElevationChanged(incomingElevation); + } } } else // It is not in the chunk provider, so get it from the archive @@ -1576,6 +1594,12 @@ RadarProductManager::GetLevel2Data(wsr88d::rda::DataBlockType dataBlockType, elevationCut = recordElevationCut; elevationCuts = std::move(recordElevationCuts); foundTime = collectionTime; + + if (p->incomingLevel2Elevation_ != -90) + { + p->incomingLevel2Elevation_ = -90; + Q_EMIT IncomingLevel2ElevationChanged(-90); + } } } } diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.hpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.hpp index ee54f147..6efd125d 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.hpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.hpp @@ -43,10 +43,11 @@ public: [[nodiscard]] const std::vector& coordinates(common::RadialSize radialSize, bool smoothingEnabled) const; - [[nodiscard]] const scwx::util::time_zone* default_time_zone() const; - [[nodiscard]] bool is_tdwr() const; - [[nodiscard]] float gate_size() const; - [[nodiscard]] std::string radar_id() const; + [[nodiscard]] const scwx::util::time_zone* default_time_zone() const; + [[nodiscard]] float gate_size() const; + [[nodiscard]] float incoming_level_2_elevation() const; + [[nodiscard]] bool is_tdwr() const; + [[nodiscard]] std::string radar_id() const; [[nodiscard]] std::shared_ptr radar_site() const; void Initialize(); @@ -148,6 +149,7 @@ signals: void NewDataAvailable(common::RadarProductGroup group, const std::string& product, std::chrono::system_clock::time_point latestTime); + void IncomingLevel2ElevationChanged(float incomingElevation); private: std::unique_ptr p; diff --git a/scwx-qt/source/scwx/qt/map/map_widget.cpp b/scwx-qt/source/scwx/qt/map/map_widget.cpp index dafa801c..edb4c3d2 100644 --- a/scwx-qt/source/scwx/qt/map/map_widget.cpp +++ b/scwx-qt/source/scwx/qt/map/map_widget.cpp @@ -656,6 +656,12 @@ std::vector MapWidget::GetElevationCuts() const } } +float MapWidget::GetIncomingLevel2Elevation() const +{ + return p->radarProductManager_->incoming_level_2_elevation(); +} + + common::Level2Product MapWidgetImpl::GetLevel2ProductOrDefault(const std::string& productName) const { @@ -1796,6 +1802,14 @@ void MapWidgetImpl::RadarProductManagerConnect() { if (radarProductManager_ != nullptr) { + connect(radarProductManager_.get(), + &manager::RadarProductManager::IncomingLevel2ElevationChanged, + this, + [this](float incomingElevation) + { + Q_EMIT widget_->IncomingLevel2ElevationChanged( + incomingElevation); + }); connect(radarProductManager_.get(), &manager::RadarProductManager::Level3ProductsChanged, this, @@ -1916,6 +1930,10 @@ void MapWidgetImpl::RadarProductManagerDisconnect() &manager::RadarProductManager::NewDataAvailable, this, nullptr); + disconnect(radarProductManager_.get(), + &manager::RadarProductManager::IncomingLevel2ElevationChanged, + this, + nullptr); } } diff --git a/scwx-qt/source/scwx/qt/map/map_widget.hpp b/scwx-qt/source/scwx/qt/map/map_widget.hpp index 845832e4..5cc2e0a1 100644 --- a/scwx-qt/source/scwx/qt/map/map_widget.hpp +++ b/scwx-qt/source/scwx/qt/map/map_widget.hpp @@ -45,6 +45,7 @@ public: GetAvailableLevel3Categories(); [[nodiscard]] std::optional GetElevation() const; [[nodiscard]] std::vector GetElevationCuts() const; + [[nodiscard]] float GetIncomingLevel2Elevation() const; [[nodiscard]] std::vector GetLevel3Products(); [[nodiscard]] std::string GetMapStyle() const; [[nodiscard]] common::RadarProductGroup GetRadarProductGroup() const; @@ -184,6 +185,7 @@ signals: void RadarSweepUpdated(); void RadarSweepNotUpdated(types::NoUpdateReason reason); void WidgetPainted(); + void IncomingLevel2ElevationChanged(float incomingElevation); }; } // namespace map diff --git a/scwx-qt/source/scwx/qt/ui/level2_settings_widget.cpp b/scwx-qt/source/scwx/qt/ui/level2_settings_widget.cpp index 1b851e72..4a3d0967 100644 --- a/scwx-qt/source/scwx/qt/ui/level2_settings_widget.cpp +++ b/scwx-qt/source/scwx/qt/ui/level2_settings_widget.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -30,6 +31,7 @@ public: self_ {self}, layout_ {new QVBoxLayout(self)}, elevationGroupBox_ {}, + incomingElevationLabel_ {}, elevationButtons_ {}, elevationCuts_ {}, elevationButtonsChanged_ {false}, @@ -39,10 +41,14 @@ public: { layout_->setContentsMargins(0, 0, 0, 0); + incomingElevationLabel_ = new QLabel("", self); + layout_->addWidget(incomingElevationLabel_); + elevationGroupBox_ = new QGroupBox(tr("Elevation"), self); new ui::FlowLayout(elevationGroupBox_); layout_->addWidget(elevationGroupBox_); + settingsGroupBox_ = new QGroupBox(tr("Settings"), self); QLayout* settingsLayout = new QVBoxLayout(settingsGroupBox_); layout_->addWidget(settingsGroupBox_); @@ -67,6 +73,7 @@ public: QLayout* layout_; QGroupBox* elevationGroupBox_; + QLabel* incomingElevationLabel_; std::list elevationButtons_; std::vector elevationCuts_; bool elevationButtonsChanged_; @@ -240,12 +247,19 @@ void Level2SettingsWidget::UpdateElevationSelection(float elevation) p->currentElevationButton_ = newElevationButton; } +void Level2SettingsWidget::UpdateIncomingElevation(float incomingElevation) +{ + p->incomingElevationLabel_->setText("Incoming Elevation: " + + QString::number(incomingElevation, 'f', 1) + common::Characters::DEGREE); +} + void Level2SettingsWidget::UpdateSettings(map::MapWidget* activeMap) { std::optional currentElevationOption = activeMap->GetElevation(); const float currentElevation = currentElevationOption.has_value() ? *currentElevationOption : 0.0f; std::vector elevationCuts = activeMap->GetElevationCuts(); + float incomingElevation = activeMap->GetIncomingLevel2Elevation(); if (p->elevationCuts_ != elevationCuts) { @@ -279,6 +293,7 @@ void Level2SettingsWidget::UpdateSettings(map::MapWidget* activeMap) } UpdateElevationSelection(currentElevation); + UpdateIncomingElevation(incomingElevation); } } // namespace ui diff --git a/scwx-qt/source/scwx/qt/ui/level2_settings_widget.hpp b/scwx-qt/source/scwx/qt/ui/level2_settings_widget.hpp index ce2e443f..796b1ade 100644 --- a/scwx-qt/source/scwx/qt/ui/level2_settings_widget.hpp +++ b/scwx-qt/source/scwx/qt/ui/level2_settings_widget.hpp @@ -23,6 +23,7 @@ public: void showEvent(QShowEvent* event) override; void UpdateElevationSelection(float elevation); + void UpdateIncomingElevation(float incomingElevation); void UpdateSettings(map::MapWidget* activeMap); signals: diff --git a/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp b/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp index 106b6605..a82dffb2 100644 --- a/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp +++ b/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp @@ -57,6 +57,8 @@ public: void RequestAvailableProducts() override; std::vector GetAvailableProducts() override; + float GetCurrentElevation(); + private: class Impl; std::unique_ptr p; diff --git a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp index 02fdbca8..0d97c7b8 100644 --- a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp @@ -1,3 +1,4 @@ +#include "scwx/wsr88d/rda/digital_radar_data.hpp" #include #include #include @@ -704,4 +705,42 @@ AwsLevel2ChunksDataProvider::AwsLevel2ChunksDataProvider( AwsLevel2ChunksDataProvider& AwsLevel2ChunksDataProvider::operator=( AwsLevel2ChunksDataProvider&&) noexcept = default; +float AwsLevel2ChunksDataProvider::GetCurrentElevation() +{ + if (!p->currentScan_.valid_ || p->currentScan_.nexradFile_ == nullptr) + { + // Does not have any scan elevation. -90 is beyond what is possible + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) + return -90; + } + + auto vcpData = p->currentScan_.nexradFile_->vcp_data(); + auto radarData = p->currentScan_.nexradFile_->radar_data(); + if (radarData.size() == 0) + { + // Does not have any scan elevation. -90 is beyond what is possible + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) + return -90; + } + + const auto& lastElevation = radarData.crbegin(); + std::shared_ptr digitalRadarData0 = + std::dynamic_pointer_cast( + lastElevation->second->cbegin()->second); + + if (vcpData != nullptr) + { + // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions) Float is plenty + return vcpData->elevation_angle(lastElevation->first); + } + else if (digitalRadarData0 != nullptr) + { + return digitalRadarData0->elevation_angle().value(); + } + + // Does not have any scan elevation. -90 is beyond what is possible + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) + return -90; +} + } // namespace scwx::provider From 0f95439b61621e5a20a472edfeaefc950612434e Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Tue, 8 Apr 2025 13:49:41 -0400 Subject: [PATCH 16/34] Initial clang format/tidy fixes for level_2_chunks --- .../scwx/qt/manager/radar_product_manager.cpp | 24 ++-- scwx-qt/source/scwx/qt/map/map_widget.cpp | 1 - .../scwx/qt/map/radar_product_layer.cpp | 2 +- .../scwx/qt/ui/level2_settings_widget.cpp | 32 +++-- .../aws_level2_chunks_data_provider.hpp | 1 + .../scwx/provider/nexrad_data_provider.hpp | 7 +- .../aws_level2_chunks_data_provider.cpp | 111 +++++++++--------- wxdata/source/scwx/wsr88d/ar2v_file.cpp | 31 ++--- .../rda/volume_coverage_pattern_data.cpp | 2 +- 9 files changed, 108 insertions(+), 103 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp index ed952ed7..39501b29 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp @@ -272,7 +272,8 @@ public: common::Level3ProductCategoryMap availableCategoryMap_ {}; std::shared_mutex availableCategoryMutex_ {}; - float incomingLevel2Elevation_ {-90}; + float incomingLevel2Elevation_ { + provider::AwsLevel2ChunksDataProvider::INVALID_ELEVATION}; std::unordered_map, @@ -654,7 +655,7 @@ void RadarProductManager::EnableRefresh(common::RadarProductGroup group, { if (group == common::RadarProductGroup::Level2) { - //p->EnableRefresh(uuid, p->level2ProviderManager_, enabled); + // p->EnableRefresh(uuid, p->level2ProviderManager_, enabled); p->EnableRefresh(uuid, p->level2ChunksProviderManager_, enabled); } else @@ -1431,7 +1432,7 @@ std::shared_ptr RadarProductManagerImpl::StoreRadarProductRecord( std::shared_ptr record) { - //logger_->debug("StoreRadarProductRecord()"); + // logger_->debug("StoreRadarProductRecord()"); std::shared_ptr storedRecord = nullptr; @@ -1571,9 +1572,10 @@ RadarProductManager::GetLevel2Data(wsr88d::rda::DataBlockType dataBlockType, if (record != nullptr) { - std::shared_ptr recordRadarData = nullptr; - float recordElevationCut = 0.0f; - std::vector recordElevationCuts; + std::shared_ptr recordRadarData = + nullptr; + float recordElevationCut = 0.0f; + std::vector recordElevationCuts; std::tie(recordRadarData, recordElevationCut, recordElevationCuts) = record->level2_file()->GetElevationScan( @@ -1595,10 +1597,14 @@ RadarProductManager::GetLevel2Data(wsr88d::rda::DataBlockType dataBlockType, elevationCuts = std::move(recordElevationCuts); foundTime = collectionTime; - if (p->incomingLevel2Elevation_ != -90) + if (p->incomingLevel2Elevation_ != + provider::AwsLevel2ChunksDataProvider::INVALID_ELEVATION) { - p->incomingLevel2Elevation_ = -90; - Q_EMIT IncomingLevel2ElevationChanged(-90); + p->incomingLevel2Elevation_ = provider:: + AwsLevel2ChunksDataProvider::INVALID_ELEVATION; + Q_EMIT IncomingLevel2ElevationChanged( + provider::AwsLevel2ChunksDataProvider:: + INVALID_ELEVATION); } } } diff --git a/scwx-qt/source/scwx/qt/map/map_widget.cpp b/scwx-qt/source/scwx/qt/map/map_widget.cpp index edb4c3d2..845e0a51 100644 --- a/scwx-qt/source/scwx/qt/map/map_widget.cpp +++ b/scwx-qt/source/scwx/qt/map/map_widget.cpp @@ -661,7 +661,6 @@ float MapWidget::GetIncomingLevel2Elevation() const return p->radarProductManager_->incoming_level_2_elevation(); } - common::Level2Product MapWidgetImpl::GetLevel2ProductOrDefault(const std::string& productName) const { diff --git a/scwx-qt/source/scwx/qt/map/radar_product_layer.cpp b/scwx-qt/source/scwx/qt/map/radar_product_layer.cpp index 8640ba03..ee6b3c3c 100644 --- a/scwx-qt/source/scwx/qt/map/radar_product_layer.cpp +++ b/scwx-qt/source/scwx/qt/map/radar_product_layer.cpp @@ -170,7 +170,7 @@ void RadarProductLayer::UpdateSweep() std::try_to_lock); if (!sweepLock.owns_lock()) { - //logger_->debug("Sweep locked, deferring update"); + // logger_->debug("Sweep locked, deferring update"); return; } logger_->debug("UpdateSweep()"); diff --git a/scwx-qt/source/scwx/qt/ui/level2_settings_widget.cpp b/scwx-qt/source/scwx/qt/ui/level2_settings_widget.cpp index 4a3d0967..78e4de37 100644 --- a/scwx-qt/source/scwx/qt/ui/level2_settings_widget.cpp +++ b/scwx-qt/source/scwx/qt/ui/level2_settings_widget.cpp @@ -30,15 +30,10 @@ public: explicit Level2SettingsWidgetImpl(Level2SettingsWidget* self) : self_ {self}, layout_ {new QVBoxLayout(self)}, - elevationGroupBox_ {}, - incomingElevationLabel_ {}, elevationButtons_ {}, - elevationCuts_ {}, - elevationButtonsChanged_ {false}, - resizeElevationButtons_ {false}, - settingsGroupBox_ {}, - declutterCheckBox_ {} + elevationCuts_ {} { + // NOLINTBEGIN(cppcoreguidelines-owning-memory) Qt takes care of this layout_->setContentsMargins(0, 0, 0, 0); incomingElevationLabel_ = new QLabel("", self); @@ -48,7 +43,6 @@ public: new ui::FlowLayout(elevationGroupBox_); layout_->addWidget(elevationGroupBox_); - settingsGroupBox_ = new QGroupBox(tr("Settings"), self); QLayout* settingsLayout = new QVBoxLayout(settingsGroupBox_); layout_->addWidget(settingsGroupBox_); @@ -57,6 +51,7 @@ public: settingsLayout->addWidget(declutterCheckBox_); settingsGroupBox_->setVisible(false); + // NOLINTEND(cppcoreguidelines-owning-memory) Qt takes care of this QObject::connect(hotkeyManager_.get(), &manager::HotkeyManager::HotkeyPressed, @@ -72,15 +67,15 @@ public: Level2SettingsWidget* self_; QLayout* layout_; - QGroupBox* elevationGroupBox_; - QLabel* incomingElevationLabel_; + QGroupBox* elevationGroupBox_ {}; + QLabel* incomingElevationLabel_ {}; std::list elevationButtons_; std::vector elevationCuts_; - bool elevationButtonsChanged_; - bool resizeElevationButtons_; + bool elevationButtonsChanged_ {}; + bool resizeElevationButtons_ {}; - QGroupBox* settingsGroupBox_; - QCheckBox* declutterCheckBox_; + QGroupBox* settingsGroupBox_ {}; + QCheckBox* declutterCheckBox_ {}; float currentElevation_ {}; QToolButton* currentElevationButton_ {nullptr}; @@ -249,8 +244,9 @@ void Level2SettingsWidget::UpdateElevationSelection(float elevation) void Level2SettingsWidget::UpdateIncomingElevation(float incomingElevation) { - p->incomingElevationLabel_->setText("Incoming Elevation: " + - QString::number(incomingElevation, 'f', 1) + common::Characters::DEGREE); + p->incomingElevationLabel_->setText( + "Incoming Elevation: " + QString::number(incomingElevation, 'f', 1) + + common::Characters::DEGREE); } void Level2SettingsWidget::UpdateSettings(map::MapWidget* activeMap) @@ -258,8 +254,8 @@ void Level2SettingsWidget::UpdateSettings(map::MapWidget* activeMap) std::optional currentElevationOption = activeMap->GetElevation(); const float currentElevation = currentElevationOption.has_value() ? *currentElevationOption : 0.0f; - std::vector elevationCuts = activeMap->GetElevationCuts(); - float incomingElevation = activeMap->GetIncomingLevel2Elevation(); + std::vector elevationCuts = activeMap->GetElevationCuts(); + const float incomingElevation = activeMap->GetIncomingLevel2Elevation(); if (p->elevationCuts_ != elevationCuts) { diff --git a/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp b/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp index a82dffb2..052f639b 100644 --- a/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp +++ b/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp @@ -16,6 +16,7 @@ namespace scwx::provider class AwsLevel2ChunksDataProvider : public NexradDataProvider { public: + constexpr static const float INVALID_ELEVATION = -90.0; explicit AwsLevel2ChunksDataProvider(const std::string& radarSite); explicit AwsLevel2ChunksDataProvider(const std::string& radarSite, const std::string& bucketName, diff --git a/wxdata/include/scwx/provider/nexrad_data_provider.hpp b/wxdata/include/scwx/provider/nexrad_data_provider.hpp index a0e09f04..dc490be0 100644 --- a/wxdata/include/scwx/provider/nexrad_data_provider.hpp +++ b/wxdata/include/scwx/provider/nexrad_data_provider.hpp @@ -103,17 +103,14 @@ public: * * @return NEXRAD data */ - virtual std::shared_ptr - LoadLatestObject() = 0; + virtual std::shared_ptr LoadLatestObject() = 0; /** * Loads the second NEXRAD file object * * @return NEXRAD data */ - virtual std::shared_ptr - LoadSecondLatestObject() = 0; - + virtual std::shared_ptr LoadSecondLatestObject() = 0; /** * Lists NEXRAD objects for the current date, and adds them to the cache. If diff --git a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp index 0d97c7b8..16fb0e44 100644 --- a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp @@ -73,6 +73,7 @@ public: currentScan_ {"", false}, scansMutex_ {}, lastTimeListed_ {}, + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) about average updatePeriod_ {7}, self_ {self} { @@ -80,10 +81,11 @@ public: util::SetEnvironment("AWS_EC2_METADATA_DISABLED", "true"); // Use anonymous credentials - Aws::Auth::AWSCredentials credentials {}; + const Aws::Auth::AWSCredentials credentials {}; Aws::Client::ClientConfiguration config; - config.region = region_; + config.region = region_; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) arbitrary config.connectTimeoutMs = 10000; client_ = std::make_shared( @@ -104,10 +106,9 @@ public: const std::chrono::system_clock::time_point& time, int last); - bool LoadScan(Impl::ScanRecord& scanRecord); + bool LoadScan(Impl::ScanRecord& scanRecord); std::tuple ListObjects(); - std::string radarSite_; std::string bucketName_; std::string region_; @@ -190,6 +191,7 @@ size_t AwsLevel2ChunksDataProvider::cache_size() const std::chrono::system_clock::time_point AwsLevel2ChunksDataProvider::last_modified() const { + const std::shared_lock lock(p->scansMutex_); if (p->currentScan_.valid_ && p->currentScan_.lastModified_ != std::chrono::system_clock::time_point {}) { @@ -207,7 +209,7 @@ AwsLevel2ChunksDataProvider::last_modified() const } std::chrono::seconds AwsLevel2ChunksDataProvider::update_period() const { - std::shared_lock lock(p->scansMutex_); + const std::shared_lock lock(p->scansMutex_); // Add an extra second of delay static const auto extra = std::chrono::seconds(2); // get update period from time between chunks @@ -233,7 +235,7 @@ AwsLevel2ChunksDataProvider::FindKey(std::chrono::system_clock::time_point time) { logger_->debug("FindKey: {}", util::TimeString(time)); - std::shared_lock lock(p->scansMutex_); + const std::shared_lock lock(p->scansMutex_); if (p->currentScan_.valid_ && time >= p->currentScan_.time_) { return p->currentScan_.prefix_; @@ -248,7 +250,7 @@ AwsLevel2ChunksDataProvider::FindKey(std::chrono::system_clock::time_point time) std::string AwsLevel2ChunksDataProvider::FindLatestKey() { - std::shared_lock lock(p->scansMutex_); + const std::shared_lock lock(p->scansMutex_); if (!p->currentScan_.valid_) { return ""; @@ -260,7 +262,7 @@ std::string AwsLevel2ChunksDataProvider::FindLatestKey() std::chrono::system_clock::time_point AwsLevel2ChunksDataProvider::FindLatestTime() { - std::shared_lock lock(p->scansMutex_); + const std::shared_lock lock(p->scansMutex_); if (!p->currentScan_.valid_) { return {}; @@ -325,10 +327,11 @@ std::string AwsLevel2ChunksDataProvider::Impl::GetScanKey( std::tuple AwsLevel2ChunksDataProvider::Impl::ListObjects() { - size_t newObjects = 0; - size_t totalObjects = 0; + size_t newObjects = 0; + const size_t totalObjects = 0; - std::chrono::system_clock::time_point now = std::chrono::system_clock::now(); + const std::chrono::system_clock::time_point now = + std::chrono::system_clock::now(); if (currentScan_.valid_ && !currentScan_.hasAllFiles_ && lastTimeListed_ + std::chrono::minutes(2) > now) @@ -366,8 +369,8 @@ AwsLevel2ChunksDataProvider::Impl::ListObjects() int lastScanNumber = -1; // Start with last scan - int previousScanNumber = scanNumberMap.crbegin()->first; - const int firstScanNumber = scanNumberMap.cbegin()->first; + int previousScanNumber = scanNumberMap.crbegin()->first; + const int firstScanNumber = scanNumberMap.cbegin()->first; // Look for a gap in scan numbers. This indicates that is the latest // scan. @@ -383,7 +386,7 @@ AwsLevel2ChunksDataProvider::Impl::ListObjects() { // Have already checked scan with highest number, so skip first previousScanNumber = firstScanNumber; - bool first = true; + bool first = true; for (const auto& scan : scanNumberMap) { if (first) @@ -415,15 +418,16 @@ AwsLevel2ChunksDataProvider::Impl::ListObjects() } } - std::string& lastScanPrefix = scanNumberMap.at(lastScanNumber); - int secondLastScanNumber = + const std::string& lastScanPrefix = scanNumberMap.at(lastScanNumber); + const int secondLastScanNumber = + // 999 is the last file possible + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) lastScanNumber == 1 ? 999 : lastScanNumber - 1; const auto& secondLastScanPrefix = scanNumberMap.find(secondLastScanNumber); - if (!currentScan_.valid_ || - currentScan_.prefix_ != lastScanPrefix) + if (!currentScan_.valid_ || currentScan_.prefix_ != lastScanPrefix) { if (currentScan_.valid_ && (secondLastScanPrefix == scanNumberMap.cend() || @@ -445,9 +449,9 @@ AwsLevel2ChunksDataProvider::Impl::ListObjects() newObjects += 1; } - currentScan_.valid_ = true; - currentScan_.prefix_ = lastScanPrefix; - currentScan_.nexradFile_ = nullptr; + currentScan_.valid_ = true; + currentScan_.prefix_ = lastScanPrefix; + currentScan_.nexradFile_ = nullptr; currentScan_.time_ = GetScanTime(lastScanPrefix); currentScan_.lastModified_ = {}; currentScan_.secondLastModified_ = {}; @@ -502,7 +506,7 @@ bool AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) return false; } - bool hasNew = false; + bool hasNew = false; auto& chunks = listOutcome.GetResult().GetContents(); logger_->debug("Found {} new chunks.", chunks.size()); for (const auto& chunk : chunks) @@ -511,27 +515,26 @@ bool AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) // KIND/585/20250324-134727-001-S // KIND/5/20250324-134727-001-S - static const size_t firstSlash = std::string("KIND/").size(); - const size_t secondSlash = key.find('/', firstSlash); + static const size_t firstSlash = std::string("KIND/").size(); + const size_t secondSlash = key.find('/', firstSlash); static const size_t startNumberPosOffset = std::string("/20250324-134727-").size(); - const size_t startNumberPos = - secondSlash + startNumberPosOffset; - const std::string& keyNumberStr = key.substr(startNumberPos, 3); - const int keyNumber = std::stoi(keyNumberStr); - if (keyNumber != scanRecord.nextFile_) + const size_t startNumberPos = secondSlash + startNumberPosOffset; + const std::string& keyNumberStr = key.substr(startNumberPos, 3); + const int keyNumber = std::stoi(keyNumberStr); + // As far as order goes, only the first one matters. This may cause some + // issues if keys come in out of order, but usually they just skip chunks + if (scanRecord.nextFile_ == 1 && keyNumber != scanRecord.nextFile_) { - logger_->warn("Chunk found that was not in order {} {} {}", - key, - scanRecord.nextFile_, - keyNumber); + logger_->warn("Chunk found that was not in order {} {}", + scanRecord.lastKey_, + key); continue; } // Now we want the ending char // KIND/585/20250324-134727-001-S - static const size_t charPos = - std::string("/20250324-134727-001-").size(); + static const size_t charPos = std::string("/20250324-134727-001-").size(); if (secondSlash + charPos >= key.size()) { logger_->warn("Chunk key was not long enough"); @@ -591,15 +594,16 @@ bool AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) } hasNew = true; - std::chrono::seconds lastModifiedSeconds { + const std::chrono::seconds lastModifiedSeconds { outcome.GetResult().GetLastModified().Seconds()}; - std::chrono::system_clock::time_point lastModified {lastModifiedSeconds}; + const std::chrono::system_clock::time_point lastModified { + lastModifiedSeconds}; scanRecord.secondLastModified_ = scanRecord.lastModified_; - scanRecord.lastModified_ = lastModified; + scanRecord.lastModified_ = lastModified; - scanRecord.nextFile_ += 1; - scanRecord.lastKey_ = key; + scanRecord.nextFile_ = keyNumber + 1; + scanRecord.lastKey_ = key; } if (scanRecord.nexradFile_ == nullptr) @@ -618,7 +622,7 @@ std::shared_ptr AwsLevel2ChunksDataProvider::LoadObjectByTime( std::chrono::system_clock::time_point time) { - std::unique_lock lock(p->scansMutex_); + const std::unique_lock lock(p->scansMutex_); static const std::chrono::system_clock::time_point epoch {}; if (p->currentScan_.valid_ && @@ -640,10 +644,10 @@ AwsLevel2ChunksDataProvider::LoadObjectByTime( std::shared_ptr AwsLevel2ChunksDataProvider::LoadLatestObject() { - std::unique_lock lock(p->scansMutex_); + const std::unique_lock lock(p->scansMutex_); return std::make_shared(p->currentScan_.nexradFile_, p->lastScan_.nexradFile_); - //return p->currentScan_.nexradFile_; + // return p->currentScan_.nexradFile_; } std::shared_ptr @@ -667,8 +671,8 @@ std::pair AwsLevel2ChunksDataProvider::Refresh() boost::timer::cpu_timer timer {}; timer.start(); - std::unique_lock lock(p->refreshMutex_); - std::unique_lock scanLock(p->scansMutex_); + const std::unique_lock lock(p->refreshMutex_); + const std::unique_lock scanLock(p->scansMutex_); auto [success, newObjects, totalObjects] = p->ListObjects(); @@ -682,6 +686,8 @@ std::pair AwsLevel2ChunksDataProvider::Refresh() } if (p->lastScan_.valid_) { + // TODO this is slow when initially loading data. If possible, loading + // this from the archive may speed it up a lot. if (p->LoadScan(p->lastScan_)) { newObjects += 1; @@ -690,6 +696,7 @@ std::pair AwsLevel2ChunksDataProvider::Refresh() } timer.stop(); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) format to 6 digits logger_->debug("Refresh() in {}", timer.format(6, "%ws")); return std::make_pair(newObjects, totalObjects); } @@ -710,8 +717,7 @@ float AwsLevel2ChunksDataProvider::GetCurrentElevation() if (!p->currentScan_.valid_ || p->currentScan_.nexradFile_ == nullptr) { // Does not have any scan elevation. -90 is beyond what is possible - // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) - return -90; + return INVALID_ELEVATION; } auto vcpData = p->currentScan_.nexradFile_->vcp_data(); @@ -719,18 +725,17 @@ float AwsLevel2ChunksDataProvider::GetCurrentElevation() if (radarData.size() == 0) { // Does not have any scan elevation. -90 is beyond what is possible - // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) - return -90; + return INVALID_ELEVATION; } const auto& lastElevation = radarData.crbegin(); - std::shared_ptr digitalRadarData0 = + const std::shared_ptr digitalRadarData0 = std::dynamic_pointer_cast( lastElevation->second->cbegin()->second); if (vcpData != nullptr) { - // NOLINTNEXTLINE(cppcoreguidelines-narrowing-conversions) Float is plenty + // NOLINTNEXTLINE(*-narrowing-conversions) Float is plenty return vcpData->elevation_angle(lastElevation->first); } else if (digitalRadarData0 != nullptr) @@ -738,9 +743,7 @@ float AwsLevel2ChunksDataProvider::GetCurrentElevation() return digitalRadarData0->elevation_angle().value(); } - // Does not have any scan elevation. -90 is beyond what is possible - // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) - return -90; + return INVALID_ELEVATION; } } // namespace scwx::provider diff --git a/wxdata/source/scwx/wsr88d/ar2v_file.cpp b/wxdata/source/scwx/wsr88d/ar2v_file.cpp index f804685b..d9335b50 100644 --- a/wxdata/source/scwx/wsr88d/ar2v_file.cpp +++ b/wxdata/source/scwx/wsr88d/ar2v_file.cpp @@ -138,7 +138,7 @@ Ar2vFile::GetElevationScan(rda::DataBlockType dataBlockType, float elevation, std::chrono::system_clock::time_point time) const { - //logger_->debug("GetElevationScan: {} degrees", elevation); + // logger_->debug("GetElevationScan: {} degrees", elevation); std::shared_ptr elevationScan = nullptr; float elevationCut = 0.0f; @@ -273,7 +273,7 @@ bool Ar2vFile::LoadData(std::istream& is) std::size_t Ar2vFileImpl::DecompressLDMRecords(std::istream& is) { - //logger_->debug("Decompressing LDM Records"); + // logger_->debug("Decompressing LDM Records"); std::size_t numRecords = 0; @@ -321,22 +321,22 @@ std::size_t Ar2vFileImpl::DecompressLDMRecords(std::istream& is) ++numRecords; } - //logger_->debug("Decompressed {} LDM Records", numRecords); + // logger_->debug("Decompressed {} LDM Records", numRecords); return numRecords; } void Ar2vFileImpl::ParseLDMRecords() { - //logger_->debug("Parsing LDM Records"); + // logger_->debug("Parsing LDM Records"); - //std::size_t count = 0; + // std::size_t count = 0; for (auto it = rawRecords_.begin(); it != rawRecords_.end(); it++) { std::stringstream& ss = *it; - //logger_->trace("Record {}", count++); + // logger_->trace("Record {}", count++); ParseLDMRecord(ss); } @@ -445,7 +445,7 @@ void Ar2vFileImpl::ProcessRadarData( void Ar2vFileImpl::IndexFile() { - //logger_->debug("Indexing file"); + // logger_->debug("Indexing file"); for (auto& elevationCut : radarData_) { @@ -465,6 +465,7 @@ void Ar2vFileImpl::IndexFile() if (vcpData_ != nullptr) { + // NOLINTNEXTLINE(*-narrowing-conversions) Float is plenty elevationAngle = vcpData_->elevation_angle(elevationCut.first); waveformType = vcpData_->waveform_type(elevationCut.first); } @@ -500,15 +501,15 @@ void Ar2vFileImpl::IndexFile() auto time = util::TimePoint(radial0->modified_julian_date(), radial0->collection_time()); - index_[dataBlockType][elevationAngle][time] = - elevationCut.second; + index_[dataBlockType][elevationAngle][time] = elevationCut.second; } } } } -bool Ar2vFile::LoadLDMRecords(std::istream& is) { - size_t decompressedRecords = p->DecompressLDMRecords(is); +bool Ar2vFile::LoadLDMRecords(std::istream& is) +{ + const size_t decompressedRecords = p->DecompressLDMRecords(is); if (decompressedRecords == 0) { p->ParseLDMRecord(is); @@ -528,6 +529,7 @@ bool Ar2vFile::IndexFile() } // TODO not good +// NOLINTNEXTLINE bool IsRadarDataIncomplete( const std::shared_ptr& radarData) { @@ -695,12 +697,13 @@ Ar2vFile::Ar2vFile(const std::shared_ptr& current, // Find the highest elevation this type has for the current scan // Start below any reasonable elevation // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) - float highestCurrentElevation = -90; - const auto& maybe1 = p->index_.find(type.first); + float highestCurrentElevation = -90; + const auto& maybe1 = p->index_.find(type.first); if (maybe1 != p->index_.cend()) { const auto& maybe2 = maybe1->second.crbegin(); - if (maybe2 != maybe1->second.crend()) { + if (maybe2 != maybe1->second.crend()) + { // Add a slight offset to ensure good floating point compare. // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) highestCurrentElevation = maybe2->first + 0.01f; diff --git a/wxdata/source/scwx/wsr88d/rda/volume_coverage_pattern_data.cpp b/wxdata/source/scwx/wsr88d/rda/volume_coverage_pattern_data.cpp index 42ed3685..a30a2df1 100644 --- a/wxdata/source/scwx/wsr88d/rda/volume_coverage_pattern_data.cpp +++ b/wxdata/source/scwx/wsr88d/rda/volume_coverage_pattern_data.cpp @@ -221,8 +221,8 @@ uint16_t VolumeCoveragePatternData::number_of_base_tilts() const double VolumeCoveragePatternData::elevation_angle(uint16_t e) const { - // NOLINTNEXTLINE This conversion is accurate float elevationAngleConverted = + // NOLINTNEXTLINE This conversion is accurate p->elevationCuts_[e].elevationAngle_ * ANGLE_DATA_SCALE; // Any elevation above 90 degrees should be interpreted as a // negative angle From 0ac0e03ff88458b78b51b79964437e6e1600ef86 Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Tue, 8 Apr 2025 13:55:23 -0400 Subject: [PATCH 17/34] Relaod all the settings, just to make sure everything is updated --- scwx-qt/source/scwx/qt/main/main_window.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scwx-qt/source/scwx/qt/main/main_window.cpp b/scwx-qt/source/scwx/qt/main/main_window.cpp index 089d6749..8308be9b 100644 --- a/scwx-qt/source/scwx/qt/main/main_window.cpp +++ b/scwx-qt/source/scwx/qt/main/main_window.cpp @@ -967,8 +967,7 @@ void MainWindowImpl::ConnectMapSignals() mapWidget, &map::MapWidget::IncomingLevel2ElevationChanged, this, - [this](float incomingElevation) - { level2SettingsWidget_->UpdateIncomingElevation(incomingElevation); }, + [this](float) { level2SettingsWidget_->UpdateSettings(activeMap_); }, Qt::QueuedConnection); } } From 34eb3af69806dd6e8e91e0baa67b077e67068128 Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Tue, 8 Apr 2025 14:32:09 -0400 Subject: [PATCH 18/34] Use static cast when getting elevations to convert from double to float --- .../source/scwx/provider/aws_level2_chunks_data_provider.cpp | 3 +-- wxdata/source/scwx/wsr88d/rda/volume_coverage_pattern_data.cpp | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp index 16fb0e44..77de4593 100644 --- a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp @@ -735,8 +735,7 @@ float AwsLevel2ChunksDataProvider::GetCurrentElevation() if (vcpData != nullptr) { - // NOLINTNEXTLINE(*-narrowing-conversions) Float is plenty - return vcpData->elevation_angle(lastElevation->first); + return static_cast(vcpData->elevation_angle(lastElevation->first)); } else if (digitalRadarData0 != nullptr) { diff --git a/wxdata/source/scwx/wsr88d/rda/volume_coverage_pattern_data.cpp b/wxdata/source/scwx/wsr88d/rda/volume_coverage_pattern_data.cpp index a30a2df1..b2159648 100644 --- a/wxdata/source/scwx/wsr88d/rda/volume_coverage_pattern_data.cpp +++ b/wxdata/source/scwx/wsr88d/rda/volume_coverage_pattern_data.cpp @@ -221,8 +221,7 @@ uint16_t VolumeCoveragePatternData::number_of_base_tilts() const double VolumeCoveragePatternData::elevation_angle(uint16_t e) const { - float elevationAngleConverted = - // NOLINTNEXTLINE This conversion is accurate + double elevationAngleConverted = p->elevationCuts_[e].elevationAngle_ * ANGLE_DATA_SCALE; // Any elevation above 90 degrees should be interpreted as a // negative angle From 314d3f5b9b9fb6f8c3812369f112bb54cee150ef Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Tue, 8 Apr 2025 15:17:36 -0400 Subject: [PATCH 19/34] Use static cast when getting elevations to convert from double to float take 2 --- wxdata/source/scwx/wsr88d/ar2v_file.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wxdata/source/scwx/wsr88d/ar2v_file.cpp b/wxdata/source/scwx/wsr88d/ar2v_file.cpp index d9335b50..6d0e5c02 100644 --- a/wxdata/source/scwx/wsr88d/ar2v_file.cpp +++ b/wxdata/source/scwx/wsr88d/ar2v_file.cpp @@ -465,8 +465,8 @@ void Ar2vFileImpl::IndexFile() if (vcpData_ != nullptr) { - // NOLINTNEXTLINE(*-narrowing-conversions) Float is plenty - elevationAngle = vcpData_->elevation_angle(elevationCut.first); + elevationAngle = + static_cast(vcpData_->elevation_angle(elevationCut.first)); waveformType = vcpData_->waveform_type(elevationCut.first); } else if ((digitalRadarData0 = From 1f0d2a7a66a80c3dc2df4667d7d9f19bf0c673f2 Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Tue, 8 Apr 2025 16:18:45 -0400 Subject: [PATCH 20/34] Change the selection of the most recent level 2 scan to avoid certain improper removal from causing issues. --- .../aws_level2_chunks_data_provider.cpp | 77 ++++++++++--------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp index 77de4593..fc6c022a 100644 --- a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp @@ -367,7 +367,6 @@ AwsLevel2ChunksDataProvider::Impl::ListObjects() scanPrefix); } - int lastScanNumber = -1; // Start with last scan int previousScanNumber = scanNumberMap.crbegin()->first; const int firstScanNumber = scanNumberMap.cbegin()->first; @@ -375,50 +374,54 @@ AwsLevel2ChunksDataProvider::Impl::ListObjects() // Look for a gap in scan numbers. This indicates that is the latest // scan. - // This indicates that highest number scan is the last scan + auto possibleLastNumbers = std::unordered_set(); + // This indicates that highest number scan may be the last scan // (including if there is only 1 scan) // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) - if (previousScanNumber != 999 || scans.size() == 1) + if (previousScanNumber != 999 || firstScanNumber != 1) { - lastScanNumber = previousScanNumber; + possibleLastNumbers.emplace(previousScanNumber); } - else + // Have already checked scan with highest number, so skip first + previousScanNumber = firstScanNumber; + bool first = true; + for (const auto& scan : scanNumberMap) { - // Have already checked scan with highest number, so skip first - previousScanNumber = firstScanNumber; - bool first = true; - for (const auto& scan : scanNumberMap) + if (first) { - if (first) - { - first = false; - continue; - } - if (scan.first != previousScanNumber + 1) - { - lastScanNumber = previousScanNumber; - break; - } - previousScanNumber = scan.first; + first = false; + continue; + } + if (scan.first != previousScanNumber + 1) + { + possibleLastNumbers.emplace(previousScanNumber); + } + previousScanNumber = scan.first; + } + + if (possibleLastNumbers.empty()) + { + logger_->warn("Could not find last scan"); + // TODO make sure this makes sence + return {false, 0, 0}; + } + + int lastScanNumber = -1; + std::chrono::system_clock::time_point lastScanTime = {}; + std::string lastScanPrefix; + + for (const int scanNumber : possibleLastNumbers) + { + const std::string& scanPrefix = scanNumberMap.at(scanNumber); + auto scanTime = GetScanTime(scanPrefix); + if (scanTime > lastScanTime) + { + lastScanTime = scanTime; + lastScanPrefix = scanPrefix; + lastScanNumber = scanNumber; } } - if (lastScanNumber == -1) - { - // 999 is the last scan - if (firstScanNumber != 1) - { - lastScanNumber = previousScanNumber; - } - else - { - logger_->warn("Could not find last scan"); - // TODO make sure this makes sence - return {false, 0, 0}; - } - } - - const std::string& lastScanPrefix = scanNumberMap.at(lastScanNumber); const int secondLastScanNumber = // 999 is the last file possible // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) @@ -452,7 +455,7 @@ AwsLevel2ChunksDataProvider::Impl::ListObjects() currentScan_.valid_ = true; currentScan_.prefix_ = lastScanPrefix; currentScan_.nexradFile_ = nullptr; - currentScan_.time_ = GetScanTime(lastScanPrefix); + currentScan_.time_ = lastScanTime; currentScan_.lastModified_ = {}; currentScan_.secondLastModified_ = {}; currentScan_.lastKey_ = ""; From 309a5ed25e796bc08dd0f9e06ed3404dccb524d7 Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Thu, 10 Apr 2025 13:28:06 -0400 Subject: [PATCH 21/34] Clean up some functions in chunks data provider --- .../aws_level2_chunks_data_provider.cpp | 30 +++++-------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp index fc6c022a..228426c2 100644 --- a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp @@ -101,10 +101,7 @@ public: Impl& operator=(Impl&&) = delete; std::chrono::system_clock::time_point GetScanTime(const std::string& prefix); - int GetScanNumber(const std::string& prefix); - std::string GetScanKey(const std::string& prefix, - const std::chrono::system_clock::time_point& time, - int last); + int GetScanNumber(const std::string& prefix); bool LoadScan(Impl::ScanRecord& scanRecord); std::tuple ListObjects(); @@ -311,19 +308,6 @@ AwsLevel2ChunksDataProvider::Impl::GetScanTime(const std::string& prefix) return {}; } -std::string AwsLevel2ChunksDataProvider::Impl::GetScanKey( - const std::string& prefix, - const std::chrono::system_clock::time_point& time, - int last) -{ - - static const std::string timeFormat {"%Y%m%d-%H%M%S"}; - - // TODO - return fmt::format( - "{0}/{1:%Y%m%d-%H%M%S}-{2}", prefix, fmt::gmtime(time), last - 1); -} - std::tuple AwsLevel2ChunksDataProvider::Impl::ListObjects() { @@ -406,23 +390,23 @@ AwsLevel2ChunksDataProvider::Impl::ListObjects() return {false, 0, 0}; } - int lastScanNumber = -1; - std::chrono::system_clock::time_point lastScanTime = {}; - std::string lastScanPrefix; + int lastScanNumber = -1; + std::chrono::system_clock::time_point lastScanTime = {}; + std::string lastScanPrefix; for (const int scanNumber : possibleLastNumbers) { const std::string& scanPrefix = scanNumberMap.at(scanNumber); - auto scanTime = GetScanTime(scanPrefix); + auto scanTime = GetScanTime(scanPrefix); if (scanTime > lastScanTime) { - lastScanTime = scanTime; + lastScanTime = scanTime; lastScanPrefix = scanPrefix; lastScanNumber = scanNumber; } } - const int secondLastScanNumber = + const int secondLastScanNumber = // 999 is the last file possible // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) lastScanNumber == 1 ? 999 : lastScanNumber - 1; From 16a73ed872a9a4fd9c19077ce37463298d6d6f36 Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Fri, 18 Apr 2025 10:59:19 -0400 Subject: [PATCH 22/34] Add previous scans for stepping back in time when merging level2 files --- wxdata/source/scwx/wsr88d/ar2v_file.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/wxdata/source/scwx/wsr88d/ar2v_file.cpp b/wxdata/source/scwx/wsr88d/ar2v_file.cpp index 6d0e5c02..b4dc6248 100644 --- a/wxdata/source/scwx/wsr88d/ar2v_file.cpp +++ b/wxdata/source/scwx/wsr88d/ar2v_file.cpp @@ -568,6 +568,15 @@ Ar2vFile::Ar2vFile(const std::shared_ptr& current, continue; } + // Add previous scans for stepping back in time + for (auto scan = ++(elevation.second.rbegin()); + scan != elevation.second.rend(); + ++scan) + { + p->index_[type.first][elevation.first][scan->first] = + scan->second; + } + // Merge this scan with the last one if it is incomplete if (IsRadarDataIncomplete(mostRecent->second)) { From f481d57ed1a53dcd4e7905fab5ca3c7fad97b4bf Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Fri, 18 Apr 2025 11:12:39 -0400 Subject: [PATCH 23/34] clang format/tidy fixes for level2_chunks --- scwx-qt/source/scwx/qt/ui/level2_settings_widget.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scwx-qt/source/scwx/qt/ui/level2_settings_widget.cpp b/scwx-qt/source/scwx/qt/ui/level2_settings_widget.cpp index 78e4de37..582cf64b 100644 --- a/scwx-qt/source/scwx/qt/ui/level2_settings_widget.cpp +++ b/scwx-qt/source/scwx/qt/ui/level2_settings_widget.cpp @@ -254,8 +254,8 @@ void Level2SettingsWidget::UpdateSettings(map::MapWidget* activeMap) std::optional currentElevationOption = activeMap->GetElevation(); const float currentElevation = currentElevationOption.has_value() ? *currentElevationOption : 0.0f; - std::vector elevationCuts = activeMap->GetElevationCuts(); - const float incomingElevation = activeMap->GetIncomingLevel2Elevation(); + const std::vector elevationCuts = activeMap->GetElevationCuts(); + const float incomingElevation = activeMap->GetIncomingLevel2Elevation(); if (p->elevationCuts_ != elevationCuts) { From e10ebdeb5e5424270be3347fbd3e4305cf80798c Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Sun, 20 Apr 2025 10:56:50 -0400 Subject: [PATCH 24/34] switch level2 incoming elevation to optional --- scwx-qt/source/scwx/qt/main/main_window.cpp | 3 ++- .../scwx/qt/manager/radar_product_manager.cpp | 16 ++++++-------- .../scwx/qt/manager/radar_product_manager.hpp | 8 +++---- scwx-qt/source/scwx/qt/map/map_widget.cpp | 4 ++-- scwx-qt/source/scwx/qt/map/map_widget.hpp | 4 ++-- .../scwx/qt/ui/level2_settings_widget.cpp | 21 +++++++++++++------ .../scwx/qt/ui/level2_settings_widget.hpp | 4 +++- .../aws_level2_chunks_data_provider.hpp | 5 +++-- .../aws_level2_chunks_data_provider.cpp | 8 +++---- 9 files changed, 41 insertions(+), 32 deletions(-) diff --git a/scwx-qt/source/scwx/qt/main/main_window.cpp b/scwx-qt/source/scwx/qt/main/main_window.cpp index 8308be9b..30e3b699 100644 --- a/scwx-qt/source/scwx/qt/main/main_window.cpp +++ b/scwx-qt/source/scwx/qt/main/main_window.cpp @@ -967,7 +967,8 @@ void MainWindowImpl::ConnectMapSignals() mapWidget, &map::MapWidget::IncomingLevel2ElevationChanged, this, - [this](float) { level2SettingsWidget_->UpdateSettings(activeMap_); }, + [this](std::optional) + { level2SettingsWidget_->UpdateSettings(activeMap_); }, Qt::QueuedConnection); } } diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp index 39501b29..1814905d 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp @@ -272,8 +272,7 @@ public: common::Level3ProductCategoryMap availableCategoryMap_ {}; std::shared_mutex availableCategoryMutex_ {}; - float incomingLevel2Elevation_ { - provider::AwsLevel2ChunksDataProvider::INVALID_ELEVATION}; + std::optional incomingLevel2Elevation_ {}; std::unordered_map, @@ -454,7 +453,7 @@ float RadarProductManager::gate_size() const return (is_tdwr()) ? 150.0f : 250.0f; } -float RadarProductManager::incoming_level_2_elevation() const +std::optional RadarProductManager::incoming_level_2_elevation() const { return p->incomingLevel2Elevation_; } @@ -1552,7 +1551,7 @@ RadarProductManager::GetLevel2Data(wsr88d::rda::DataBlockType dataBlockType, scwx::util::TimePoint(radarData0->modified_julian_date(), radarData0->collection_time())); - const float incomingElevation = + const std::optional incomingElevation = std::dynamic_pointer_cast( p->level2ChunksProviderManager_->provider_) ->GetCurrentElevation(); @@ -1597,14 +1596,11 @@ RadarProductManager::GetLevel2Data(wsr88d::rda::DataBlockType dataBlockType, elevationCuts = std::move(recordElevationCuts); foundTime = collectionTime; - if (p->incomingLevel2Elevation_ != - provider::AwsLevel2ChunksDataProvider::INVALID_ELEVATION) + if (!p->incomingLevel2Elevation_.has_value()) { - p->incomingLevel2Elevation_ = provider:: - AwsLevel2ChunksDataProvider::INVALID_ELEVATION; + p->incomingLevel2Elevation_ = {}; Q_EMIT IncomingLevel2ElevationChanged( - provider::AwsLevel2ChunksDataProvider:: - INVALID_ELEVATION); + p->incomingLevel2Elevation_); } } } diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.hpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.hpp index 6efd125d..c0a49dff 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.hpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.hpp @@ -45,9 +45,9 @@ public: coordinates(common::RadialSize radialSize, bool smoothingEnabled) const; [[nodiscard]] const scwx::util::time_zone* default_time_zone() const; [[nodiscard]] float gate_size() const; - [[nodiscard]] float incoming_level_2_elevation() const; - [[nodiscard]] bool is_tdwr() const; - [[nodiscard]] std::string radar_id() const; + [[nodiscard]] std::optional incoming_level_2_elevation() const; + [[nodiscard]] bool is_tdwr() const; + [[nodiscard]] std::string radar_id() const; [[nodiscard]] std::shared_ptr radar_site() const; void Initialize(); @@ -149,7 +149,7 @@ signals: void NewDataAvailable(common::RadarProductGroup group, const std::string& product, std::chrono::system_clock::time_point latestTime); - void IncomingLevel2ElevationChanged(float incomingElevation); + void IncomingLevel2ElevationChanged(std::optional incomingElevation); private: std::unique_ptr p; diff --git a/scwx-qt/source/scwx/qt/map/map_widget.cpp b/scwx-qt/source/scwx/qt/map/map_widget.cpp index 845e0a51..ad808626 100644 --- a/scwx-qt/source/scwx/qt/map/map_widget.cpp +++ b/scwx-qt/source/scwx/qt/map/map_widget.cpp @@ -656,7 +656,7 @@ std::vector MapWidget::GetElevationCuts() const } } -float MapWidget::GetIncomingLevel2Elevation() const +std::optional MapWidget::GetIncomingLevel2Elevation() const { return p->radarProductManager_->incoming_level_2_elevation(); } @@ -1804,7 +1804,7 @@ void MapWidgetImpl::RadarProductManagerConnect() connect(radarProductManager_.get(), &manager::RadarProductManager::IncomingLevel2ElevationChanged, this, - [this](float incomingElevation) + [this](std::optional incomingElevation) { Q_EMIT widget_->IncomingLevel2ElevationChanged( incomingElevation); diff --git a/scwx-qt/source/scwx/qt/map/map_widget.hpp b/scwx-qt/source/scwx/qt/map/map_widget.hpp index 5cc2e0a1..d474cd2e 100644 --- a/scwx-qt/source/scwx/qt/map/map_widget.hpp +++ b/scwx-qt/source/scwx/qt/map/map_widget.hpp @@ -45,7 +45,7 @@ public: GetAvailableLevel3Categories(); [[nodiscard]] std::optional GetElevation() const; [[nodiscard]] std::vector GetElevationCuts() const; - [[nodiscard]] float GetIncomingLevel2Elevation() const; + [[nodiscard]] std::optional GetIncomingLevel2Elevation() const; [[nodiscard]] std::vector GetLevel3Products(); [[nodiscard]] std::string GetMapStyle() const; [[nodiscard]] common::RadarProductGroup GetRadarProductGroup() const; @@ -185,7 +185,7 @@ signals: void RadarSweepUpdated(); void RadarSweepNotUpdated(types::NoUpdateReason reason); void WidgetPainted(); - void IncomingLevel2ElevationChanged(float incomingElevation); + void IncomingLevel2ElevationChanged(std::optional incomingElevation); }; } // namespace map diff --git a/scwx-qt/source/scwx/qt/ui/level2_settings_widget.cpp b/scwx-qt/source/scwx/qt/ui/level2_settings_widget.cpp index 582cf64b..3d530733 100644 --- a/scwx-qt/source/scwx/qt/ui/level2_settings_widget.cpp +++ b/scwx-qt/source/scwx/qt/ui/level2_settings_widget.cpp @@ -242,11 +242,19 @@ void Level2SettingsWidget::UpdateElevationSelection(float elevation) p->currentElevationButton_ = newElevationButton; } -void Level2SettingsWidget::UpdateIncomingElevation(float incomingElevation) +void Level2SettingsWidget::UpdateIncomingElevation( + std::optional incomingElevation) { - p->incomingElevationLabel_->setText( - "Incoming Elevation: " + QString::number(incomingElevation, 'f', 1) + - common::Characters::DEGREE); + if (incomingElevation.has_value()) + { + p->incomingElevationLabel_->setText( + "Incoming Elevation: " + QString::number(*incomingElevation, 'f', 1) + + common::Characters::DEGREE); + } + else + { + p->incomingElevationLabel_->setText("Incoming Elevation: None"); + } } void Level2SettingsWidget::UpdateSettings(map::MapWidget* activeMap) @@ -254,8 +262,9 @@ void Level2SettingsWidget::UpdateSettings(map::MapWidget* activeMap) std::optional currentElevationOption = activeMap->GetElevation(); const float currentElevation = currentElevationOption.has_value() ? *currentElevationOption : 0.0f; - const std::vector elevationCuts = activeMap->GetElevationCuts(); - const float incomingElevation = activeMap->GetIncomingLevel2Elevation(); + const std::vector elevationCuts = activeMap->GetElevationCuts(); + const std::optional incomingElevation = + activeMap->GetIncomingLevel2Elevation(); if (p->elevationCuts_ != elevationCuts) { diff --git a/scwx-qt/source/scwx/qt/ui/level2_settings_widget.hpp b/scwx-qt/source/scwx/qt/ui/level2_settings_widget.hpp index 796b1ade..32f788bb 100644 --- a/scwx-qt/source/scwx/qt/ui/level2_settings_widget.hpp +++ b/scwx-qt/source/scwx/qt/ui/level2_settings_widget.hpp @@ -2,6 +2,8 @@ #include +#include + namespace scwx { namespace qt @@ -23,7 +25,7 @@ public: void showEvent(QShowEvent* event) override; void UpdateElevationSelection(float elevation); - void UpdateIncomingElevation(float incomingElevation); + void UpdateIncomingElevation(std::optional incomingElevation); void UpdateSettings(map::MapWidget* activeMap); signals: diff --git a/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp b/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp index 052f639b..976f0663 100644 --- a/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp +++ b/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp @@ -2,6 +2,8 @@ #include +#include + namespace Aws::S3 { class S3Client; @@ -16,7 +18,6 @@ namespace scwx::provider class AwsLevel2ChunksDataProvider : public NexradDataProvider { public: - constexpr static const float INVALID_ELEVATION = -90.0; explicit AwsLevel2ChunksDataProvider(const std::string& radarSite); explicit AwsLevel2ChunksDataProvider(const std::string& radarSite, const std::string& bucketName, @@ -58,7 +59,7 @@ public: void RequestAvailableProducts() override; std::vector GetAvailableProducts() override; - float GetCurrentElevation(); + std::optional GetCurrentElevation(); private: class Impl; diff --git a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp index 228426c2..ddac82e5 100644 --- a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp @@ -699,12 +699,12 @@ AwsLevel2ChunksDataProvider::AwsLevel2ChunksDataProvider( AwsLevel2ChunksDataProvider& AwsLevel2ChunksDataProvider::operator=( AwsLevel2ChunksDataProvider&&) noexcept = default; -float AwsLevel2ChunksDataProvider::GetCurrentElevation() +std::optional AwsLevel2ChunksDataProvider::GetCurrentElevation() { if (!p->currentScan_.valid_ || p->currentScan_.nexradFile_ == nullptr) { // Does not have any scan elevation. -90 is beyond what is possible - return INVALID_ELEVATION; + return {}; } auto vcpData = p->currentScan_.nexradFile_->vcp_data(); @@ -712,7 +712,7 @@ float AwsLevel2ChunksDataProvider::GetCurrentElevation() if (radarData.size() == 0) { // Does not have any scan elevation. -90 is beyond what is possible - return INVALID_ELEVATION; + return {}; } const auto& lastElevation = radarData.crbegin(); @@ -729,7 +729,7 @@ float AwsLevel2ChunksDataProvider::GetCurrentElevation() return digitalRadarData0->elevation_angle().value(); } - return INVALID_ELEVATION; + return {}; } } // namespace scwx::provider From 759a9e43798b98d7b0ef8c514e58f1e3254a36a3 Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Sun, 20 Apr 2025 13:02:02 -0400 Subject: [PATCH 25/34] Parallelize the chunks loading and load from archive when possible --- .../scwx/qt/manager/radar_product_manager.cpp | 10 +++ .../aws_level2_chunks_data_provider.hpp | 4 + .../aws_level2_chunks_data_provider.cpp | 86 ++++++++++++++++--- 3 files changed, 88 insertions(+), 12 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp index 1814905d..e56be177 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp @@ -151,6 +151,16 @@ public: level2ChunksProviderManager_->provider_ = provider::NexradDataProviderFactory::CreateLevel2ChunksDataProvider( radarId); + + auto level2ChunksProvider = + std::dynamic_pointer_cast( + level2ChunksProviderManager_->provider_); + if (level2ChunksProvider != nullptr) + { + level2ChunksProvider->SetLevel2DataProvider( + std::dynamic_pointer_cast( + level2ProviderManager_->provider_)); + } } ~RadarProductManagerImpl() { diff --git a/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp b/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp index 976f0663..476ff111 100644 --- a/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp +++ b/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include @@ -61,6 +62,9 @@ public: std::optional GetCurrentElevation(); + void SetLevel2DataProvider( + const std::shared_ptr& provider); + private: class Impl; std::unique_ptr p; diff --git a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp index ddac82e5..bb1b31c1 100644 --- a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp @@ -13,10 +13,24 @@ #include #include #include -#include #include #include +// Avoid circular refrence errors in boost +// NOLINTBEGIN(misc-header-include-cycle) +#if defined(_MSC_VER) +# pragma warning(push, 0) +#endif + +#include +#include +#include + +#if defined(_MSC_VER) +# pragma warning(pop) +#endif +// NOLINTEND(misc-header-include-cycle) + #if (__cpp_lib_chrono < 201907L) # include #endif @@ -75,6 +89,7 @@ public: lastTimeListed_ {}, // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) about average updatePeriod_ {7}, + level2DataProvider_ {}, self_ {self} { // Disable HTTP request for region @@ -122,6 +137,8 @@ public: std::chrono::seconds updatePeriod_; + std::weak_ptr level2DataProvider_; + AwsLevel2ChunksDataProvider* self_; }; @@ -634,7 +651,6 @@ AwsLevel2ChunksDataProvider::LoadLatestObject() const std::unique_lock lock(p->scansMutex_); return std::make_shared(p->currentScan_.nexradFile_, p->lastScan_.nexradFile_); - // return p->currentScan_.nexradFile_; } std::shared_ptr @@ -663,23 +679,63 @@ std::pair AwsLevel2ChunksDataProvider::Refresh() auto [success, newObjects, totalObjects] = p->ListObjects(); + auto threadPool = boost::asio::thread_pool(3); + bool newCurrent = false; + bool newLast = false; if (p->currentScan_.valid_) { - if (p->LoadScan(p->currentScan_)) - { - newObjects += 1; - } + boost::asio::post(threadPool, + [this, &newCurrent]() + { newCurrent = p->LoadScan(p->currentScan_); }); totalObjects += 1; } + if (p->lastScan_.valid_) { - // TODO this is slow when initially loading data. If possible, loading - // this from the archive may speed it up a lot. - if (p->LoadScan(p->lastScan_)) - { - newObjects += 1; - } totalObjects += 1; + boost::asio::post( + threadPool, + [this, &newLast]() + { + if (!p->lastScan_.hasAllFiles_) + { + // If we have chunks, use chunks + if (p->lastScan_.nextFile_ != 1) + { + newLast = p->LoadScan(p->lastScan_); + } + else + { + auto level2DataProvider = p->level2DataProvider_.lock(); + if (level2DataProvider != nullptr) + { + level2DataProvider->ListObjects(p->lastScan_.time_); + p->lastScan_.nexradFile_ = + std::dynamic_pointer_cast( + level2DataProvider->LoadObjectByTime( + p->lastScan_.time_)); + if (p->lastScan_.nexradFile_ != nullptr) + { + p->lastScan_.hasAllFiles_ = true; + // TODO maybe set lastModified for timing + } + } + // Fall back to chunks if files did not load + newLast = p->lastScan_.nexradFile_ != nullptr || + p->LoadScan(p->lastScan_); + } + } + }); + } + + threadPool.wait(); + if (newCurrent) + { + newObjects += 1; + } + if (newLast) + { + newObjects += 1; } timer.stop(); @@ -732,4 +788,10 @@ std::optional AwsLevel2ChunksDataProvider::GetCurrentElevation() return {}; } +void AwsLevel2ChunksDataProvider::SetLevel2DataProvider( + const std::shared_ptr& provider) +{ + p->level2DataProvider_ = provider; +} + } // namespace scwx::provider From 2daf4d8ba4b6ef59cad88eb01fefdb681e747c7b Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Mon, 21 Apr 2025 19:44:31 -0400 Subject: [PATCH 26/34] Remove some unneded methods added in level_2_chunks --- .../aws_level2_chunks_data_provider.hpp | 4 +-- .../provider/aws_nexrad_data_provider.hpp | 2 -- .../scwx/provider/nexrad_data_provider.hpp | 14 -------- .../aws_level2_chunks_data_provider.cpp | 34 ++++++------------- .../provider/aws_nexrad_data_provider.cpp | 11 ------ 5 files changed, 12 insertions(+), 53 deletions(-) diff --git a/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp b/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp index 476ff111..abd70787 100644 --- a/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp +++ b/wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp @@ -53,9 +53,7 @@ public: LoadObjectByKey(const std::string& key) override; std::shared_ptr LoadObjectByTime(std::chrono::system_clock::time_point time) override; - std::shared_ptr LoadLatestObject() override; - std::shared_ptr LoadSecondLatestObject() override; - std::pair Refresh() override; + std::pair Refresh() override; void RequestAvailableProducts() override; std::vector GetAvailableProducts() override; diff --git a/wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp b/wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp index b2946f38..d6ddcd7c 100644 --- a/wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp +++ b/wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp @@ -48,8 +48,6 @@ public: LoadObjectByKey(const std::string& key) override; std::shared_ptr LoadObjectByTime(std::chrono::system_clock::time_point time) override; - std::shared_ptr LoadLatestObject() override; - std::shared_ptr LoadSecondLatestObject() override; std::pair Refresh() override; protected: diff --git a/wxdata/include/scwx/provider/nexrad_data_provider.hpp b/wxdata/include/scwx/provider/nexrad_data_provider.hpp index dc490be0..2a7320d2 100644 --- a/wxdata/include/scwx/provider/nexrad_data_provider.hpp +++ b/wxdata/include/scwx/provider/nexrad_data_provider.hpp @@ -98,20 +98,6 @@ public: virtual std::shared_ptr LoadObjectByTime(std::chrono::system_clock::time_point time) = 0; - /** - * Loads the latest NEXRAD file object - * - * @return NEXRAD data - */ - virtual std::shared_ptr LoadLatestObject() = 0; - - /** - * Loads the second NEXRAD file object - * - * @return NEXRAD data - */ - virtual std::shared_ptr LoadSecondLatestObject() = 0; - /** * Lists NEXRAD objects for the current date, and adds them to the cache. If * no objects have been added to the cache for the current date, the previous diff --git a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp index bb1b31c1..672ee582 100644 --- a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp @@ -205,16 +205,20 @@ size_t AwsLevel2ChunksDataProvider::cache_size() const std::chrono::system_clock::time_point AwsLevel2ChunksDataProvider::last_modified() const { + // There is a slight delay between the "modified time" and when it is + // actually available. Radar product manager uses this as available time + static const auto extra = std::chrono::seconds(2); + const std::shared_lock lock(p->scansMutex_); if (p->currentScan_.valid_ && p->currentScan_.lastModified_ != std::chrono::system_clock::time_point {}) { - return p->currentScan_.lastModified_; + return p->currentScan_.lastModified_ + extra; } else if (p->lastScan_.valid_ && p->lastScan_.lastModified_ != std::chrono::system_clock::time_point {}) { - return p->lastScan_.lastModified_; + return p->lastScan_.lastModified_ + extra; } else { @@ -224,20 +228,18 @@ AwsLevel2ChunksDataProvider::last_modified() const std::chrono::seconds AwsLevel2ChunksDataProvider::update_period() const { const std::shared_lock lock(p->scansMutex_); - // Add an extra second of delay - static const auto extra = std::chrono::seconds(2); // get update period from time between chunks if (p->currentScan_.valid_ && p->currentScan_.nextFile_ > 2) { auto delta = p->currentScan_.lastModified_ - p->currentScan_.secondLastModified_; - return std::chrono::duration_cast(delta) + extra; + return std::chrono::duration_cast(delta); } else if (p->lastScan_.valid_ && p->lastScan_.nextFile_ > 2) { auto delta = p->lastScan_.lastModified_ - p->lastScan_.secondLastModified_; - return std::chrono::duration_cast(delta) + extra; + return std::chrono::duration_cast(delta); } // default to a set update period @@ -645,20 +647,6 @@ AwsLevel2ChunksDataProvider::LoadObjectByTime( } } -std::shared_ptr -AwsLevel2ChunksDataProvider::LoadLatestObject() -{ - const std::unique_lock lock(p->scansMutex_); - return std::make_shared(p->currentScan_.nexradFile_, - p->lastScan_.nexradFile_); -} - -std::shared_ptr -AwsLevel2ChunksDataProvider::LoadSecondLatestObject() -{ - return p->lastScan_.nexradFile_; -} - int AwsLevel2ChunksDataProvider::Impl::GetScanNumber(const std::string& prefix) { // KIND/585/20250324-134727-001-S @@ -728,7 +716,7 @@ std::pair AwsLevel2ChunksDataProvider::Refresh() }); } - threadPool.wait(); + threadPool.join(); if (newCurrent) { newObjects += 1; @@ -759,7 +747,7 @@ std::optional AwsLevel2ChunksDataProvider::GetCurrentElevation() { if (!p->currentScan_.valid_ || p->currentScan_.nexradFile_ == nullptr) { - // Does not have any scan elevation. -90 is beyond what is possible + // Does not have any scan elevation. return {}; } @@ -767,7 +755,7 @@ std::optional AwsLevel2ChunksDataProvider::GetCurrentElevation() auto radarData = p->currentScan_.nexradFile_->radar_data(); if (radarData.size() == 0) { - // Does not have any scan elevation. -90 is beyond what is possible + // Does not have any scan elevation. return {}; } diff --git a/wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp b/wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp index c0611804..dceb45d8 100644 --- a/wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp @@ -346,17 +346,6 @@ std::shared_ptr AwsNexradDataProvider::LoadObjectByTime( } } -std::shared_ptr AwsNexradDataProvider::LoadLatestObject() -{ - return LoadObjectByKey(FindLatestKey()); -} - -std::shared_ptr -AwsNexradDataProvider::LoadSecondLatestObject() -{ - return nullptr; -} - std::pair AwsNexradDataProvider::Refresh() { using namespace std::chrono; From 3288ba30ecd28a118d1d979827380d7bfe7141d5 Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Fri, 25 Apr 2025 12:24:45 -0400 Subject: [PATCH 27/34] Rework refreshing in RadarProductManager to allow for multiple refreshes at once. --- .../scwx/qt/manager/radar_product_manager.cpp | 128 +++++++++--------- 1 file changed, 65 insertions(+), 63 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp index e56be177..86f31cd4 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #if defined(_MSC_VER) # pragma warning(push, 0) @@ -87,16 +88,14 @@ class ProviderManager : public QObject Q_OBJECT public: explicit ProviderManager(RadarProductManager* self, - const std::string& radarId, - common::RadarProductGroup group) : - ProviderManager(self, radarId, group, "???") - { - } - explicit ProviderManager(RadarProductManager* self, - const std::string& radarId, + std::string radarId, common::RadarProductGroup group, - const std::string& product) : - radarId_ {radarId}, group_ {group}, product_ {product} + std::string product = "???", + bool fastRefresh = false) : + radarId_ {std::move(radarId)}, + group_ {group}, + product_ {std::move(product)}, + fastRefresh_ {fastRefresh} { connect(this, &ProviderManager::NewDataAvailable, @@ -114,10 +113,12 @@ public: const std::string radarId_; const common::RadarProductGroup group_; const std::string product_; + const bool fastRefresh_; bool refreshEnabled_ {false}; boost::asio::steady_timer refreshTimer_ {threadPool_}; std::mutex refreshTimerMutex_ {}; std::shared_ptr provider_ {nullptr}; + size_t refreshCount_ {0}; signals: void NewDataAvailable(common::RadarProductGroup group, @@ -138,7 +139,7 @@ public: level2ProviderManager_ {std::make_shared( self_, radarId_, common::RadarProductGroup::Level2)}, level2ChunksProviderManager_ {std::make_shared( - self_, radarId_, common::RadarProductGroup::Level2)} + self_, radarId_, common::RadarProductGroup::Level2, "???", true)} { if (radarSite_ == nullptr) { @@ -191,9 +192,10 @@ public: std::shared_ptr GetLevel3ProviderManager(const std::string& product); - void EnableRefresh(boost::uuids::uuid uuid, - std::shared_ptr providerManager, - bool enabled); + void EnableRefresh( + boost::uuids::uuid uuid, + const std::set>& providerManagers, + bool enabled); void RefreshData(std::shared_ptr providerManager); void RefreshDataSync(std::shared_ptr providerManager); @@ -285,7 +287,7 @@ public: std::optional incomingLevel2Elevation_ {}; std::unordered_map, + std::set>, boost::hash> refreshMap_ {}; std::shared_mutex refreshMapMutex_ {}; @@ -664,8 +666,10 @@ void RadarProductManager::EnableRefresh(common::RadarProductGroup group, { if (group == common::RadarProductGroup::Level2) { - // p->EnableRefresh(uuid, p->level2ProviderManager_, enabled); - p->EnableRefresh(uuid, p->level2ChunksProviderManager_, enabled); + p->EnableRefresh( + uuid, + {p->level2ProviderManager_, p->level2ChunksProviderManager_}, + enabled); } else { @@ -688,7 +692,7 @@ void RadarProductManager::EnableRefresh(common::RadarProductGroup group, availableProducts.cend(), product) != availableProducts.cend()) { - p->EnableRefresh(uuid, providerManager, enabled); + p->EnableRefresh(uuid, {providerManager}, enabled); } } catch (const std::exception& ex) @@ -700,50 +704,45 @@ void RadarProductManager::EnableRefresh(common::RadarProductGroup group, } void RadarProductManagerImpl::EnableRefresh( - boost::uuids::uuid uuid, - std::shared_ptr providerManager, - bool enabled) + boost::uuids::uuid uuid, + const std::set>& providerManagers, + bool enabled) { // Lock the refresh map std::unique_lock lock {refreshMapMutex_}; - auto currentProviderManager = refreshMap_.find(uuid); - if (currentProviderManager != refreshMap_.cend()) + auto currentProviderManagers = refreshMap_.find(uuid); + if (currentProviderManagers != refreshMap_.cend()) { - // If the enabling refresh for a different product, or disabling refresh - if (currentProviderManager->second != providerManager || !enabled) + for (const auto& currentProviderManager : currentProviderManagers->second) { - // Determine number of entries in the map for the current provider - // manager - auto currentProviderManagerCount = std::count_if( - refreshMap_.cbegin(), - refreshMap_.cend(), - [&](const auto& provider) - { return provider.second == currentProviderManager->second; }); - - // If this is the last reference to the provider in the refresh map - if (currentProviderManagerCount == 1) + currentProviderManager->refreshCount_ -= 1; + // If the enabling refresh for a different product, or disabling + // refresh + if (!providerManagers.contains(currentProviderManager) || !enabled) { - // Disable current provider - currentProviderManager->second->Disable(); - } - - // Dissociate uuid from current provider manager - refreshMap_.erase(currentProviderManager); - - // If we are enabling a new provider manager - if (enabled) - { - // Associate uuid to providerManager - refreshMap_.emplace(uuid, providerManager); + // If this is the last reference to the provider in the refresh map + if (currentProviderManager->refreshCount_ == 0) + { + // Disable current provider + currentProviderManager->Disable(); + } } } + + // Dissociate uuid from current provider managers + refreshMap_.erase(currentProviderManagers); } - else if (enabled) + + if (enabled) { - // We are enabling a new provider manager + // We are enabling provider managers // Associate uuid to provider manager - refreshMap_.emplace(uuid, providerManager); + refreshMap_.emplace(uuid, providerManagers); + for (const auto& providerManager : providerManagers) + { + providerManager->refreshCount_ += 1; + } } // Release the refresh map mutex @@ -751,13 +750,15 @@ void RadarProductManagerImpl::EnableRefresh( // We have already handled a disable request by this point. If enabling, and // the provider manager refresh isn't already enabled, enable it. - if (enabled && providerManager->refreshEnabled_ != enabled) + if (enabled) { - providerManager->refreshEnabled_ = enabled; - - if (enabled) + for (const auto& providerManager : providerManagers) { - RefreshData(providerManager); + if (providerManager->refreshEnabled_ != enabled) + { + providerManager->refreshEnabled_ = enabled; + RefreshData(providerManager); + } } } } @@ -795,13 +796,11 @@ void RadarProductManagerImpl::RefreshDataSync( // Level2 chunked data is updated quickly and uses a faster interval const std::chrono::milliseconds fastRetryInterval = - providerManager == level2ChunksProviderManager_ ? - kFastRetryIntervalChunks_ : - kFastRetryInterval_; + providerManager->fastRefresh_ ? kFastRetryIntervalChunks_ : + kFastRetryInterval_; const std::chrono::milliseconds slowRetryInterval = - providerManager == level2ChunksProviderManager_ ? - kSlowRetryIntervalChunks_ : - kSlowRetryInterval_; + providerManager->fastRefresh_ ? kSlowRetryIntervalChunks_ : + kSlowRetryInterval_; std::chrono::milliseconds interval = fastRetryInterval; if (totalObjects > 0) @@ -896,10 +895,13 @@ RadarProductManager::GetActiveVolumeTimes( std::shared_lock refreshLock {p->refreshMapMutex_}; // For each entry in the refresh map (refresh is enabled) - for (auto& refreshEntry : p->refreshMap_) + for (auto& refreshSet : p->refreshMap_) { - // Add the provider for the current entry - providers.insert(refreshEntry.second->provider_); + for (const auto& refreshEntry : refreshSet.second) + { + // Add the provider for the current entry + providers.insert(refreshEntry->provider_); + } } // Unlock the refresh map From 2821eff71f7df099db3d03e46dcd4c874ffe1ce0 Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Fri, 25 Apr 2025 13:11:57 -0400 Subject: [PATCH 28/34] Fall back to archive if chunks get too old --- .../scwx/qt/manager/radar_product_manager.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp index 86f31cd4..ea8546f2 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp @@ -1548,6 +1548,12 @@ RadarProductManager::GetLevel2Data(wsr88d::rda::DataBlockType dataBlockType, std::vector elevationCuts {}; std::chrono::system_clock::time_point foundTime {}; + const bool isEpox = time == std::chrono::system_clock::time_point{}; + bool needArchive = true; + static const auto maxChunkDelay = std::chrono::minutes(10); + const std::chrono::system_clock::time_point firstValidChunkTime = + (isEpox ? std::chrono::system_clock::now() : time) - maxChunkDelay; + // See if we have this one in the chunk provider. auto chunkFile = std::dynamic_pointer_cast( p->level2ChunksProviderManager_->provider_->LoadObjectByTime(time)); @@ -1572,9 +1578,16 @@ RadarProductManager::GetLevel2Data(wsr88d::rda::DataBlockType dataBlockType, p->incomingLevel2Elevation_ = incomingElevation; Q_EMIT IncomingLevel2ElevationChanged(incomingElevation); } + + if (foundTime >= firstValidChunkTime) + { + needArchive = false; + } } } - else // It is not in the chunk provider, so get it from the archive + + // It is not in the chunk provider, so get it from the archive + if (needArchive) { auto records = p->GetLevel2ProductRecords(time); for (auto& recordPair : records) @@ -1601,7 +1614,8 @@ RadarProductManager::GetLevel2Data(wsr88d::rda::DataBlockType dataBlockType, // Find the newest radar data, not newer than the selected time if (radarData == nullptr || - (collectionTime <= time && foundTime < collectionTime)) + (collectionTime <= time && foundTime < collectionTime) || + (isEpox && foundTime < collectionTime)) { radarData = recordRadarData; elevationCut = recordElevationCut; From 781aa40e8ce4dcd71637f3897d73faf8973b8a9f Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Fri, 25 Apr 2025 13:31:19 -0400 Subject: [PATCH 29/34] Make radar data fall back if it ends up being too old --- scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp index ea8546f2..8cd3f837 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp @@ -1548,8 +1548,8 @@ RadarProductManager::GetLevel2Data(wsr88d::rda::DataBlockType dataBlockType, std::vector elevationCuts {}; std::chrono::system_clock::time_point foundTime {}; - const bool isEpox = time == std::chrono::system_clock::time_point{}; - bool needArchive = true; + const bool isEpox = time == std::chrono::system_clock::time_point {}; + bool needArchive = true; static const auto maxChunkDelay = std::chrono::minutes(10); const std::chrono::system_clock::time_point firstValidChunkTime = (isEpox ? std::chrono::system_clock::now() : time) - maxChunkDelay; From 3d7da7d9710da90f55c58bb7fee0940ff12923d0 Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Sat, 26 Apr 2025 18:51:11 -0400 Subject: [PATCH 30/34] Disable logging for level 2 chunks --- .../source/scwx/qt/manager/radar_product_manager.cpp | 10 ++++++---- .../scwx/provider/aws_level2_chunks_data_provider.cpp | 6 +++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp index 8cd3f837..b7d99467 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp @@ -766,7 +766,7 @@ void RadarProductManagerImpl::EnableRefresh( void RadarProductManagerImpl::RefreshData( std::shared_ptr providerManager) { - logger_->debug("RefreshData: {}", providerManager->name()); + // logger_->debug("RefreshData: {}", providerManager->name()); { std::unique_lock lock(providerManager->refreshTimerMutex_); @@ -846,10 +846,12 @@ void RadarProductManagerImpl::RefreshDataSync( if (providerManager->refreshEnabled_) { + /* logger_->debug( "[{}] Scheduled refresh in {:%M:%S}", providerManager->name(), std::chrono::duration_cast(interval)); + */ { providerManager->refreshTimer_.expires_after(interval); @@ -960,9 +962,9 @@ void RadarProductManagerImpl::LoadProviderData( std::mutex& loadDataMutex, const std::shared_ptr& request) { - logger_->debug("LoadProviderData: {}, {}", + /*logger_->debug("LoadProviderData: {}, {}", providerManager->name(), - scwx::util::TimeString(time)); + scwx::util::TimeString(time));*/ LoadNexradFileAsync( [=, &recordMap, &recordMutex]() -> std::shared_ptr @@ -1013,7 +1015,7 @@ void RadarProductManager::LoadLevel2Data( std::chrono::system_clock::time_point time, const std::shared_ptr& request) { - logger_->debug("LoadLevel2Data: {}", scwx::util::TimeString(time)); + // logger_->debug("LoadLevel2Data: {}", scwx::util::TimeString(time)); p->LoadProviderData(time, p->level2ProviderManager_, diff --git a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp index 672ee582..27b062b7 100644 --- a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp @@ -356,7 +356,7 @@ AwsLevel2ChunksDataProvider::Impl::ListObjects() if (outcome.IsSuccess()) { auto& scans = outcome.GetResult().GetCommonPrefixes(); - logger_->debug("Found {} scans", scans.size()); + // logger_->debug("Found {} scans", scans.size()); if (scans.size() > 0) { @@ -514,7 +514,7 @@ bool AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) bool hasNew = false; auto& chunks = listOutcome.GetResult().GetContents(); - logger_->debug("Found {} new chunks.", chunks.size()); + // logger_->debug("Found {} new chunks.", chunks.size()); for (const auto& chunk : chunks) { const std::string& key = chunk.GetKey(); @@ -728,7 +728,7 @@ std::pair AwsLevel2ChunksDataProvider::Refresh() timer.stop(); // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) format to 6 digits - logger_->debug("Refresh() in {}", timer.format(6, "%ws")); + // logger_->debug("Refresh() in {}", timer.format(6, "%ws")); return std::make_pair(newObjects, totalObjects); } From 4906800a22be312dc5facb2181fb1afb7883ab54 Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Wed, 7 May 2025 17:16:42 -0400 Subject: [PATCH 31/34] Resolve TODOs in level_2_chunks --- .../aws_level2_chunks_data_provider.cpp | 2 -- wxdata/source/scwx/wsr88d/ar2v_file.cpp | 21 +++++++++---------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp index 27b062b7..64ce04b0 100644 --- a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp @@ -405,7 +405,6 @@ AwsLevel2ChunksDataProvider::Impl::ListObjects() if (possibleLastNumbers.empty()) { logger_->warn("Could not find last scan"); - // TODO make sure this makes sence return {false, 0, 0}; } @@ -705,7 +704,6 @@ std::pair AwsLevel2ChunksDataProvider::Refresh() if (p->lastScan_.nexradFile_ != nullptr) { p->lastScan_.hasAllFiles_ = true; - // TODO maybe set lastModified for timing } } // Fall back to chunks if files did not load diff --git a/wxdata/source/scwx/wsr88d/ar2v_file.cpp b/wxdata/source/scwx/wsr88d/ar2v_file.cpp index b4dc6248..7772ea44 100644 --- a/wxdata/source/scwx/wsr88d/ar2v_file.cpp +++ b/wxdata/source/scwx/wsr88d/ar2v_file.cpp @@ -528,7 +528,6 @@ bool Ar2vFile::IndexFile() return true; } -// TODO not good // NOLINTNEXTLINE bool IsRadarDataIncomplete( const std::shared_ptr& radarData) @@ -583,10 +582,10 @@ Ar2vFile::Ar2vFile(const std::shared_ptr& current, std::shared_ptr secondMostRecent = nullptr; // check if this volume scan has an earlier elevation scan - auto maybe = elevation.second.rbegin(); // TODO name - ++maybe; + auto possibleSecondMostRecent = elevation.second.rbegin(); + ++possibleSecondMostRecent; - if (maybe == elevation.second.rend()) + if (possibleSecondMostRecent == elevation.second.rend()) { if (last == nullptr) { @@ -613,7 +612,7 @@ Ar2vFile::Ar2vFile(const std::shared_ptr& current, } else { - secondMostRecent = maybe->second; + secondMostRecent = possibleSecondMostRecent->second; } // Make the new scan @@ -706,16 +705,16 @@ Ar2vFile::Ar2vFile(const std::shared_ptr& current, // Find the highest elevation this type has for the current scan // Start below any reasonable elevation // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) - float highestCurrentElevation = -90; - const auto& maybe1 = p->index_.find(type.first); - if (maybe1 != p->index_.cend()) + float highestCurrentElevation = -90; + const auto& elevationScans = p->index_.find(type.first); + if (elevationScans != p->index_.cend()) { - const auto& maybe2 = maybe1->second.crbegin(); - if (maybe2 != maybe1->second.crend()) + const auto& highestElevation = elevationScans->second.crbegin(); + if (highestElevation != elevationScans->second.crend()) { // Add a slight offset to ensure good floating point compare. // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) - highestCurrentElevation = maybe2->first + 0.01f; + highestCurrentElevation = highestElevation->first + 0.01f; } } From 969267b661c838bb94b61e99f59064d9adc65cf4 Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Wed, 7 May 2025 17:33:44 -0400 Subject: [PATCH 32/34] Added back logging as traces for level_2_chunks --- .../scwx/qt/manager/radar_product_manager.cpp | 23 ++++++++----------- .../scwx/qt/map/radar_product_layer.cpp | 2 +- .../aws_level2_chunks_data_provider.cpp | 8 +++---- wxdata/source/scwx/wsr88d/ar2v_file.cpp | 14 +++++------ 4 files changed, 21 insertions(+), 26 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp index b7d99467..d6b85685 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp @@ -766,7 +766,7 @@ void RadarProductManagerImpl::EnableRefresh( void RadarProductManagerImpl::RefreshData( std::shared_ptr providerManager) { - // logger_->debug("RefreshData: {}", providerManager->name()); + logger_->trace("RefreshData: {}", providerManager->name()); { std::unique_lock lock(providerManager->refreshTimerMutex_); @@ -846,12 +846,10 @@ void RadarProductManagerImpl::RefreshDataSync( if (providerManager->refreshEnabled_) { - /* - logger_->debug( + logger_->trace( "[{}] Scheduled refresh in {:%M:%S}", providerManager->name(), std::chrono::duration_cast(interval)); - */ { providerManager->refreshTimer_.expires_after(interval); @@ -962,9 +960,9 @@ void RadarProductManagerImpl::LoadProviderData( std::mutex& loadDataMutex, const std::shared_ptr& request) { - /*logger_->debug("LoadProviderData: {}, {}", + logger_->trace("LoadProviderData: {}, {}", providerManager->name(), - scwx::util::TimeString(time));*/ + scwx::util::TimeString(time)); LoadNexradFileAsync( [=, &recordMap, &recordMutex]() -> std::shared_ptr @@ -980,13 +978,11 @@ void RadarProductManagerImpl::LoadProviderData( { existingRecord = it->second.lock(); - /* if (existingRecord != nullptr) { - logger_->debug( + logger_->trace( "Data previously loaded, loading from data cache"); } - */ } } @@ -1015,7 +1011,7 @@ void RadarProductManager::LoadLevel2Data( std::chrono::system_clock::time_point time, const std::shared_ptr& request) { - // logger_->debug("LoadLevel2Data: {}", scwx::util::TimeString(time)); + logger_->trace("LoadLevel2Data: {}", scwx::util::TimeString(time)); p->LoadProviderData(time, p->level2ProviderManager_, @@ -1445,7 +1441,7 @@ std::shared_ptr RadarProductManagerImpl::StoreRadarProductRecord( std::shared_ptr record) { - // logger_->debug("StoreRadarProductRecord()"); + logger_->trace("StoreRadarProductRecord()"); std::shared_ptr storedRecord = nullptr; @@ -1462,12 +1458,11 @@ RadarProductManagerImpl::StoreRadarProductRecord( { storedRecord = it->second.lock(); - /* if (storedRecord != nullptr) { - logger_->debug( + logger_->trace( "Level 2 product previously loaded, loading from cache"); - }*/ + } } if (storedRecord == nullptr) diff --git a/scwx-qt/source/scwx/qt/map/radar_product_layer.cpp b/scwx-qt/source/scwx/qt/map/radar_product_layer.cpp index ee6b3c3c..7bda264f 100644 --- a/scwx-qt/source/scwx/qt/map/radar_product_layer.cpp +++ b/scwx-qt/source/scwx/qt/map/radar_product_layer.cpp @@ -170,7 +170,7 @@ void RadarProductLayer::UpdateSweep() std::try_to_lock); if (!sweepLock.owns_lock()) { - // logger_->debug("Sweep locked, deferring update"); + logger_->trace("Sweep locked, deferring update"); return; } logger_->debug("UpdateSweep()"); diff --git a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp index 64ce04b0..74a9afd7 100644 --- a/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp @@ -341,7 +341,7 @@ AwsLevel2ChunksDataProvider::Impl::ListObjects() { return {true, newObjects, totalObjects}; } - logger_->debug("ListObjects"); + logger_->trace("ListObjects"); lastTimeListed_ = now; const std::string prefix = radarSite_ + "/"; @@ -356,7 +356,7 @@ AwsLevel2ChunksDataProvider::Impl::ListObjects() if (outcome.IsSuccess()) { auto& scans = outcome.GetResult().GetCommonPrefixes(); - // logger_->debug("Found {} scans", scans.size()); + logger_->trace("Found {} scans", scans.size()); if (scans.size() > 0) { @@ -513,7 +513,7 @@ bool AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord) bool hasNew = false; auto& chunks = listOutcome.GetResult().GetContents(); - // logger_->debug("Found {} new chunks.", chunks.size()); + logger_->trace("Found {} new chunks.", chunks.size()); for (const auto& chunk : chunks) { const std::string& key = chunk.GetKey(); @@ -726,7 +726,7 @@ std::pair AwsLevel2ChunksDataProvider::Refresh() timer.stop(); // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) format to 6 digits - // logger_->debug("Refresh() in {}", timer.format(6, "%ws")); + logger_->debug("Refresh() in {}", timer.format(6, "%ws")); return std::make_pair(newObjects, totalObjects); } diff --git a/wxdata/source/scwx/wsr88d/ar2v_file.cpp b/wxdata/source/scwx/wsr88d/ar2v_file.cpp index 7772ea44..3ee99fe1 100644 --- a/wxdata/source/scwx/wsr88d/ar2v_file.cpp +++ b/wxdata/source/scwx/wsr88d/ar2v_file.cpp @@ -138,7 +138,7 @@ Ar2vFile::GetElevationScan(rda::DataBlockType dataBlockType, float elevation, std::chrono::system_clock::time_point time) const { - // logger_->debug("GetElevationScan: {} degrees", elevation); + logger_->trace("GetElevationScan: {} degrees", elevation); std::shared_ptr elevationScan = nullptr; float elevationCut = 0.0f; @@ -273,7 +273,7 @@ bool Ar2vFile::LoadData(std::istream& is) std::size_t Ar2vFileImpl::DecompressLDMRecords(std::istream& is) { - // logger_->debug("Decompressing LDM Records"); + logger_->trace("Decompressing LDM Records"); std::size_t numRecords = 0; @@ -321,22 +321,22 @@ std::size_t Ar2vFileImpl::DecompressLDMRecords(std::istream& is) ++numRecords; } - // logger_->debug("Decompressed {} LDM Records", numRecords); + logger_->trace("Decompressed {} LDM Records", numRecords); return numRecords; } void Ar2vFileImpl::ParseLDMRecords() { - // logger_->debug("Parsing LDM Records"); + logger_->trace("Parsing LDM Records"); - // std::size_t count = 0; + std::size_t count = 0; for (auto it = rawRecords_.begin(); it != rawRecords_.end(); it++) { std::stringstream& ss = *it; - // logger_->trace("Record {}", count++); + logger_->trace("Record {}", count++); ParseLDMRecord(ss); } @@ -445,7 +445,7 @@ void Ar2vFileImpl::ProcessRadarData( void Ar2vFileImpl::IndexFile() { - // logger_->debug("Indexing file"); + logger_->trace("Indexing file"); for (auto& elevationCut : radarData_) { From 0438b65208d73003ec218c077df16c23deab2906 Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Wed, 7 May 2025 17:44:28 -0400 Subject: [PATCH 33/34] Fix clang format errors for level_2_chunks --- wxdata/source/scwx/wsr88d/ar2v_file.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wxdata/source/scwx/wsr88d/ar2v_file.cpp b/wxdata/source/scwx/wsr88d/ar2v_file.cpp index 3ee99fe1..f4a71c15 100644 --- a/wxdata/source/scwx/wsr88d/ar2v_file.cpp +++ b/wxdata/source/scwx/wsr88d/ar2v_file.cpp @@ -467,7 +467,7 @@ void Ar2vFileImpl::IndexFile() { elevationAngle = static_cast(vcpData_->elevation_angle(elevationCut.first)); - waveformType = vcpData_->waveform_type(elevationCut.first); + waveformType = vcpData_->waveform_type(elevationCut.first); } else if ((digitalRadarData0 = std::dynamic_pointer_cast(radial0)) != @@ -705,8 +705,8 @@ Ar2vFile::Ar2vFile(const std::shared_ptr& current, // Find the highest elevation this type has for the current scan // Start below any reasonable elevation // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) - float highestCurrentElevation = -90; - const auto& elevationScans = p->index_.find(type.first); + float highestCurrentElevation = -90; + const auto& elevationScans = p->index_.find(type.first); if (elevationScans != p->index_.cend()) { const auto& highestElevation = elevationScans->second.crbegin(); From 8989c0e88ce044560c5b5aa7c9c21c18d7bf34e7 Mon Sep 17 00:00:00 2001 From: AdenKoperczak Date: Sat, 10 May 2025 15:06:46 -0400 Subject: [PATCH 34/34] Fix issue where level 2 archive files where put in a cache at times of level 2 chunk files --- .../scwx/qt/manager/radar_product_manager.cpp | 32 ++--- .../scwx/qt/manager/radar_product_manager.hpp | 1 + scwx-qt/source/scwx/qt/map/map_widget.cpp | 117 ++++++++++-------- .../scwx/qt/view/overlay_product_view.cpp | 5 +- 4 files changed, 84 insertions(+), 71 deletions(-) diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp index d6b85685..81f0fc52 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp @@ -90,17 +90,23 @@ public: explicit ProviderManager(RadarProductManager* self, std::string radarId, common::RadarProductGroup group, - std::string product = "???", - bool fastRefresh = false) : + std::string product = "???", + bool isChunks = false) : radarId_ {std::move(radarId)}, group_ {group}, product_ {std::move(product)}, - fastRefresh_ {fastRefresh} + isChunks_ {isChunks} { connect(this, &ProviderManager::NewDataAvailable, self, - &RadarProductManager::NewDataAvailable); + [this, self](common::RadarProductGroup group, + const std::string& product, + std::chrono::system_clock::time_point latestTime) + { + Q_EMIT self->NewDataAvailable( + group, product, isChunks_, latestTime); + }); } ~ProviderManager() { threadPool_.join(); }; @@ -113,7 +119,7 @@ public: const std::string radarId_; const common::RadarProductGroup group_; const std::string product_; - const bool fastRefresh_; + const bool isChunks_; bool refreshEnabled_ {false}; boost::asio::steady_timer refreshTimer_ {threadPool_}; std::mutex refreshTimerMutex_ {}; @@ -796,11 +802,11 @@ void RadarProductManagerImpl::RefreshDataSync( // Level2 chunked data is updated quickly and uses a faster interval const std::chrono::milliseconds fastRetryInterval = - providerManager->fastRefresh_ ? kFastRetryIntervalChunks_ : - kFastRetryInterval_; + providerManager->isChunks_ ? kFastRetryIntervalChunks_ : + kFastRetryInterval_; const std::chrono::milliseconds slowRetryInterval = - providerManager->fastRefresh_ ? kSlowRetryIntervalChunks_ : - kSlowRetryInterval_; + providerManager->isChunks_ ? kSlowRetryIntervalChunks_ : + kSlowRetryInterval_; std::chrono::milliseconds interval = fastRetryInterval; if (totalObjects > 0) @@ -1019,12 +1025,6 @@ void RadarProductManager::LoadLevel2Data( p->level2ProductRecordMutex_, p->loadLevel2DataMutex_, request); - p->LoadProviderData(time, - p->level2ChunksProviderManager_, - p->level2ProductRecords_, - p->level2ProductRecordMutex_, - p->loadLevel2DataMutex_, - request); } void RadarProductManager::LoadLevel3Data( @@ -1460,7 +1460,7 @@ RadarProductManagerImpl::StoreRadarProductRecord( if (storedRecord != nullptr) { - logger_->trace( + logger_->error( "Level 2 product previously loaded, loading from cache"); } } diff --git a/scwx-qt/source/scwx/qt/manager/radar_product_manager.hpp b/scwx-qt/source/scwx/qt/manager/radar_product_manager.hpp index c0a49dff..e8c72193 100644 --- a/scwx-qt/source/scwx/qt/manager/radar_product_manager.hpp +++ b/scwx-qt/source/scwx/qt/manager/radar_product_manager.hpp @@ -148,6 +148,7 @@ signals: void Level3ProductsChanged(); void NewDataAvailable(common::RadarProductGroup group, const std::string& product, + bool isChunks, std::chrono::system_clock::time_point latestTime); void IncomingLevel2ElevationChanged(std::optional incomingElevation); diff --git a/scwx-qt/source/scwx/qt/map/map_widget.cpp b/scwx-qt/source/scwx/qt/map/map_widget.cpp index ad808626..e8d0e380 100644 --- a/scwx-qt/source/scwx/qt/map/map_widget.cpp +++ b/scwx-qt/source/scwx/qt/map/map_widget.cpp @@ -1843,6 +1843,7 @@ void MapWidgetImpl::RadarProductManagerConnect() this, [this](common::RadarProductGroup group, const std::string& product, + bool isChunks, std::chrono::system_clock::time_point latestTime) { if (autoRefreshEnabled_ && @@ -1850,71 +1851,81 @@ void MapWidgetImpl::RadarProductManagerConnect() (group == common::RadarProductGroup::Level2 || context_->radar_product() == product)) { - // Create file request - std::shared_ptr request = - std::make_shared( - radarProductManager_->radar_id()); - - // File request callback - if (autoUpdateEnabled_) + if (isChunks && autoUpdateEnabled_) { - connect( - request.get(), - &request::NexradFileRequest::RequestComplete, - this, - [=, - this](std::shared_ptr request) - { - // Select loaded record - auto record = request->radar_product_record(); + // Level 2 products may have multiple time points, + // ensure the latest is selected + widget_->SelectRadarProduct(group, product); + } + else + { + // Create file request + const std::shared_ptr request = + std::make_shared( + radarProductManager_->radar_id()); - // Validate record, and verify current map context - // still displays site and product - if (record != nullptr && - radarProductManager_ != nullptr && - radarProductManager_->radar_id() == - request->current_radar_site() && - context_->radar_product_group() == group && - (group == common::RadarProductGroup::Level2 || - context_->radar_product() == product)) + // File request callback + if (autoUpdateEnabled_) + { + connect( + request.get(), + &request::NexradFileRequest::RequestComplete, + this, + [group, product, this]( + const std::shared_ptr& + request) + { + // Select loaded record + auto record = request->radar_product_record(); + + // Validate record, and verify current map context + // still displays site and product + if (record != nullptr && + radarProductManager_ != nullptr && + radarProductManager_->radar_id() == + request->current_radar_site() && + context_->radar_product_group() == group && + (group == common::RadarProductGroup::Level2 || + context_->radar_product() == product)) + { + if (group == common::RadarProductGroup::Level2) + { + // Level 2 products may have multiple time + // points, ensure the latest is selected + widget_->SelectRadarProduct(group, product); + } + else + { + widget_->SelectRadarProduct(record); + } + } + }); + } + + // Load file + boost::asio::post( + threadPool_, + [group, latestTime, request, product, this]() + { + try { if (group == common::RadarProductGroup::Level2) { - // Level 2 products may have multiple time points, - // ensure the latest is selected - widget_->SelectRadarProduct(group, product); + radarProductManager_->LoadLevel2Data(latestTime, + request); } else { - widget_->SelectRadarProduct(record); + radarProductManager_->LoadLevel3Data( + product, latestTime, request); } } + catch (const std::exception& ex) + { + logger_->error(ex.what()); + } }); } - - // Load file - boost::asio::post( - threadPool_, - [=, this]() - { - try - { - if (group == common::RadarProductGroup::Level2) - { - radarProductManager_->LoadLevel2Data(latestTime, - request); - } - else - { - radarProductManager_->LoadLevel3Data( - product, latestTime, request); - } - } - catch (const std::exception& ex) - { - logger_->error(ex.what()); - } - }); } }, Qt::QueuedConnection); diff --git a/scwx-qt/source/scwx/qt/view/overlay_product_view.cpp b/scwx-qt/source/scwx/qt/view/overlay_product_view.cpp index c33494c2..3200dcca 100644 --- a/scwx-qt/source/scwx/qt/view/overlay_product_view.cpp +++ b/scwx-qt/source/scwx/qt/view/overlay_product_view.cpp @@ -116,8 +116,9 @@ void OverlayProductView::Impl::ConnectRadarProductManager() radarProductManager_.get(), &manager::RadarProductManager::NewDataAvailable, self_, - [this](common::RadarProductGroup group, - const std::string& product, + [this](common::RadarProductGroup group, + const std::string& product, + bool /*isChunks*/, std::chrono::system_clock::time_point latestTime) { if (autoRefreshEnabled_ &&