From 05d795d18fc58c4effba5b75e16ec14490fa4064 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Thu, 9 Jun 2022 23:53:34 -0500 Subject: [PATCH] List available level 3 products --- .../provider/aws_nexrad_data_provider.hpp | 4 + .../provider/aws_level3_data_provider.cpp | 88 ++++++++++++++++++- .../provider/aws_nexrad_data_provider.cpp | 10 ++- 3 files changed, 95 insertions(+), 7 deletions(-) diff --git a/wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp b/wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp index ca393c69..999a4952 100644 --- a/wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp +++ b/wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp @@ -2,6 +2,8 @@ #include +#include + namespace scwx { namespace provider @@ -37,6 +39,8 @@ public: std::pair Refresh(); protected: + std::shared_ptr client(); + virtual std::string GetPrefix(std::chrono::system_clock::time_point date) = 0; diff --git a/wxdata/source/scwx/provider/aws_level3_data_provider.cpp b/wxdata/source/scwx/provider/aws_level3_data_provider.cpp index 48410974..eac3171c 100644 --- a/wxdata/source/scwx/provider/aws_level3_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level3_data_provider.cpp @@ -3,6 +3,9 @@ #include #include +#include + +#include #include #include @@ -18,21 +21,33 @@ static const auto logger_ = util::Logger::Create(logPrefix_); static const std::string kDefaultBucketName_ = "unidata-nexrad-level3"; static const std::string kDefaultRegion_ = "us-east-1"; +std::unordered_map> productMap_; +std::shared_mutex productMutex_; + class AwsLevel3DataProvider::Impl { public: - explicit Impl(const std::string& radarSite, const std::string& product) : + explicit Impl(AwsLevel3DataProvider* self, + const std::string& radarSite, + const std::string& product, + const std::string& bucketName) : + self_ {self}, radarSite_ {radarSite}, siteId_ {common::GetSiteId(radarSite_)}, - product_ {product} + product_ {product}, + bucketName_ {bucketName} { } + ~Impl() = default; - ~Impl() {} + void ListProducts(); + + AwsLevel3DataProvider* self_; std::string radarSite_; std::string siteId_; std::string product_; + std::string bucketName_; }; AwsLevel3DataProvider::AwsLevel3DataProvider(const std::string& radarSite, @@ -46,7 +61,7 @@ AwsLevel3DataProvider::AwsLevel3DataProvider(const std::string& radarSite, const std::string& bucketName, const std::string& region) : AwsNexradDataProvider(radarSite, bucketName, region), - p(std::make_unique(radarSite, product)) + p(std::make_unique(this, radarSite, product, bucketName)) { } AwsLevel3DataProvider::~AwsLevel3DataProvider() = default; @@ -103,5 +118,70 @@ AwsLevel3DataProvider::GetTimePointFromKey(const std::string& key) return time; } +void AwsLevel3DataProvider::Impl::ListProducts() +{ + std::shared_lock readLock(productMutex_); + + // Only list once per radar site + if (productMap_.contains(radarSite_)) + { + return; + } + + readLock.unlock(); + + // Prefix format: GGG_ + const std::string prefix = fmt::format("{0}_", siteId_); + + Aws::S3::Model::ListObjectsV2Request request; + request.SetBucket(bucketName_); + request.SetPrefix(prefix); + request.SetDelimiter("_"); + + auto outcome = self_->client()->ListObjectsV2(request); + + if (outcome.IsSuccess()) + { + std::unique_lock writeLock(productMutex_); + + // If the product was populated since first checked, don't re-populate + if (productMap_.contains(radarSite_)) + { + return; + } + + auto& prefixes = outcome.GetResult().GetCommonPrefixes(); + + // Create a vector with reserved capacity + std::vector productList; + productList.reserve(prefixes.size()); + + std::for_each(prefixes.cbegin(), + prefixes.cend(), + [&](const Aws::S3::Model::CommonPrefix& commonPrefix) + { + // Prefix format: GGG_PPP_ + std::string prefix = commonPrefix.GetPrefix(); + size_t left = prefix.find('_'); + size_t right = prefix.rfind('_'); + + // If left != npos, right != npos + if (left != std::string::npos && right > left) + { + // The product starts after the left delimeter, and + // ends before the right delimeter. + ++left; + productList.push_back( + prefix.substr(left, right - left)); + } + }); + + // Remove extra capacity if necessary + productList.shrink_to_fit(); + + productMap_.emplace(radarSite_, std::move(productList)); + } +} + } // namespace provider } // namespace scwx diff --git a/wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp b/wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp index e8512ca2..c23e1e25 100644 --- a/wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp @@ -6,7 +6,6 @@ #include -#include #include #include @@ -55,7 +54,7 @@ public: Aws::Client::ClientConfiguration config; config.region = region_; - client_ = std::make_unique(config); + client_ = std::make_shared(config); } ~Impl() {} @@ -68,7 +67,7 @@ public: std::string bucketName_; std::string region_; - std::unique_ptr client_; + std::shared_ptr client_; std::map objects_; std::shared_mutex objectsMutex_; @@ -96,6 +95,11 @@ size_t AwsNexradDataProvider::cache_size() const return p->objects_.size(); } +std::shared_ptr AwsNexradDataProvider::client() +{ + return p->client_; +} + std::chrono::seconds AwsNexradDataProvider::update_period() const { return p->updatePeriod_;