From 1681b6772b2590e6c8f31d28319104a960376863 Mon Sep 17 00:00:00 2001 From: Dan Paulat Date: Fri, 22 Apr 2022 09:26:45 -0500 Subject: [PATCH] Add initial AWS Level 2 Data Provider --- .../aws_level2_data_provider.test.cpp | 31 ++++ test/test.cmake | 3 + .../provider/aws_level2_data_provider.hpp | 40 ++++++ .../provider/aws_level2_data_provider.cpp | 134 ++++++++++++++++++ wxdata/wxdata.cmake | 6 + 5 files changed, 214 insertions(+) create mode 100644 test/source/scwx/provider/aws_level2_data_provider.test.cpp create mode 100644 wxdata/include/scwx/provider/aws_level2_data_provider.hpp create mode 100644 wxdata/source/scwx/provider/aws_level2_data_provider.cpp diff --git a/test/source/scwx/provider/aws_level2_data_provider.test.cpp b/test/source/scwx/provider/aws_level2_data_provider.test.cpp new file mode 100644 index 00000000..a245b177 --- /dev/null +++ b/test/source/scwx/provider/aws_level2_data_provider.test.cpp @@ -0,0 +1,31 @@ +#include + +#include + +namespace scwx +{ +namespace provider +{ + +TEST(AwsLevel2DataProvider, LoadObjectByKey) +{ + const std::string key = "2022/04/21/KLSX/KLSX20220421_160055_V06"; + + AwsLevel2DataProvider provider("KLSX"); + + auto file = provider.LoadObjectByKey(key); + + EXPECT_NE(file, nullptr); +} + +TEST(AwsLevel2DataProvider, Refresh) +{ + AwsLevel2DataProvider provider("KLSX"); + + provider.Refresh(); + + // TODO: Check object count +} + +} // namespace provider +} // namespace scwx diff --git a/test/test.cmake b/test/test.cmake index 2820dc09..d5c4042c 100644 --- a/test/test.cmake +++ b/test/test.cmake @@ -13,6 +13,7 @@ set(SRC_AWIPS_TESTS source/scwx/awips/coded_location.test.cpp source/scwx/awips/pvtec.test.cpp source/scwx/awips/text_product_file.test.cpp) set(SRC_COMMON_TESTS source/scwx/common/color_table.test.cpp) +set(SRC_PROVIDER_TESTS source/scwx/provider/aws_level2_data_provider.test.cpp) set(SRC_QT_CONFIG_TESTS source/scwx/qt/config/radar_site.test.cpp) set(SRC_QT_MANAGER_TESTS source/scwx/qt/manager/settings_manager.test.cpp) set(SRC_UTIL_TESTS source/scwx/util/float.test.cpp @@ -28,6 +29,7 @@ set(CMAKE_FILES test.cmake) add_executable(wxtest ${SRC_MAIN} ${SRC_AWIPS_TESTS} ${SRC_COMMON_TESTS} + ${SRC_PROVIDER_TESTS} ${SRC_QT_CONFIG_TESTS} ${SRC_QT_MANAGER_TESTS} ${SRC_UTIL_TESTS} @@ -37,6 +39,7 @@ add_executable(wxtest ${SRC_MAIN} source_group("Source Files\\main" FILES ${SRC_MAIN}) source_group("Source Files\\awips" FILES ${SRC_AWIPS_TESTS}) source_group("Source Files\\common" FILES ${SRC_COMMON_TESTS}) +source_group("Source Files\\provider" FILES ${SRC_PROVIDER_TESTS}) source_group("Source Files\\qt\\config" FILES ${SRC_QT_CONFIG_TESTS}) source_group("Source Files\\qt\\manager" FILES ${SRC_QT_MANAGER_TESTS}) source_group("Source Files\\util" FILES ${SRC_UTIL_TESTS}) diff --git a/wxdata/include/scwx/provider/aws_level2_data_provider.hpp b/wxdata/include/scwx/provider/aws_level2_data_provider.hpp new file mode 100644 index 00000000..a263ae3b --- /dev/null +++ b/wxdata/include/scwx/provider/aws_level2_data_provider.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include + +#include +#include +#include + +namespace scwx +{ +namespace provider +{ + +class AwsLevel2DataProviderImpl; + +class AwsLevel2DataProvider +{ +public: + explicit AwsLevel2DataProvider(const std::string& radarSite); + explicit AwsLevel2DataProvider(const std::string& radarSite, + const std::string& bucketName, + const std::string& region); + ~AwsLevel2DataProvider(); + + AwsLevel2DataProvider(const AwsLevel2DataProvider&) = delete; + AwsLevel2DataProvider& operator=(const AwsLevel2DataProvider&) = delete; + + AwsLevel2DataProvider(AwsLevel2DataProvider&&) noexcept; + AwsLevel2DataProvider& operator=(AwsLevel2DataProvider&&) noexcept; + + void ListObjects(std::chrono::system_clock::time_point date); + std::shared_ptr LoadObjectByKey(const std::string& key); + void Refresh(); + +private: + std::unique_ptr p; +}; + +} // namespace provider +} // namespace scwx diff --git a/wxdata/source/scwx/provider/aws_level2_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_data_provider.cpp new file mode 100644 index 00000000..8a09c24a --- /dev/null +++ b/wxdata/source/scwx/provider/aws_level2_data_provider.cpp @@ -0,0 +1,134 @@ +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace scwx +{ +namespace provider +{ + +static const std::string logPrefix_ = + "scwx::provider::aws_level2_data_provider"; +static const auto logger_ = scwx::util::Logger::Create(logPrefix_); + +static const std::string kDefaultBucketName_ = "noaa-nexrad-level2"; +static const std::string kDefaultRegion_ = "us-east-1"; + +class AwsLevel2DataProviderImpl +{ +public: + explicit AwsLevel2DataProviderImpl(const std::string& radarSite, + const std::string& bucketName, + const std::string& region) : + radarSite_ {radarSite}, + bucketName_ {bucketName}, + region_ {region}, + client_ {nullptr} + { + Aws::Client::ClientConfiguration config; + config.region = region_; + + client_ = std::make_unique(config); + } + + ~AwsLevel2DataProviderImpl() {} + + std::string radarSite_; + std::string bucketName_; + std::string region_; + + std::unique_ptr client_; +}; + +AwsLevel2DataProvider::AwsLevel2DataProvider(const std::string& radarSite) : + AwsLevel2DataProvider(radarSite, kDefaultBucketName_, kDefaultRegion_) +{ +} +AwsLevel2DataProvider::AwsLevel2DataProvider(const std::string& radarSite, + const std::string& bucketName, + const std::string& region) : + p(std::make_unique( + radarSite, bucketName, region)) +{ +} +AwsLevel2DataProvider::~AwsLevel2DataProvider() = default; + +AwsLevel2DataProvider::AwsLevel2DataProvider(AwsLevel2DataProvider&&) noexcept = + default; +AwsLevel2DataProvider& +AwsLevel2DataProvider::operator=(AwsLevel2DataProvider&&) noexcept = default; + +void AwsLevel2DataProvider::ListObjects( + std::chrono::system_clock::time_point date) +{ + const std::string prefix = + fmt::format("{0:%Y/%m/%d}/{1}/", date, p->radarSite_); + + logger_->debug("ListObjects: {}", prefix); + + Aws::S3::Model::ListObjectsV2Request request; + request.SetBucket(p->bucketName_); + request.SetPrefix(prefix); + + auto outcome = p->client_->ListObjectsV2(request); + + if (outcome.IsSuccess()) + { + auto& objects = outcome.GetResult().GetContents(); + + logger_->debug("Found {} objects", objects.size()); + + // TODO: Store + } + else + { + logger_->warn("Could not list objects: {}", + outcome.GetError().GetMessage()); + } +} + +std::shared_ptr +AwsLevel2DataProvider::LoadObjectByKey(const std::string& key) +{ + std::shared_ptr level2File = nullptr; + + Aws::S3::Model::GetObjectRequest request; + request.SetBucket(p->bucketName_); + request.SetKey(key); + + auto outcome = p->client_->GetObject(request); + + if (outcome.IsSuccess()) + { + auto& body = outcome.GetResultWithOwnership().GetBody(); + + std::shared_ptr nexradFile = + wsr88d::NexradFileFactory::Create(body); + + level2File = std::dynamic_pointer_cast(nexradFile); + } + else + { + logger_->warn("Could not get object: {}", + outcome.GetError().GetMessage()); + } + + return level2File; +} + +void AwsLevel2DataProvider::Refresh() +{ + logger_->debug("Refresh()"); + + // TODO: What if the date just rolled, we might miss from the previous date? + ListObjects(std::chrono::system_clock::now()); +} + +} // namespace provider +} // namespace scwx diff --git a/wxdata/wxdata.cmake b/wxdata/wxdata.cmake index 7c1008ce..7e5feb04 100644 --- a/wxdata/wxdata.cmake +++ b/wxdata/wxdata.cmake @@ -34,6 +34,8 @@ set(SRC_COMMON source/scwx/common/color_table.cpp source/scwx/common/products.cpp source/scwx/common/sites.cpp source/scwx/common/vcp.cpp) +set(HDR_PROVIDER include/scwx/provider/aws_level2_data_provider.hpp) +set(SRC_PROVIDER source/scwx/provider/aws_level2_data_provider.cpp) set(HDR_UTIL include/scwx/util/float.hpp include/scwx/util/iterator.hpp include/scwx/util/logger.hpp @@ -162,6 +164,8 @@ add_library(wxdata OBJECT ${HDR_AWIPS} ${SRC_AWIPS} ${HDR_COMMON} ${SRC_COMMON} + ${HDR_PROVIDER} + ${SRC_PROVIDER} ${HDR_UTIL} ${SRC_UTIL} ${HDR_WSR88D} @@ -176,6 +180,8 @@ source_group("Header Files\\awips" FILES ${HDR_AWIPS}) source_group("Source Files\\awips" FILES ${SRC_AWIPS}) source_group("Header Files\\common" FILES ${HDR_COMMON}) source_group("Source Files\\common" FILES ${SRC_COMMON}) +source_group("Header Files\\provider" FILES ${HDR_PROVIDER}) +source_group("Source Files\\provider" FILES ${SRC_PROVIDER}) source_group("Header Files\\util" FILES ${HDR_UTIL}) source_group("Source Files\\util" FILES ${SRC_UTIL}) source_group("Header Files\\wsr88d" FILES ${HDR_WSR88D})