blakeridgway/supercell-wx
1
0
Fork 0
mirror of https://github.com/ciphervance/supercell-wx.git synced 2025-10-30 05:50:06 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add initial AWS Level 2 Data Provider

Browse source
This commit is contained in:
Dan Paulat 2022-04-22 09:26:45 -05:00
parent bd68f1adbd
commit 1681b6772b
5 changed files with 214 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

31
test/source/scwx/provider/aws_level2_data_provider.test.cpp Normal file
View file

@ -0,0 +1,31 @@
#include <scwx/provider/aws_level2_data_provider.hpp>
#include <gtest/gtest.h>
namespace scwx
{
namespace provider
{
TEST(AwsLevel2DataProvider, LoadObjectByKey)
{
const std::string key = "2022/04/21/KLSX/KLSX20220421_160055_V06";
AwsLevel2DataProvider provider("KLSX");
auto file = provider.LoadObjectByKey(key);
EXPECT_NE(file, nullptr);
}
TEST(AwsLevel2DataProvider, Refresh)
{
AwsLevel2DataProvider provider("KLSX");
provider.Refresh();
// TODO: Check object count
}
} // namespace provider
} // namespace scwx

3
test/test.cmake
View file

@ -13,6 +13,7 @@ set(SRC_AWIPS_TESTS source/scwx/awips/coded_location.test.cpp
source/scwx/awips/pvtec.test.cpp
source/scwx/awips/text_product_file.test.cpp)
set(SRC_COMMON_TESTS source/scwx/common/color_table.test.cpp)
set(SRC_PROVIDER_TESTS source/scwx/provider/aws_level2_data_provider.test.cpp)
set(SRC_QT_CONFIG_TESTS source/scwx/qt/config/radar_site.test.cpp)
set(SRC_QT_MANAGER_TESTS source/scwx/qt/manager/settings_manager.test.cpp)
set(SRC_UTIL_TESTS source/scwx/util/float.test.cpp
@ -28,6 +29,7 @@ set(CMAKE_FILES test.cmake)
add_executable(wxtest ${SRC_MAIN}
${SRC_AWIPS_TESTS}
${SRC_COMMON_TESTS}
${SRC_PROVIDER_TESTS}
${SRC_QT_CONFIG_TESTS}
${SRC_QT_MANAGER_TESTS}
${SRC_UTIL_TESTS}
@ -37,6 +39,7 @@ add_executable(wxtest ${SRC_MAIN}
source_group("Source Files\\main" FILES ${SRC_MAIN})
source_group("Source Files\\awips" FILES ${SRC_AWIPS_TESTS})
source_group("Source Files\\common" FILES ${SRC_COMMON_TESTS})
source_group("Source Files\\provider" FILES ${SRC_PROVIDER_TESTS})
source_group("Source Files\\qt\\config" FILES ${SRC_QT_CONFIG_TESTS})
source_group("Source Files\\qt\\manager" FILES ${SRC_QT_MANAGER_TESTS})
source_group("Source Files\\util" FILES ${SRC_UTIL_TESTS})

40
wxdata/include/scwx/provider/aws_level2_data_provider.hpp Normal file
View file

@ -0,0 +1,40 @@
#pragma once
#include <scwx/wsr88d/ar2v_file.hpp>
#include <chrono>
#include <memory>
#include <string>
namespace scwx
{
namespace provider
{
class AwsLevel2DataProviderImpl;
class AwsLevel2DataProvider
{
public:
explicit AwsLevel2DataProvider(const std::string& radarSite);
explicit AwsLevel2DataProvider(const std::string& radarSite,
const std::string& bucketName,
const std::string& region);
~AwsLevel2DataProvider();
AwsLevel2DataProvider(const AwsLevel2DataProvider&) = delete;
AwsLevel2DataProvider& operator=(const AwsLevel2DataProvider&) = delete;
AwsLevel2DataProvider(AwsLevel2DataProvider&&) noexcept;
AwsLevel2DataProvider& operator=(AwsLevel2DataProvider&&) noexcept;
void ListObjects(std::chrono::system_clock::time_point date);
std::shared_ptr<wsr88d::Ar2vFile> LoadObjectByKey(const std::string& key);
void Refresh();
private:
std::unique_ptr<AwsLevel2DataProviderImpl> p;
};
} // namespace provider
} // namespace scwx

134
wxdata/source/scwx/provider/aws_level2_data_provider.cpp Normal file
View file

@ -0,0 +1,134 @@
#include <scwx/provider/aws_level2_data_provider.hpp>
#include <scwx/util/logger.hpp>
#include <scwx/wsr88d/nexrad_file_factory.hpp>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/GetObjectRequest.h>
#include <aws/s3/model/ListObjectsV2Request.h>
#include <fmt/chrono.h>
#include <fmt/format.h>
namespace scwx
{
namespace provider
{
static const std::string logPrefix_ =
"scwx::provider::aws_level2_data_provider";
static const auto logger_ = scwx::util::Logger::Create(logPrefix_);
static const std::string kDefaultBucketName_ = "noaa-nexrad-level2";
static const std::string kDefaultRegion_ = "us-east-1";
class AwsLevel2DataProviderImpl
{
public:
explicit AwsLevel2DataProviderImpl(const std::string& radarSite,
const std::string& bucketName,
const std::string& region) :
radarSite_ {radarSite},
bucketName_ {bucketName},
region_ {region},
client_ {nullptr}
{
Aws::Client::ClientConfiguration config;
config.region = region_;
client_ = std::make_unique<Aws::S3::S3Client>(config);
}
~AwsLevel2DataProviderImpl() {}
std::string radarSite_;
std::string bucketName_;
std::string region_;
std::unique_ptr<Aws::S3::S3Client> client_;
};
AwsLevel2DataProvider::AwsLevel2DataProvider(const std::string& radarSite) :
AwsLevel2DataProvider(radarSite, kDefaultBucketName_, kDefaultRegion_)
{
}
AwsLevel2DataProvider::AwsLevel2DataProvider(const std::string& radarSite,
const std::string& bucketName,
const std::string& region) :
p(std::make_unique<AwsLevel2DataProviderImpl>(
radarSite, bucketName, region))
{
}
AwsLevel2DataProvider::~AwsLevel2DataProvider() = default;
AwsLevel2DataProvider::AwsLevel2DataProvider(AwsLevel2DataProvider&&) noexcept =
default;
AwsLevel2DataProvider&
AwsLevel2DataProvider::operator=(AwsLevel2DataProvider&&) noexcept = default;
void AwsLevel2DataProvider::ListObjects(
std::chrono::system_clock::time_point date)
{
const std::string prefix =
fmt::format("{0:%Y/%m/%d}/{1}/", date, p->radarSite_);
logger_->debug("ListObjects: {}", prefix);
Aws::S3::Model::ListObjectsV2Request request;
request.SetBucket(p->bucketName_);
request.SetPrefix(prefix);
auto outcome = p->client_->ListObjectsV2(request);
if (outcome.IsSuccess())
{
auto& objects = outcome.GetResult().GetContents();
logger_->debug("Found {} objects", objects.size());
// TODO: Store
}
else
{
logger_->warn("Could not list objects: {}",
outcome.GetError().GetMessage());
}
}
std::shared_ptr<wsr88d::Ar2vFile>
AwsLevel2DataProvider::LoadObjectByKey(const std::string& key)
{
std::shared_ptr<wsr88d::Ar2vFile> level2File = nullptr;
Aws::S3::Model::GetObjectRequest request;
request.SetBucket(p->bucketName_);
request.SetKey(key);
auto outcome = p->client_->GetObject(request);
if (outcome.IsSuccess())
{
auto& body = outcome.GetResultWithOwnership().GetBody();
std::shared_ptr<wsr88d::NexradFile> nexradFile =
wsr88d::NexradFileFactory::Create(body);
level2File = std::dynamic_pointer_cast<wsr88d::Ar2vFile>(nexradFile);
}
else
{
logger_->warn("Could not get object: {}",
outcome.GetError().GetMessage());
}
return level2File;
}
void AwsLevel2DataProvider::Refresh()
{
logger_->debug("Refresh()");
// TODO: What if the date just rolled, we might miss from the previous date?
ListObjects(std::chrono::system_clock::now());
}
} // namespace provider
} // namespace scwx

6
wxdata/wxdata.cmake
View file

@ -34,6 +34,8 @@ set(SRC_COMMON source/scwx/common/color_table.cpp
source/scwx/common/products.cpp
source/scwx/common/sites.cpp
source/scwx/common/vcp.cpp)
set(HDR_PROVIDER include/scwx/provider/aws_level2_data_provider.hpp)
set(SRC_PROVIDER source/scwx/provider/aws_level2_data_provider.cpp)
set(HDR_UTIL include/scwx/util/float.hpp
include/scwx/util/iterator.hpp
include/scwx/util/logger.hpp
@ -162,6 +164,8 @@ add_library(wxdata OBJECT ${HDR_AWIPS}
${SRC_AWIPS}
${HDR_COMMON}
${SRC_COMMON}
${HDR_PROVIDER}
${SRC_PROVIDER}
${HDR_UTIL}
${SRC_UTIL}
${HDR_WSR88D}
@ -176,6 +180,8 @@ source_group("Header Files\\awips" FILES ${HDR_AWIPS})
source_group("Source Files\\awips" FILES ${SRC_AWIPS})
source_group("Header Files\\common" FILES ${HDR_COMMON})
source_group("Source Files\\common" FILES ${SRC_COMMON})
source_group("Header Files\\provider" FILES ${HDR_PROVIDER})
source_group("Source Files\\provider" FILES ${SRC_PROVIDER})
source_group("Header Files\\util" FILES ${HDR_UTIL})
source_group("Source Files\\util" FILES ${SRC_UTIL})
source_group("Header Files\\wsr88d" FILES ${HDR_WSR88D})