blakeridgway/supercell-wx
1
0
Fork 0
mirror of https://github.com/ciphervance/supercell-wx.git synced 2025-10-30 16:00:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add initial AWS Level 2 Data Provider

Browse source
This commit is contained in:
Dan Paulat 2022-04-22 09:26:45 -05:00
parent bd68f1adbd
commit 1681b6772b
5 changed files with 214 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

40
wxdata/include/scwx/provider/aws_level2_data_provider.hpp Normal file
View file

@ -0,0 +1,40 @@
#pragma once
#include <scwx/wsr88d/ar2v_file.hpp>
#include <chrono>
#include <memory>
#include <string>
namespace scwx
{
namespace provider
{
class AwsLevel2DataProviderImpl;
class AwsLevel2DataProvider
{
public:
explicit AwsLevel2DataProvider(const std::string& radarSite);
explicit AwsLevel2DataProvider(const std::string& radarSite,
const std::string& bucketName,
const std::string& region);
~AwsLevel2DataProvider();
AwsLevel2DataProvider(const AwsLevel2DataProvider&) = delete;
AwsLevel2DataProvider& operator=(const AwsLevel2DataProvider&) = delete;
AwsLevel2DataProvider(AwsLevel2DataProvider&&) noexcept;
AwsLevel2DataProvider& operator=(AwsLevel2DataProvider&&) noexcept;
void ListObjects(std::chrono::system_clock::time_point date);
std::shared_ptr<wsr88d::Ar2vFile> LoadObjectByKey(const std::string& key);
void Refresh();
private:
std::unique_ptr<AwsLevel2DataProviderImpl> p;
};
} // namespace provider
} // namespace scwx

134
wxdata/source/scwx/provider/aws_level2_data_provider.cpp Normal file
View file

@ -0,0 +1,134 @@
#include <scwx/provider/aws_level2_data_provider.hpp>
#include <scwx/util/logger.hpp>
#include <scwx/wsr88d/nexrad_file_factory.hpp>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/GetObjectRequest.h>
#include <aws/s3/model/ListObjectsV2Request.h>
#include <fmt/chrono.h>
#include <fmt/format.h>
namespace scwx
{
namespace provider
{
static const std::string logPrefix_ =
"scwx::provider::aws_level2_data_provider";
static const auto logger_ = scwx::util::Logger::Create(logPrefix_);
static const std::string kDefaultBucketName_ = "noaa-nexrad-level2";
static const std::string kDefaultRegion_ = "us-east-1";
class AwsLevel2DataProviderImpl
{
public:
explicit AwsLevel2DataProviderImpl(const std::string& radarSite,
const std::string& bucketName,
const std::string& region) :
radarSite_ {radarSite},
bucketName_ {bucketName},
region_ {region},
client_ {nullptr}
{
Aws::Client::ClientConfiguration config;
config.region = region_;
client_ = std::make_unique<Aws::S3::S3Client>(config);
}
~AwsLevel2DataProviderImpl() {}
std::string radarSite_;
std::string bucketName_;
std::string region_;
std::unique_ptr<Aws::S3::S3Client> client_;
};
AwsLevel2DataProvider::AwsLevel2DataProvider(const std::string& radarSite) :
AwsLevel2DataProvider(radarSite, kDefaultBucketName_, kDefaultRegion_)
{
}
AwsLevel2DataProvider::AwsLevel2DataProvider(const std::string& radarSite,
const std::string& bucketName,
const std::string& region) :
p(std::make_unique<AwsLevel2DataProviderImpl>(
radarSite, bucketName, region))
{
}
AwsLevel2DataProvider::~AwsLevel2DataProvider() = default;
AwsLevel2DataProvider::AwsLevel2DataProvider(AwsLevel2DataProvider&&) noexcept =
default;
AwsLevel2DataProvider&
AwsLevel2DataProvider::operator=(AwsLevel2DataProvider&&) noexcept = default;
void AwsLevel2DataProvider::ListObjects(
std::chrono::system_clock::time_point date)
{
const std::string prefix =
fmt::format("{0:%Y/%m/%d}/{1}/", date, p->radarSite_);
logger_->debug("ListObjects: {}", prefix);
Aws::S3::Model::ListObjectsV2Request request;
request.SetBucket(p->bucketName_);
request.SetPrefix(prefix);
auto outcome = p->client_->ListObjectsV2(request);
if (outcome.IsSuccess())
{
auto& objects = outcome.GetResult().GetContents();
logger_->debug("Found {} objects", objects.size());
// TODO: Store
}
else
{
logger_->warn("Could not list objects: {}",
outcome.GetError().GetMessage());
}
}
std::shared_ptr<wsr88d::Ar2vFile>
AwsLevel2DataProvider::LoadObjectByKey(const std::string& key)
{
std::shared_ptr<wsr88d::Ar2vFile> level2File = nullptr;
Aws::S3::Model::GetObjectRequest request;
request.SetBucket(p->bucketName_);
request.SetKey(key);
auto outcome = p->client_->GetObject(request);
if (outcome.IsSuccess())
{
auto& body = outcome.GetResultWithOwnership().GetBody();
std::shared_ptr<wsr88d::NexradFile> nexradFile =
wsr88d::NexradFileFactory::Create(body);
level2File = std::dynamic_pointer_cast<wsr88d::Ar2vFile>(nexradFile);
}
else
{
logger_->warn("Could not get object: {}",
outcome.GetError().GetMessage());
}
return level2File;
}
void AwsLevel2DataProvider::Refresh()
{
logger_->debug("Refresh()");
// TODO: What if the date just rolled, we might miss from the previous date?
ListObjects(std::chrono::system_clock::now());
}
} // namespace provider
} // namespace scwx

6
wxdata/wxdata.cmake
View file

@ -34,6 +34,8 @@ set(SRC_COMMON source/scwx/common/color_table.cpp
source/scwx/common/products.cpp
source/scwx/common/sites.cpp
source/scwx/common/vcp.cpp)
set(HDR_PROVIDER include/scwx/provider/aws_level2_data_provider.hpp)
set(SRC_PROVIDER source/scwx/provider/aws_level2_data_provider.cpp)
set(HDR_UTIL include/scwx/util/float.hpp
include/scwx/util/iterator.hpp
include/scwx/util/logger.hpp
@ -162,6 +164,8 @@ add_library(wxdata OBJECT ${HDR_AWIPS}
${SRC_AWIPS}
${HDR_COMMON}
${SRC_COMMON}
${HDR_PROVIDER}
${SRC_PROVIDER}
${HDR_UTIL}
${SRC_UTIL}
${HDR_WSR88D}
@ -176,6 +180,8 @@ source_group("Header Files\\awips" FILES ${HDR_AWIPS})
source_group("Source Files\\awips" FILES ${SRC_AWIPS})
source_group("Header Files\\common" FILES ${HDR_COMMON})
source_group("Source Files\\common" FILES ${SRC_COMMON})
source_group("Header Files\\provider" FILES ${HDR_PROVIDER})
source_group("Source Files\\provider" FILES ${SRC_PROVIDER})
source_group("Header Files\\util" FILES ${HDR_UTIL})
source_group("Source Files\\util" FILES ${SRC_UTIL})
source_group("Header Files\\wsr88d" FILES ${HDR_WSR88D})