blakeridgway/supercell-wx
1
0
Fork 0
mirror of https://github.com/ciphervance/supercell-wx.git synced 2025-10-30 19:50:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Store objects and get time point

Browse source
This commit is contained in:
Dan Paulat 2022-04-30 21:07:44 -05:00
parent 1681b6772b
commit 690f3f6216
3 changed files with 93 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

60
wxdata/source/scwx/provider/aws_level2_data_provider.cpp
View file

@ -2,6 +2,8 @@
#include <scwx/util/logger.hpp>
#include <scwx/wsr88d/nexrad_file_factory.hpp>
#include <shared_mutex>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/GetObjectRequest.h>
#include <aws/s3/model/ListObjectsV2Request.h>
@ -15,7 +17,7 @@ namespace provider
static const std::string logPrefix_ =
"scwx::provider::aws_level2_data_provider";
static const auto logger_ = scwx::util::Logger::Create(logPrefix_);
static const auto logger_ = util::Logger::Create(logPrefix_);
static const std::string kDefaultBucketName_ = "noaa-nexrad-level2";
static const std::string kDefaultRegion_ = "us-east-1";
@ -29,7 +31,9 @@ public:
radarSite_ {radarSite},
bucketName_ {bucketName},
region_ {region},
client_ {nullptr}
client_ {nullptr},
objects_ {},
objectsMutex_ {}
{
Aws::Client::ClientConfiguration config;
config.region = region_;
@ -44,6 +48,9 @@ public:
std::string region_;
std::unique_ptr<Aws::S3::S3Client> client_;
std::map<std::chrono::system_clock::time_point, std::string> objects_;
std::shared_mutex objectsMutex_;
};
AwsLevel2DataProvider::AwsLevel2DataProvider(const std::string& radarSite) :
@ -84,7 +91,19 @@ void AwsLevel2DataProvider::ListObjects(
logger_->debug("Found {} objects", objects.size());
// TODO: Store
// Store objects
std::for_each(objects.cbegin(),
objects.cend(),
[&](const Aws::S3::Model::Object& object)
{
// TODO: Skip MDM
std::string key = object.GetKey();
auto time = GetTimePointFromKey(key);
std::unique_lock lock(p->objectsMutex_);
p->objects_[time] = key;
});
}
else
{
@ -130,5 +149,40 @@ void AwsLevel2DataProvider::Refresh()
ListObjects(std::chrono::system_clock::now());
}
std::chrono::system_clock::time_point
AwsLevel2DataProvider::GetTimePointFromKey(const std::string& key)
{
std::chrono::system_clock::time_point time {};
const size_t lastSeparator = key.rfind('/');
const size_t offset =
(lastSeparator == std::string::npos) ? 0 : lastSeparator + 5;
// Filename format is GGGGYYYYMMDD_TTTTTT(_V##).gz
static constexpr size_t formatSize = std::string("YYYYMMDD_TTTTTT").size();
if (key.size() >= offset + formatSize)
{
using namespace std::chrono;
static const std::string timeFormat {"%Y%m%d_%H%M%S"};
std::string timeStr {key.substr(offset, formatSize)};
std::istringstream in {timeStr};
in >> parse(timeFormat, time);
if (in.fail())
{
logger_->warn("Invalid time: \"{}\"", timeStr);
}
}
else
{
logger_->warn("Time not parsable from key: \"{}\"", key);
}
return time;
}
} // namespace provider
} // namespace scwx