blakeridgway/supercell-wx
1
0
Fork 0
mirror of https://github.com/ciphervance/supercell-wx.git synced 2025-10-31 03:20:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Refresh AWS level 2 data over multiple days

Browse source
This commit is contained in:
Dan Paulat 2022-05-22 07:59:25 -05:00
parent 690f3f6216
commit 80310029e5
3 changed files with 54 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

60
wxdata/source/scwx/provider/aws_level2_data_provider.cpp
View file

@ -71,11 +71,16 @@ AwsLevel2DataProvider::AwsLevel2DataProvider(AwsLevel2DataProvider&&) noexcept =
AwsLevel2DataProvider&
AwsLevel2DataProvider::operator=(AwsLevel2DataProvider&&) noexcept = default;
void AwsLevel2DataProvider::ListObjects(
std::chrono::system_clock::time_point date)
size_t AwsLevel2DataProvider::cache_size() const
{
return p->objects_.size();
}
size_t
AwsLevel2DataProvider::ListObjects(std::chrono::system_clock::time_point date)
{
const std::string prefix =
fmt::format("{0:%Y/%m/%d}/{1}/", date, p->radarSite_);
fmt::format("{0:%Y/%m/%d}/{1}/", fmt::gmtime(date), p->radarSite_);
logger_->debug("ListObjects: {}", prefix);
@ -85,6 +90,8 @@ void AwsLevel2DataProvider::ListObjects(
auto outcome = p->client_->ListObjectsV2(request);
size_t totalObjects = 0;
if (outcome.IsSuccess())
{
auto& objects = outcome.GetResult().GetContents();
@ -96,13 +103,18 @@ void AwsLevel2DataProvider::ListObjects(
objects.cend(),
[&](const Aws::S3::Model::Object& object)
{
// TODO: Skip MDM
std::string key = object.GetKey();
auto time = GetTimePointFromKey(key);
std::string key = object.GetKey();
std::unique_lock lock(p->objectsMutex_);
if (!key.ends_with("_MDM"))
{
auto time = GetTimePointFromKey(key);
p->objects_[time] = key;
std::unique_lock lock(p->objectsMutex_);
p->objects_[time] = key;
totalObjects++;
}
});
}
else
@ -110,6 +122,8 @@ void AwsLevel2DataProvider::ListObjects(
logger_->warn("Could not list objects: {}",
outcome.GetError().GetMessage());
}
return totalObjects;
}
std::shared_ptr<wsr88d::Ar2vFile>
@ -143,10 +157,36 @@ AwsLevel2DataProvider::LoadObjectByKey(const std::string& key)
void AwsLevel2DataProvider::Refresh()
{
using namespace std::chrono;
logger_->debug("Refresh()");
// TODO: What if the date just rolled, we might miss from the previous date?
ListObjects(std::chrono::system_clock::now());
static std::mutex refreshMutex;
static system_clock::time_point refreshDate {};
auto today = floor<days>(system_clock::now());
auto yesterday = today - days {1};
std::unique_lock lock(refreshMutex);
size_t objectCount;
// If we haven't gotten any objects from today, first list objects for
// yesterday, to ensure we haven't missed any objects near midnight
if (refreshDate < today)
{
objectCount = ListObjects(yesterday);
if (objectCount > 0)
{
refreshDate = yesterday;
}
}
objectCount = ListObjects(today);
if (objectCount > 0)
{
refreshDate = today;
}
}
std::chrono::system_clock::time_point