blakeridgway/supercell-wx
1
0
Fork 0
mirror of https://github.com/ciphervance/supercell-wx.git synced 2025-10-30 13:10:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Optimization of object listing by date

Browse source
This commit is contained in:
Dan Paulat 2023-05-24 22:56:17 -05:00
parent f452d3f15d
commit da835d7226
4 changed files with 29 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

2
wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp
View file

@ -35,7 +35,7 @@ public:
std::string FindLatestKey() override;
std::vector<std::chrono::system_clock::time_point>
GetTimePointsByDate(std::chrono::system_clock::time_point date) override;
std::pair<size_t, size_t>
std::tuple<bool, size_t, size_t>
ListObjects(std::chrono::system_clock::time_point date) override;
std::shared_ptr<wsr88d::NexradFile>
LoadObjectByKey(const std::string& key) override;

5
wxdata/include/scwx/provider/nexrad_data_provider.hpp
View file

@ -64,10 +64,11 @@ public:
*
* @param date Date for which to list objects
*
* @return - New objects found for the given date
* @return - Whether query was successful
* - New objects found for the given date
* - Total objects found for the given date
*/
virtual std::pair<size_t, size_t>
virtual std::tuple<bool, size_t, size_t>
ListObjects(std::chrono::system_clock::time_point date) = 0;
/**

20
wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp
View file

@ -165,7 +165,7 @@ AwsNexradDataProvider::GetTimePointsByDate(
std::vector<std::chrono::system_clock::time_point> timePoints {};
logger_->debug("GetTimePointsByDate: {}", day);
logger_->debug("GetTimePointsByDate: {}", util::TimeString(date));
std::shared_lock lock(p->objectsMutex_);
@ -177,7 +177,11 @@ AwsNexradDataProvider::GetTimePointsByDate(
lock.unlock();
// List objects, since the date is not present in the date list
ListObjects(date);
auto [success, newObjects, totalObjects] = ListObjects(date);
if (success)
{
p->UpdateObjectDates(date);
}
// Re-lock mutex
lock.lock();
@ -196,7 +200,7 @@ AwsNexradDataProvider::GetTimePointsByDate(
return timePoints;
}
std::pair<size_t, size_t>
std::tuple<bool, size_t, size_t>
AwsNexradDataProvider::ListObjects(std::chrono::system_clock::time_point date)
{
const std::string prefix {GetPrefix(date)};
@ -262,7 +266,7 @@ AwsNexradDataProvider::ListObjects(std::chrono::system_clock::time_point date)
outcome.GetError().GetMessage());
}
return std::make_pair(newObjects, totalObjects);
return {outcome.IsSuccess(), newObjects, totalObjects};
}
std::shared_ptr<wsr88d::NexradFile>
@ -309,16 +313,16 @@ std::pair<size_t, size_t> AwsNexradDataProvider::Refresh()
// yesterday, to ensure we haven't missed any objects near midnight
if (p->refreshDate_ < today)
{
auto [newObjects, totalObjects] = ListObjects(yesterday);
allNewObjects = newObjects;
allTotalObjects = totalObjects;
auto [success, newObjects, totalObjects] = ListObjects(yesterday);
allNewObjects = newObjects;
allTotalObjects = totalObjects;
if (totalObjects > 0)
{
p->refreshDate_ = yesterday;
}
}
auto [newObjects, totalObjects] = ListObjects(today);
auto [success, newObjects, totalObjects] = ListObjects(today);
allNewObjects += newObjects;
allTotalObjects += totalObjects;
if (totalObjects > 0)