blakeridgway/supercell-wx
1
0
Fork 0
mirror of https://github.com/ciphervance/supercell-wx.git synced 2025-10-31 07:00:06 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Optimization of object listing by date

Browse source
This commit is contained in:
Dan Paulat 2023-05-24 22:56:17 -05:00
parent f452d3f15d
commit da835d7226
4 changed files with 29 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

14
scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp
View file

@ -683,9 +683,15 @@ RadarProductManager::GetActiveVolumeTimes(
{ {
std::unordered_set<std::shared_ptr<provider::NexradDataProvider>> std::unordered_set<std::shared_ptr<provider::NexradDataProvider>>
providers {}; providers {};
std::set<std::chrono::system_clock::time_point> volumeTimes; std::set<std::chrono::system_clock::time_point> volumeTimes {};
std::mutex volumeTimesMutex {}; std::mutex volumeTimesMutex {};
// Return a default set of volume times if the default time point is given
if (time == std::chrono::system_clock::time_point {})
{
return volumeTimes;
}
// Lock the refresh map // Lock the refresh map
std::shared_lock refreshLock {p->refreshMapMutex_}; std::shared_lock refreshLock {p->refreshMapMutex_};
@ -717,6 +723,12 @@ RadarProductManager::GetActiveVolumeTimes(
dates.end(), dates.end(),
[&](const auto& date) [&](const auto& date)
{ {
// Don't query for a time point in the future
if (date > std::chrono::system_clock::now())
{
return;
}
// Query the provider for volume time points // Query the provider for volume time points
auto timePoints = provider->GetTimePointsByDate(date); auto timePoints = provider->GetTimePointsByDate(date);

2
wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp
View file

@ -35,7 +35,7 @@ public:
std::string FindLatestKey() override; std::string FindLatestKey() override;
std::vector<std::chrono::system_clock::time_point> std::vector<std::chrono::system_clock::time_point>
GetTimePointsByDate(std::chrono::system_clock::time_point date) override; GetTimePointsByDate(std::chrono::system_clock::time_point date) override;
std::pair<size_t, size_t> std::tuple<bool, size_t, size_t>
ListObjects(std::chrono::system_clock::time_point date) override; ListObjects(std::chrono::system_clock::time_point date) override;
std::shared_ptr<wsr88d::NexradFile> std::shared_ptr<wsr88d::NexradFile>
LoadObjectByKey(const std::string& key) override; LoadObjectByKey(const std::string& key) override;

5
wxdata/include/scwx/provider/nexrad_data_provider.hpp
View file

@ -64,10 +64,11 @@ public:
* *
* @param date Date for which to list objects * @param date Date for which to list objects
* *
* @return - New objects found for the given date * @return - Whether query was successful
* - New objects found for the given date
* - Total objects found for the given date * - Total objects found for the given date
*/ */
virtual std::pair<size_t, size_t> virtual std::tuple<bool, size_t, size_t>
ListObjects(std::chrono::system_clock::time_point date) = 0; ListObjects(std::chrono::system_clock::time_point date) = 0;
/** /**

16
wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp
View file

@ -165,7 +165,7 @@ AwsNexradDataProvider::GetTimePointsByDate(
std::vector<std::chrono::system_clock::time_point> timePoints {}; std::vector<std::chrono::system_clock::time_point> timePoints {};
logger_->debug("GetTimePointsByDate: {}", day); logger_->debug("GetTimePointsByDate: {}", util::TimeString(date));
std::shared_lock lock(p->objectsMutex_); std::shared_lock lock(p->objectsMutex_);
@ -177,7 +177,11 @@ AwsNexradDataProvider::GetTimePointsByDate(
lock.unlock(); lock.unlock();
// List objects, since the date is not present in the date list // List objects, since the date is not present in the date list
ListObjects(date); auto [success, newObjects, totalObjects] = ListObjects(date);
if (success)
{
p->UpdateObjectDates(date);
}
// Re-lock mutex // Re-lock mutex
lock.lock(); lock.lock();
@ -196,7 +200,7 @@ AwsNexradDataProvider::GetTimePointsByDate(
return timePoints; return timePoints;
} }
std::pair<size_t, size_t> std::tuple<bool, size_t, size_t>
AwsNexradDataProvider::ListObjects(std::chrono::system_clock::time_point date) AwsNexradDataProvider::ListObjects(std::chrono::system_clock::time_point date)
{ {
const std::string prefix {GetPrefix(date)}; const std::string prefix {GetPrefix(date)};
@ -262,7 +266,7 @@ AwsNexradDataProvider::ListObjects(std::chrono::system_clock::time_point date)
outcome.GetError().GetMessage()); outcome.GetError().GetMessage());
} }
return std::make_pair(newObjects, totalObjects); return {outcome.IsSuccess(), newObjects, totalObjects};
} }
std::shared_ptr<wsr88d::NexradFile> std::shared_ptr<wsr88d::NexradFile>
@ -309,7 +313,7 @@ std::pair<size_t, size_t> AwsNexradDataProvider::Refresh()
// yesterday, to ensure we haven't missed any objects near midnight // yesterday, to ensure we haven't missed any objects near midnight
if (p->refreshDate_ < today) if (p->refreshDate_ < today)
{ {
auto [newObjects, totalObjects] = ListObjects(yesterday); auto [success, newObjects, totalObjects] = ListObjects(yesterday);
allNewObjects = newObjects; allNewObjects = newObjects;
allTotalObjects = totalObjects; allTotalObjects = totalObjects;
if (totalObjects > 0) if (totalObjects > 0)
@ -318,7 +322,7 @@ std::pair<size_t, size_t> AwsNexradDataProvider::Refresh()
} }
} }
auto [newObjects, totalObjects] = ListObjects(today); auto [success, newObjects, totalObjects] = ListObjects(today);
allNewObjects += newObjects; allNewObjects += newObjects;
allTotalObjects += totalObjects; allTotalObjects += totalObjects;
if (totalObjects > 0) if (totalObjects > 0)