diff --git a/test/source/scwx/provider/aws_level2_data_provider.test.cpp b/test/source/scwx/provider/aws_level2_data_provider.test.cpp index 1c1646ab..69b6f4dd 100644 --- a/test/source/scwx/provider/aws_level2_data_provider.test.cpp +++ b/test/source/scwx/provider/aws_level2_data_provider.test.cpp @@ -44,6 +44,38 @@ TEST(AwsLevel2DataProvider, LoadObjectByKey) EXPECT_NE(file, nullptr); } +TEST(AwsLevel2DataProvider, Prune) +{ + using namespace std::chrono; + + AwsLevel2DataProvider provider("KLSX"); + + const auto today = floor(system_clock::now()); + const auto yesterday = today - days {1}; + auto date = today + days {1}; + + for (size_t i = 0; i < 15; i++) + { + date -= days {1}; + provider.ListObjects(date); + + // Expect the cache size to be under the prune threshold + EXPECT_LE(provider.cache_size(), 2500); + } + + std::string key = provider.FindLatestKey(); + auto time = provider.GetTimePointByKey(key); + + // Expect the most recent key to be after midnight yesterday (was not pruned) + EXPECT_GT(time, yesterday); + + key = provider.FindKey(date + 1h); + time = provider.GetTimePointByKey(key); + + // Expect the oldest key to be on the most recently listed date + EXPECT_LT(time, date + days {1}); +} + TEST(AwsLevel2DataProvider, Refresh) { AwsLevel2DataProvider provider("KLSX"); diff --git a/wxdata/source/scwx/provider/aws_level2_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_data_provider.cpp index aac6da3f..6120e419 100644 --- a/wxdata/source/scwx/provider/aws_level2_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_data_provider.cpp @@ -24,6 +24,10 @@ static const auto logger_ = util::Logger::Create(logPrefix_); static const std::string kDefaultBucketName_ = "noaa-nexrad-level2"; static const std::string kDefaultRegion_ = "us-east-1"; +// Keep at least today, yesterday, and one more date +static const size_t kMinDatesBeforePruning_ = 4; +static const size_t kMaxObjects_ = 2500; + class AwsLevel2DataProviderImpl { public: @@ -61,7 +65,9 @@ public: ~AwsLevel2DataProviderImpl() {} + void PruneObjects(); void UpdateMetadata(); + void UpdateObjectDates(std::chrono::system_clock::time_point date); std::string radarSite_; std::string bucketName_; @@ -71,6 +77,7 @@ public: std::map objects_; std::shared_mutex objectsMutex_; + std::list objectDates_; std::chrono::system_clock::time_point lastModified_; std::chrono::seconds updatePeriod_; @@ -199,6 +206,13 @@ AwsLevel2DataProvider::ListObjects(std::chrono::system_clock::time_point date) totalObjects++; } }); + + if (newObjects > 0) + { + p->UpdateObjectDates(date); + p->PruneObjects(); + p->UpdateMetadata(); + } } else { @@ -206,8 +220,6 @@ AwsLevel2DataProvider::ListObjects(std::chrono::system_clock::time_point date) outcome.GetError().GetMessage()); } - p->UpdateMetadata(); - return std::make_pair(newObjects, totalObjects); } @@ -278,6 +290,36 @@ size_t AwsLevel2DataProvider::Refresh() return totalNewObjects; } +void AwsLevel2DataProviderImpl::PruneObjects() +{ + using namespace std::chrono; + + auto today = floor(system_clock::now()); + auto yesterday = today - days {1}; + + std::unique_lock lock(objectsMutex_); + + for (auto it = objectDates_.cbegin(); + it != objectDates_.cend() && objects_.size() > kMaxObjects_ && + objectDates_.size() >= kMinDatesBeforePruning_;) + { + if (*it < yesterday) + { + // Erase oldest keys from objects list + auto eraseBegin = objects_.lower_bound(*it); + auto eraseEnd = objects_.lower_bound(*it + days {1}); + objects_.erase(eraseBegin, eraseEnd); + + // Remove oldest date from object dates list + it = objectDates_.erase(it); + } + else + { + ++it; + } + } +} + void AwsLevel2DataProviderImpl::UpdateMetadata() { std::shared_lock lock(objectsMutex_); @@ -298,6 +340,18 @@ void AwsLevel2DataProviderImpl::UpdateMetadata() } } +void AwsLevel2DataProviderImpl::UpdateObjectDates( + std::chrono::system_clock::time_point date) +{ + auto day = std::chrono::floor(date); + + std::unique_lock lock(objectsMutex_); + + // Remove any existing occurrences of day, and add to the back of the list + objectDates_.remove(day); + objectDates_.push_back(day); +} + std::chrono::system_clock::time_point AwsLevel2DataProvider::GetTimePointByKey(const std::string& key) const {