Implement pruning in the AWS Level 2 data provider cache

test/source/scwx/provider/aws_level2_data_provider.test.cpp

@@ -44,6 +44,38 @@ TEST(AwsLevel2DataProvider, LoadObjectByKey)
    EXPECT_NE(file, nullptr);
 }
 
+TEST(AwsLevel2DataProvider, Prune)
+{
+   using namespace std::chrono;
+
+   AwsLevel2DataProvider provider("KLSX");
+
+   const auto today     = floor<days>(system_clock::now());
+   const auto yesterday = today - days {1};
+   auto       date      = today + days {1};
+
+   for (size_t i = 0; i < 15; i++)
+   {
+      date -= days {1};
+      provider.ListObjects(date);
+
+      // Expect the cache size to be under the prune threshold
+      EXPECT_LE(provider.cache_size(), 2500);
+   }
+
+   std::string key  = provider.FindLatestKey();
+   auto        time = provider.GetTimePointByKey(key);
+
+   // Expect the most recent key to be after midnight yesterday (was not pruned)
+   EXPECT_GT(time, yesterday);
+
+   key  = provider.FindKey(date + 1h);
+   time = provider.GetTimePointByKey(key);
+
+   // Expect the oldest key to be on the most recently listed date
+   EXPECT_LT(time, date + days {1});
+}
+
 TEST(AwsLevel2DataProvider, Refresh)
 {
    AwsLevel2DataProvider provider("KLSX");

wxdata/source/scwx/provider/aws_level2_data_provider.cpp

@@ -24,6 +24,10 @@ static const auto logger_ = util::Logger::Create(logPrefix_);
 static const std::string kDefaultBucketName_ = "noaa-nexrad-level2";
 static const std::string kDefaultRegion_     = "us-east-1";
 
+// Keep at least today, yesterday, and one more date
+static const size_t kMinDatesBeforePruning_ = 4;
+static const size_t kMaxObjects_            = 2500;
+
 class AwsLevel2DataProviderImpl
 {
 public:
@@ -61,7 +65,9 @@ public:
 
    ~AwsLevel2DataProviderImpl() {}
 
+   void PruneObjects();
    void UpdateMetadata();
+   void UpdateObjectDates(std::chrono::system_clock::time_point date);
 
    std::string radarSite_;
    std::string bucketName_;
@@ -71,6 +77,7 @@ public:
 
    std::map<std::chrono::system_clock::time_point, ObjectRecord> objects_;
    std::shared_mutex                                             objectsMutex_;
+   std::list<std::chrono::system_clock::time_point>              objectDates_;
 
    std::chrono::system_clock::time_point lastModified_;
    std::chrono::seconds                  updatePeriod_;
@@ -199,6 +206,13 @@ AwsLevel2DataProvider::ListObjects(std::chrono::system_clock::time_point date)
                totalObjects++;
             }
          });
+
+      if (newObjects > 0)
+      {
+         p->UpdateObjectDates(date);
+         p->PruneObjects();
+         p->UpdateMetadata();
+      }
    }
    else
    {
@@ -206,8 +220,6 @@ AwsLevel2DataProvider::ListObjects(std::chrono::system_clock::time_point date)
                     outcome.GetError().GetMessage());
    }
 
-   p->UpdateMetadata();
-
    return std::make_pair(newObjects, totalObjects);
 }
 
@@ -278,6 +290,36 @@ size_t AwsLevel2DataProvider::Refresh()
    return totalNewObjects;
 }
 
+void AwsLevel2DataProviderImpl::PruneObjects()
+{
+   using namespace std::chrono;
+
+   auto today     = floor<days>(system_clock::now());
+   auto yesterday = today - days {1};
+
+   std::unique_lock lock(objectsMutex_);
+
+   for (auto it = objectDates_.cbegin();
+        it != objectDates_.cend() && objects_.size() > kMaxObjects_ &&
+        objectDates_.size() >= kMinDatesBeforePruning_;)
+   {
+      if (*it < yesterday)
+      {
+         // Erase oldest keys from objects list
+         auto eraseBegin = objects_.lower_bound(*it);
+         auto eraseEnd   = objects_.lower_bound(*it + days {1});
+         objects_.erase(eraseBegin, eraseEnd);
+
+         // Remove oldest date from object dates list
+         it = objectDates_.erase(it);
+      }
+      else
+      {
+         ++it;
+      }
+   }
+}
+
 void AwsLevel2DataProviderImpl::UpdateMetadata()
 {
    std::shared_lock lock(objectsMutex_);
@@ -298,6 +340,18 @@ void AwsLevel2DataProviderImpl::UpdateMetadata()
    }
 }
 
+void AwsLevel2DataProviderImpl::UpdateObjectDates(
+   std::chrono::system_clock::time_point date)
+{
+   auto day = std::chrono::floor<std::chrono::days>(date);
+
+   std::unique_lock lock(objectsMutex_);
+
+   // Remove any existing occurrences of day, and add to the back of the list
+   objectDates_.remove(day);
+   objectDates_.push_back(day);
+}
+
 std::chrono::system_clock::time_point
 AwsLevel2DataProvider::GetTimePointByKey(const std::string& key) const
 {