From df3d65e8aac0debf2bcdbb6756173418df1c82f6 Mon Sep 17 00:00:00 2001
From: Dan Paulat <dpaulat@gmail.com>
Date: Mon, 23 May 2022 23:41:24 -0500
Subject: [PATCH] Returning new objects from Refresh() function

---
 .../aws_level2_data_provider.test.cpp         |  3 +-
 .../provider/aws_level2_data_provider.hpp     |  5 ++--
 .../provider/aws_level2_data_provider.cpp     | 29 +++++++++++++------
 3 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/test/source/scwx/provider/aws_level2_data_provider.test.cpp b/test/source/scwx/provider/aws_level2_data_provider.test.cpp
index 54299585..1c1646ab 100644
--- a/test/source/scwx/provider/aws_level2_data_provider.test.cpp
+++ b/test/source/scwx/provider/aws_level2_data_provider.test.cpp
@@ -48,8 +48,9 @@ TEST(AwsLevel2DataProvider, Refresh)
 {
    AwsLevel2DataProvider provider("KLSX");
 
-   provider.Refresh();
+   size_t newObjects = provider.Refresh();
 
+   EXPECT_GT(newObjects, 0);
    EXPECT_GT(provider.cache_size(), 0);
 }
 
diff --git a/wxdata/include/scwx/provider/aws_level2_data_provider.hpp b/wxdata/include/scwx/provider/aws_level2_data_provider.hpp
index 4ade762b..7553bc66 100644
--- a/wxdata/include/scwx/provider/aws_level2_data_provider.hpp
+++ b/wxdata/include/scwx/provider/aws_level2_data_provider.hpp
@@ -31,9 +31,10 @@ public:
    size_t cache_size() const;
 
    std::string FindKey(std::chrono::system_clock::time_point time);
-   size_t      ListObjects(std::chrono::system_clock::time_point date);
+   std::pair<size_t, size_t>
+   ListObjects(std::chrono::system_clock::time_point date);
    std::shared_ptr<wsr88d::Ar2vFile> LoadObjectByKey(const std::string& key);
-   void                              Refresh();
+   size_t                            Refresh();
 
    static std::chrono::system_clock::time_point
    GetTimePointFromKey(const std::string& key);
diff --git a/wxdata/source/scwx/provider/aws_level2_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_data_provider.cpp
index 9e1718bf..9088dae5 100644
--- a/wxdata/source/scwx/provider/aws_level2_data_provider.cpp
+++ b/wxdata/source/scwx/provider/aws_level2_data_provider.cpp
@@ -98,7 +98,7 @@ AwsLevel2DataProvider::FindKey(std::chrono::system_clock::time_point time)
    return key;
 }
 
-size_t
+std::pair<size_t, size_t>
 AwsLevel2DataProvider::ListObjects(std::chrono::system_clock::time_point date)
 {
    const std::string prefix =
@@ -112,6 +112,7 @@ AwsLevel2DataProvider::ListObjects(std::chrono::system_clock::time_point date)
 
    auto outcome = p->client_->ListObjectsV2(request);
 
+   size_t newObjects   = 0;
    size_t totalObjects = 0;
 
    if (outcome.IsSuccess())
@@ -133,7 +134,13 @@ AwsLevel2DataProvider::ListObjects(std::chrono::system_clock::time_point date)
 
                           std::unique_lock lock(p->objectsMutex_);
 
-                          p->objects_[time] = key;
+                          auto [it, inserted] =
+                             p->objects_.insert_or_assign(time, key);
+
+                          if (inserted)
+                          {
+                             newObjects++;
+                          }
 
                           totalObjects++;
                        }
@@ -145,7 +152,7 @@ AwsLevel2DataProvider::ListObjects(std::chrono::system_clock::time_point date)
                     outcome.GetError().GetMessage());
    }
 
-   return totalObjects;
+   return std::make_pair(newObjects, totalObjects);
 }
 
 std::shared_ptr<wsr88d::Ar2vFile>
@@ -177,7 +184,7 @@ AwsLevel2DataProvider::LoadObjectByKey(const std::string& key)
    return level2File;
 }
 
-void AwsLevel2DataProvider::Refresh()
+size_t AwsLevel2DataProvider::Refresh()
 {
    using namespace std::chrono;
 
@@ -191,24 +198,28 @@ void AwsLevel2DataProvider::Refresh()
 
    std::unique_lock lock(refreshMutex);
 
-   size_t objectCount;
+   size_t totalNewObjects = 0;
 
    // If we haven't gotten any objects from today, first list objects for
    // yesterday, to ensure we haven't missed any objects near midnight
    if (refreshDate < today)
    {
-      objectCount = ListObjects(yesterday);
-      if (objectCount > 0)
+      auto [newObjects, totalObjects] = ListObjects(yesterday);
+      totalNewObjects                 = newObjects;
+      if (totalObjects > 0)
       {
          refreshDate = yesterday;
       }
    }
 
-   objectCount = ListObjects(today);
-   if (objectCount > 0)
+   auto [newObjects, totalObjects] = ListObjects(today);
+   totalNewObjects += newObjects;
+   if (totalObjects > 0)
    {
       refreshDate = today;
    }
+
+   return totalNewObjects;
 }
 
 std::chrono::system_clock::time_point