blakeridgway/supercell-wx
1
0
Fork 0
mirror of https://github.com/ciphervance/supercell-wx.git synced 2025-10-31 17:30:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Returning new objects from Refresh() function

Browse source
This commit is contained in:
Dan Paulat 2022-05-23 23:41:24 -05:00
parent b7aeecfe83
commit df3d65e8aa
3 changed files with 25 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

3
test/source/scwx/provider/aws_level2_data_provider.test.cpp
View file

@ -48,8 +48,9 @@ TEST(AwsLevel2DataProvider, Refresh)
{ {
AwsLevel2DataProvider provider("KLSX"); AwsLevel2DataProvider provider("KLSX");
provider.Refresh(); size_t newObjects = provider.Refresh();
EXPECT_GT(newObjects, 0);
EXPECT_GT(provider.cache_size(), 0); EXPECT_GT(provider.cache_size(), 0);
} }

5
wxdata/include/scwx/provider/aws_level2_data_provider.hpp
View file

@ -31,9 +31,10 @@ public:
size_t cache_size() const; size_t cache_size() const;
std::string FindKey(std::chrono::system_clock::time_point time); std::string FindKey(std::chrono::system_clock::time_point time);
size_t ListObjects(std::chrono::system_clock::time_point date); std::pair<size_t, size_t>
ListObjects(std::chrono::system_clock::time_point date);
std::shared_ptr<wsr88d::Ar2vFile> LoadObjectByKey(const std::string& key); std::shared_ptr<wsr88d::Ar2vFile> LoadObjectByKey(const std::string& key);
void Refresh(); size_t Refresh();
static std::chrono::system_clock::time_point static std::chrono::system_clock::time_point
GetTimePointFromKey(const std::string& key); GetTimePointFromKey(const std::string& key);

29
wxdata/source/scwx/provider/aws_level2_data_provider.cpp
View file

@ -98,7 +98,7 @@ AwsLevel2DataProvider::FindKey(std::chrono::system_clock::time_point time)
return key; return key;
} }
size_t std::pair<size_t, size_t>
AwsLevel2DataProvider::ListObjects(std::chrono::system_clock::time_point date) AwsLevel2DataProvider::ListObjects(std::chrono::system_clock::time_point date)
{ {
const std::string prefix = const std::string prefix =
@ -112,6 +112,7 @@ AwsLevel2DataProvider::ListObjects(std::chrono::system_clock::time_point date)
auto outcome = p->client_->ListObjectsV2(request); auto outcome = p->client_->ListObjectsV2(request);
size_t newObjects = 0;
size_t totalObjects = 0; size_t totalObjects = 0;
if (outcome.IsSuccess()) if (outcome.IsSuccess())
@ -133,7 +134,13 @@ AwsLevel2DataProvider::ListObjects(std::chrono::system_clock::time_point date)
std::unique_lock lock(p->objectsMutex_); std::unique_lock lock(p->objectsMutex_);
p->objects_[time] = key; auto [it, inserted] =
p->objects_.insert_or_assign(time, key);
if (inserted)
{
newObjects++;
}
totalObjects++; totalObjects++;
} }
@ -145,7 +152,7 @@ AwsLevel2DataProvider::ListObjects(std::chrono::system_clock::time_point date)
outcome.GetError().GetMessage()); outcome.GetError().GetMessage());
} }
return totalObjects; return std::make_pair(newObjects, totalObjects);
} }
std::shared_ptr<wsr88d::Ar2vFile> std::shared_ptr<wsr88d::Ar2vFile>
@ -177,7 +184,7 @@ AwsLevel2DataProvider::LoadObjectByKey(const std::string& key)
return level2File; return level2File;
} }
void AwsLevel2DataProvider::Refresh() size_t AwsLevel2DataProvider::Refresh()
{ {
using namespace std::chrono; using namespace std::chrono;
@ -191,24 +198,28 @@ void AwsLevel2DataProvider::Refresh()
std::unique_lock lock(refreshMutex); std::unique_lock lock(refreshMutex);
size_t objectCount; size_t totalNewObjects = 0;
// If we haven't gotten any objects from today, first list objects for // If we haven't gotten any objects from today, first list objects for
// yesterday, to ensure we haven't missed any objects near midnight // yesterday, to ensure we haven't missed any objects near midnight
if (refreshDate < today) if (refreshDate < today)
{ {
objectCount = ListObjects(yesterday); auto [newObjects, totalObjects] = ListObjects(yesterday);
if (objectCount > 0) totalNewObjects = newObjects;
if (totalObjects > 0)
{ {
refreshDate = yesterday; refreshDate = yesterday;
} }
} }
objectCount = ListObjects(today); auto [newObjects, totalObjects] = ListObjects(today);
if (objectCount > 0) totalNewObjects += newObjects;
if (totalObjects > 0)
{ {
refreshDate = today; refreshDate = today;
} }
return totalNewObjects;
} }
std::chrono::system_clock::time_point std::chrono::system_clock::time_point