diff --git a/test/source/scwx/provider/aws_level2_data_provider.test.cpp b/test/source/scwx/provider/aws_level2_data_provider.test.cpp index a245b177..cdf6eacb 100644 --- a/test/source/scwx/provider/aws_level2_data_provider.test.cpp +++ b/test/source/scwx/provider/aws_level2_data_provider.test.cpp @@ -27,5 +27,38 @@ TEST(AwsLevel2DataProvider, Refresh) // TODO: Check object count } +TEST(AwsLevel2DataProvider, TimePointValid) +{ + using namespace std::chrono; + using sys_days = time_point; + + constexpr auto expectedTime = + sys_days {2022y / April / 30d} + 17h + 27min + 34s; + + auto time = AwsLevel2DataProvider::GetTimePointFromKey( + "2022/04/30/KLSX/KLSX20220430_172734_V06.gz"); + + EXPECT_EQ(time, expectedTime); +} + +TEST(AwsLevel2DataProvider, TimePointInvalid) +{ + constexpr std::chrono::system_clock::time_point expectedTime {}; + + auto time = AwsLevel2DataProvider::GetTimePointFromKey( + "2022/04/30/KLSX/KLSX20220430-172734_V06.gz"); + + EXPECT_EQ(time, expectedTime); +} + +TEST(AwsLevel2DataProvider, TimePointBadKey) +{ + constexpr std::chrono::system_clock::time_point expectedTime {}; + + auto time = AwsLevel2DataProvider::GetTimePointFromKey("???"); + + EXPECT_EQ(time, expectedTime); +} + } // namespace provider } // namespace scwx diff --git a/wxdata/include/scwx/provider/aws_level2_data_provider.hpp b/wxdata/include/scwx/provider/aws_level2_data_provider.hpp index a263ae3b..91dd3596 100644 --- a/wxdata/include/scwx/provider/aws_level2_data_provider.hpp +++ b/wxdata/include/scwx/provider/aws_level2_data_provider.hpp @@ -32,6 +32,9 @@ public: std::shared_ptr LoadObjectByKey(const std::string& key); void Refresh(); + static std::chrono::system_clock::time_point + GetTimePointFromKey(const std::string& key); + private: std::unique_ptr p; }; diff --git a/wxdata/source/scwx/provider/aws_level2_data_provider.cpp b/wxdata/source/scwx/provider/aws_level2_data_provider.cpp index 8a09c24a..a085f41b 100644 --- a/wxdata/source/scwx/provider/aws_level2_data_provider.cpp +++ b/wxdata/source/scwx/provider/aws_level2_data_provider.cpp @@ -2,6 +2,8 @@ #include #include +#include + #include #include #include @@ -15,7 +17,7 @@ namespace provider static const std::string logPrefix_ = "scwx::provider::aws_level2_data_provider"; -static const auto logger_ = scwx::util::Logger::Create(logPrefix_); +static const auto logger_ = util::Logger::Create(logPrefix_); static const std::string kDefaultBucketName_ = "noaa-nexrad-level2"; static const std::string kDefaultRegion_ = "us-east-1"; @@ -29,7 +31,9 @@ public: radarSite_ {radarSite}, bucketName_ {bucketName}, region_ {region}, - client_ {nullptr} + client_ {nullptr}, + objects_ {}, + objectsMutex_ {} { Aws::Client::ClientConfiguration config; config.region = region_; @@ -44,6 +48,9 @@ public: std::string region_; std::unique_ptr client_; + + std::map objects_; + std::shared_mutex objectsMutex_; }; AwsLevel2DataProvider::AwsLevel2DataProvider(const std::string& radarSite) : @@ -84,7 +91,19 @@ void AwsLevel2DataProvider::ListObjects( logger_->debug("Found {} objects", objects.size()); - // TODO: Store + // Store objects + std::for_each(objects.cbegin(), + objects.cend(), + [&](const Aws::S3::Model::Object& object) + { + // TODO: Skip MDM + std::string key = object.GetKey(); + auto time = GetTimePointFromKey(key); + + std::unique_lock lock(p->objectsMutex_); + + p->objects_[time] = key; + }); } else { @@ -130,5 +149,40 @@ void AwsLevel2DataProvider::Refresh() ListObjects(std::chrono::system_clock::now()); } +std::chrono::system_clock::time_point +AwsLevel2DataProvider::GetTimePointFromKey(const std::string& key) +{ + std::chrono::system_clock::time_point time {}; + + const size_t lastSeparator = key.rfind('/'); + const size_t offset = + (lastSeparator == std::string::npos) ? 0 : lastSeparator + 5; + + // Filename format is GGGGYYYYMMDD_TTTTTT(_V##).gz + static constexpr size_t formatSize = std::string("YYYYMMDD_TTTTTT").size(); + + if (key.size() >= offset + formatSize) + { + using namespace std::chrono; + + static const std::string timeFormat {"%Y%m%d_%H%M%S"}; + + std::string timeStr {key.substr(offset, formatSize)}; + std::istringstream in {timeStr}; + in >> parse(timeFormat, time); + + if (in.fail()) + { + logger_->warn("Invalid time: \"{}\"", timeStr); + } + } + else + { + logger_->warn("Time not parsable from key: \"{}\"", key); + } + + return time; +} + } // namespace provider } // namespace scwx