blakeridgway/supercell-wx
1
0
Fork 0
mirror of https://github.com/ciphervance/supercell-wx.git synced 2025-10-29 18:10:06 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Initial moving of product listing to the background for level 3

Browse source
This commit is contained in:
Dan Paulat 2025-08-30 16:27:55 -05:00
parent 68f66c0c2f
commit f4226b487d
14 changed files with 356 additions and 178 deletions
Download patch file Download diff file Expand all files Collapse all files

1
scwx-qt/scwx-qt.cmake
View file

@ -234,6 +234,7 @@ set(HDR_TYPES source/scwx/qt/types/alert_types.hpp
source/scwx/qt/types/media_types.hpp
source/scwx/qt/types/qt_types.hpp
source/scwx/qt/types/radar_product_record.hpp
source/scwx/qt/types/radar_product_types.hpp
source/scwx/qt/types/text_event_key.hpp
source/scwx/qt/types/text_types.hpp
source/scwx/qt/types/texture_types.hpp

254
scwx-qt/source/scwx/qt/manager/radar_product_manager.cpp
View file

@ -15,6 +15,7 @@
#include <execution>
#include <mutex>
#include <shared_mutex>
#include <unordered_map>
#include <unordered_set>
#include <utility>
@ -36,11 +37,7 @@
# pragma warning(pop)
#endif
namespace scwx
{
namespace qt
{
namespace manager
namespace scwx::qt::manager
{
static const std::string logPrefix_ =
@ -210,7 +207,8 @@ public:
std::shared_ptr<types::RadarProductRecord>>
GetLevel2ProductRecords(std::chrono::system_clock::time_point time);
std::tuple<std::shared_ptr<types::RadarProductRecord>,
std::chrono::system_clock::time_point>
std::chrono::system_clock::time_point,
types::RadarProductLoadStatus>
GetLevel3ProductRecord(const std::string& product,
std::chrono::system_clock::time_point time);
std::shared_ptr<types::RadarProductRecord>
@ -224,15 +222,24 @@ public:
std::mutex& mutex,
std::chrono::system_clock::time_point time);
void
LoadProviderData(std::chrono::system_clock::time_point time,
std::shared_ptr<ProviderManager> providerManager,
RadarProductRecordMap& recordMap,
std::shared_mutex& recordMutex,
std::mutex& loadDataMutex,
const std::shared_ptr<request::NexradFileRequest>& request);
void PopulateLevel2ProductTimes(std::chrono::system_clock::time_point time);
LoadProviderData(std::chrono::system_clock::time_point time,
std::shared_ptr<ProviderManager> providerManager,
RadarProductRecordMap& recordMap,
std::shared_mutex& recordMutex,
std::mutex& loadDataMutex,
const std::shared_ptr<request::NexradFileRequest>& request);
bool AreLevel2ProductTimesPopulated(
std::chrono::system_clock::time_point time) const;
bool
AreLevel3ProductTimesPopulated(const std::string& product,
std::chrono::system_clock::time_point time);
void PopulateLevel2ProductTimes(std::chrono::system_clock::time_point time,
bool update = true);
void PopulateLevel3ProductTimes(const std::string& product,
std::chrono::system_clock::time_point time);
std::chrono::system_clock::time_point time,
bool update = true);
void UpdateAvailableProductsSync();
@ -243,11 +250,16 @@ public:
const float gateRangeOffset,
std::vector<float>& outputCoordinates);
static bool AreProductTimesPopulated(
const std::shared_ptr<ProviderManager>& providerManager,
std::chrono::system_clock::time_point time);
static void
PopulateProductTimes(std::shared_ptr<ProviderManager> providerManager,
RadarProductRecordMap& productRecordMap,
std::shared_mutex& productRecordMutex,
std::chrono::system_clock::time_point time);
std::chrono::system_clock::time_point time,
bool update);
static void
LoadNexradFile(CreateNexradFileFunction load,
@ -934,32 +946,32 @@ RadarProductManager::GetActiveVolumeTimes(
[&](const std::shared_ptr<provider::NexradDataProvider>& provider)
{
// For yesterday, today and tomorrow (in parallel)
std::for_each(std::execution::par,
dates.begin(),
dates.end(),
[&](const auto& date)
{
// Don't query for a time point in the future
if (date > scwx::util::time::now())
{
return;
}
std::for_each(
std::execution::par,
dates.begin(),
dates.end(),
[&](const auto& date)
{
// Don't query for a time point in the future
if (date > scwx::util::time::now())
{
return;
}
// Query the provider for volume time points
auto timePoints = provider->GetTimePointsByDate(date);
// Query the provider for volume time points
auto timePoints = provider->GetTimePointsByDate(date, true);
// TODO: Note, this will miss volume times present in
// Level 2 products with a second scan
// TODO: Note, this will miss volume times present in Level 2
// products with a second scan
// Lock the merged volume time list
std::unique_lock volumeTimesLock {volumeTimesMutex};
// Lock the merged volume time list
const std::unique_lock volumeTimesLock {volumeTimesMutex};
// Copy time points to the merged list
std::copy(
timePoints.begin(),
timePoints.end(),
std::inserter(volumeTimes, volumeTimes.end()));
});
// Copy time points to the merged list
std::copy(timePoints.begin(),
timePoints.end(),
std::inserter(volumeTimes, volumeTimes.end()));
});
});
// Return merged volume times list
@ -1202,21 +1214,70 @@ void RadarProductManagerImpl::LoadNexradFile(
}
}
bool RadarProductManagerImpl::AreLevel2ProductTimesPopulated(
std::chrono::system_clock::time_point time) const
{
return AreProductTimesPopulated(level2ProviderManager_, time);
}
bool RadarProductManagerImpl::AreLevel3ProductTimesPopulated(
const std::string& product, std::chrono::system_clock::time_point time)
{
// Get provider manager
const auto level3ProviderManager = GetLevel3ProviderManager(product);
return AreProductTimesPopulated(level3ProviderManager, time);
}
bool RadarProductManagerImpl::AreProductTimesPopulated(
const std::shared_ptr<ProviderManager>& providerManager,
std::chrono::system_clock::time_point time)
{
const auto today = std::chrono::floor<std::chrono::days>(time);
bool productTimesPopulated = true;
// Don't query for the epoch, assume populated
if (today == std::chrono::system_clock::time_point {})
{
return productTimesPopulated;
}
const auto yesterday = today - std::chrono::days {1};
const auto tomorrow = today + std::chrono::days {1};
const auto dates = {yesterday, today, tomorrow};
for (auto& date : dates)
{
// Don't query for a time point in the future
if (date > scwx::util::time::now())
{
continue;
}
if (!providerManager->provider_->IsDateCached(date))
{
productTimesPopulated = false;
}
}
return productTimesPopulated;
}
void RadarProductManagerImpl::PopulateLevel2ProductTimes(
std::chrono::system_clock::time_point time)
std::chrono::system_clock::time_point time, bool update)
{
PopulateProductTimes(level2ProviderManager_,
level2ProductRecords_,
level2ProductRecordMutex_,
time);
PopulateProductTimes(level2ChunksProviderManager_,
level2ProductRecords_,
level2ProductRecordMutex_,
time);
time,
update);
}
void RadarProductManagerImpl::PopulateLevel3ProductTimes(
const std::string& product, std::chrono::system_clock::time_point time)
const std::string& product,
std::chrono::system_clock::time_point time,
bool update)
{
// Get provider manager
auto level3ProviderManager = GetLevel3ProviderManager(product);
@ -1229,15 +1290,23 @@ void RadarProductManagerImpl::PopulateLevel3ProductTimes(
PopulateProductTimes(level3ProviderManager,
level3ProductRecords,
level3ProductRecordMutex_,
time);
time,
update);
}
void RadarProductManagerImpl::PopulateProductTimes(
std::shared_ptr<ProviderManager> providerManager,
RadarProductRecordMap& productRecordMap,
std::shared_mutex& productRecordMutex,
std::chrono::system_clock::time_point time)
std::chrono::system_clock::time_point time,
bool update)
{
logger_->debug("Populating product times (Update: {}): {}, {}, {}",
update,
common::GetRadarProductGroupName(providerManager->group_),
providerManager->product_,
scwx::util::time::TimeString(time));
const auto today = std::chrono::floor<std::chrono::days>(time);
// Don't query for the epoch
@ -1267,7 +1336,8 @@ void RadarProductManagerImpl::PopulateProductTimes(
// Query the provider for volume time points
auto timePoints =
providerManager->provider_->GetTimePointsByDate(date);
providerManager->provider_->GetTimePointsByDate(date,
update);
// Lock the merged volume time list
std::unique_lock volumeTimesLock {volumeTimesMutex};
@ -1381,16 +1451,46 @@ RadarProductManagerImpl::GetLevel2ProductRecords(
}
std::tuple<std::shared_ptr<types::RadarProductRecord>,
std::chrono::system_clock::time_point>
std::chrono::system_clock::time_point,
types::RadarProductLoadStatus>
RadarProductManagerImpl::GetLevel3ProductRecord(
const std::string& product, std::chrono::system_clock::time_point time)
{
std::shared_ptr<types::RadarProductRecord> record {nullptr};
RadarProductRecordMap::const_pointer recordPtr {nullptr};
std::chrono::system_clock::time_point recordTime {time};
types::RadarProductLoadStatus status {
types::RadarProductLoadStatus::ListingProducts};
// Ensure Level 3 product records are updated
PopulateLevel3ProductTimes(product, time);
if (!AreLevel3ProductTimesPopulated(product, time))
{
logger_->debug("Level 3 product times need populated: {}, {}",
product,
scwx::util::time::TimeString(time));
// Populate level 3 product times asynchronously
boost::asio::post(threadPool_,
[product, time, this]()
{
// Populate product times
PopulateLevel3ProductTimes(product, time);
// Signal finished
Q_EMIT self_->ProductTimesPopulated(
common::RadarProductGroup::Level3, product, time);
});
// Return listing products status
return {record, recordTime, status};
}
else
{
PopulateLevel3ProductTimes(product, time, false);
}
// Advance to loading product
status = types::RadarProductLoadStatus::LoadingProduct;
std::unique_lock lock {level3ProductRecordMutex_};
@ -1415,9 +1515,27 @@ RadarProductManagerImpl::GetLevel3ProductRecord(
if (recordPtr != nullptr)
{
using namespace std::chrono_literals;
// Don't check for an exact time match for level 3 products
recordTime = recordPtr->first;
record = recordPtr->second.lock();
if (
// For latest data, ensure it is from the last 24 hours
(time == std::chrono::system_clock::time_point {} &&
(recordTime > scwx::util::time::now() - 24h || recordTime == time)) ||
// For time queries, ensure data is within 24 hours of the request
(time != std::chrono::system_clock::time_point {} &&
std::chrono::abs(recordTime - time) < 24h))
{
record = recordPtr->second.lock();
}
else
{
// Reset the record
recordPtr = nullptr;
recordTime = time;
}
}
if (recordPtr != nullptr && record == nullptr &&
@ -1440,9 +1558,22 @@ RadarProductManagerImpl::GetLevel3ProductRecord(
});
self_->LoadLevel3Data(product, recordTime, request);
// Status is already set to LoadingProduct
}
return {record, recordTime};
if (recordPtr == nullptr)
{
// If the record is empty, the product is not available
status = types::RadarProductLoadStatus::ProductNotAvailable;
}
else if (record != nullptr)
{
// If the record was populated, the product has been loaded
status = types::RadarProductLoadStatus::ProductLoaded;
}
return {record, recordTime, status};
}
std::shared_ptr<types::RadarProductRecord>
@ -1543,7 +1674,8 @@ void RadarProductManagerImpl::UpdateRecentRecords(
std::tuple<std::shared_ptr<wsr88d::rda::ElevationScan>,
float,
std::vector<float>,
std::chrono::system_clock::time_point>
std::chrono::system_clock::time_point,
types::RadarProductLoadStatus>
RadarProductManager::GetLevel2Data(wsr88d::rda::DataBlockType dataBlockType,
float elevation,
std::chrono::system_clock::time_point time)
@ -1639,25 +1771,31 @@ RadarProductManager::GetLevel2Data(wsr88d::rda::DataBlockType dataBlockType,
}
}
return {radarData, elevationCut, elevationCuts, foundTime};
return {radarData,
elevationCut,
elevationCuts,
foundTime,
types::RadarProductLoadStatus::ProductLoaded};
}
std::tuple<std::shared_ptr<wsr88d::rpg::Level3Message>,
std::chrono::system_clock::time_point>
std::chrono::system_clock::time_point,
types::RadarProductLoadStatus>
RadarProductManager::GetLevel3Data(const std::string& product,
std::chrono::system_clock::time_point time)
{
std::shared_ptr<wsr88d::rpg::Level3Message> message = nullptr;
types::RadarProductLoadStatus status {};
std::shared_ptr<types::RadarProductRecord> record;
std::tie(record, time) = p->GetLevel3ProductRecord(product, time);
std::tie(record, time, status) = p->GetLevel3ProductRecord(product, time);
if (record != nullptr)
{
message = record->level3_file()->message();
}
return {message, time};
return {message, time, status};
}
common::Level3ProductCategoryMap
@ -1809,6 +1947,4 @@ RadarProductManager::Instance(const std::string& radarSite)
#include "radar_product_manager.moc"
} // namespace manager
} // namespace qt
} // namespace scwx
} // namespace scwx::qt::manager

25
scwx-qt/source/scwx/qt/manager/radar_product_manager.hpp
View file

@ -5,23 +5,19 @@
#include <scwx/qt/config/radar_site.hpp>
#include <scwx/qt/request/nexrad_file_request.hpp>
#include <scwx/qt/types/radar_product_record.hpp>
#include <scwx/qt/types/radar_product_types.hpp>
#include <scwx/util/time.hpp>
#include <scwx/wsr88d/ar2v_file.hpp>
#include <scwx/wsr88d/level3_file.hpp>
#include <memory>
#include <set>
#include <unordered_map>
#include <vector>
#include <boost/uuid/nil_generator.hpp>
#include <QObject>
namespace scwx
{
namespace qt
{
namespace manager
namespace scwx::qt::manager
{
class RadarProductManagerImpl;
@ -89,12 +85,13 @@ public:
* @param [in] time Radar product time
*
* @return Level 2 radar data, selected elevation cut, available elevation
* cuts and selected time
* cuts, selected time and product load status
*/
std::tuple<std::shared_ptr<wsr88d::rda::ElevationScan>,
float,
std::vector<float>,
std::chrono::system_clock::time_point>
std::chrono::system_clock::time_point,
types::RadarProductLoadStatus>
GetLevel2Data(wsr88d::rda::DataBlockType dataBlockType,
float elevation,
std::chrono::system_clock::time_point time = {});
@ -105,10 +102,11 @@ public:
* @param [in] product Radar product name
* @param [in] time Radar product time
*
* @return Level 3 message data and selected time
* @return Level 3 message data, selected time and product load status
*/
std::tuple<std::shared_ptr<wsr88d::rpg::Level3Message>,
std::chrono::system_clock::time_point>
std::chrono::system_clock::time_point,
types::RadarProductLoadStatus>
GetLevel3Data(const std::string& product,
std::chrono::system_clock::time_point time = {});
@ -151,6 +149,9 @@ signals:
bool isChunks,
std::chrono::system_clock::time_point latestTime);
void IncomingLevel2ElevationChanged(std::optional<float> incomingElevation);
void ProductTimesPopulated(common::RadarProductGroup group,
const std::string& product,
std::chrono::system_clock::time_point queryTime);
private:
std::unique_ptr<RadarProductManagerImpl> p;
@ -158,6 +159,4 @@ private:
friend class RadarProductManagerImpl;
};
} // namespace manager
} // namespace qt
} // namespace scwx
} // namespace scwx::qt::manager

16
scwx-qt/source/scwx/qt/types/radar_product_types.hpp Normal file
View file

@ -0,0 +1,16 @@
#pragma once
#include <cstdint>
namespace scwx::qt::types
{
enum class RadarProductLoadStatus : std::uint8_t
{
ProductLoaded,
ListingProducts,
LoadingProduct,
ProductNotAvailable
};
}

18
scwx-qt/source/scwx/qt/view/level2_product_view.cpp
View file

@ -11,11 +11,7 @@
#include <boost/range/irange.hpp>
#include <boost/timer/timer.hpp>
namespace scwx
{
namespace qt
{
namespace view
namespace scwx::qt::view
{
static const std::string logPrefix_ = "scwx::qt::view::level2_product_view";
@ -552,7 +548,11 @@ void Level2ProductView::ComputeSweep()
std::shared_ptr<wsr88d::rda::ElevationScan> radarData;
std::chrono::system_clock::time_point requestedTime {selected_time()};
std::tie(radarData, p->elevationCut_, p->elevationCuts_, std::ignore) =
std::tie(radarData,
p->elevationCut_,
p->elevationCuts_,
std::ignore,
std::ignore) =
radarProductManager->GetLevel2Data(
p->dataBlockType_, p->selectedElevation_, requestedTime);
@ -1369,7 +1369,7 @@ Level2ProductView::GetBinLevel(const common::Coordinate& coordinate) const
auto nextRadial = radarData->find((i + 1) % numRadials);
if (nextRadial != radarData->cend())
{
nextAngle = nextRadial->second->azimuth_angle();
nextAngle = nextRadial->second->azimuth_angle();
// Level 2 angles are the center of the bins.
const units::degrees<float> deltaAngle =
@ -1564,6 +1564,4 @@ std::shared_ptr<Level2ProductView> Level2ProductView::Create(
return std::make_shared<Level2ProductView>(product, radarProductManager);
}
} // namespace view
} // namespace qt
} // namespace scwx
} // namespace scwx::qt::view

27
scwx-qt/source/scwx/qt/view/level3_product_view.cpp
View file

@ -11,17 +11,12 @@
#include <scwx/wsr88d/rpg/radial_data_packet.hpp>
#include <limits>
#include <unordered_set>
#include <boost/range/irange.hpp>
#include <boost/timer/timer.hpp>
#include <fmt/format.h>
namespace scwx
{
namespace qt
{
namespace view
namespace scwx::qt::view
{
static const std::string logPrefix_ = "scwx::qt::view::level3_product_view";
@ -151,6 +146,22 @@ void Level3ProductView::ConnectRadarProductManager()
Update();
}
});
connect(radar_product_manager().get(),
&manager::RadarProductManager::ProductTimesPopulated,
this,
[this](common::RadarProductGroup group,
const std::string& product,
std::chrono::system_clock::time_point queryTime)
{
if (group == common::RadarProductGroup::Level3 &&
product == p->product_ && queryTime == selected_time())
{
// If the data associated with the currently selected time is
// reloaded, update the view
Update();
}
});
}
void Level3ProductView::DisconnectRadarProductManager()
@ -596,6 +607,4 @@ bool Level3ProductView::IgnoreUnits() const
return false;
}
} // namespace view
} // namespace qt
} // namespace scwx
} // namespace scwx::qt::view

32
scwx-qt/source/scwx/qt/view/level3_radial_view.cpp
View file

@ -10,11 +10,7 @@
#include <boost/range/irange.hpp>
#include <boost/timer/timer.hpp>
namespace scwx
{
namespace qt
{
namespace view
namespace scwx::qt::view
{
static const std::string logPrefix_ = "scwx::qt::view::level3_radial_view";
@ -31,15 +27,7 @@ static constexpr std::uint32_t VALUES_PER_VERTEX = 2u;
class Level3RadialView::Impl
{
public:
explicit Impl(Level3RadialView* self) :
self_ {self},
latitude_ {},
longitude_ {},
range_ {},
vcp_ {},
sweepTime_ {}
{
}
explicit Impl(Level3RadialView* self) : self_ {self} {}
~Impl() { threadPool_.join(); };
void ComputeCoordinates(
@ -65,13 +53,13 @@ public:
bool lastShowSmoothedRangeFolding_ {false};
bool lastSmoothingEnabled_ {false};
float latitude_;
float longitude_;
float latitude_ {};
float longitude_ {};
std::optional<float> elevation_ {};
float range_;
std::uint16_t vcp_;
float range_ {};
std::uint16_t vcp_ {};
std::chrono::system_clock::time_point sweepTime_;
std::chrono::system_clock::time_point sweepTime_ {};
};
Level3RadialView::Level3RadialView(
@ -148,7 +136,7 @@ void Level3RadialView::ComputeSweep()
std::shared_ptr<wsr88d::rpg::Level3Message> message;
std::chrono::system_clock::time_point requestedTime {selected_time()};
std::chrono::system_clock::time_point foundTime;
std::tie(message, foundTime) =
std::tie(message, foundTime, std::ignore) =
radarProductManager->GetLevel3Data(GetRadarProductName(), requestedTime);
// If a different time was found than what was requested, update it
@ -752,6 +740,4 @@ std::shared_ptr<Level3RadialView> Level3RadialView::Create(
return std::make_shared<Level3RadialView>(product, radarProductManager);
}
} // namespace view
} // namespace qt
} // namespace scwx
} // namespace scwx::qt::view

12
scwx-qt/source/scwx/qt/view/level3_raster_view.cpp
View file

@ -10,11 +10,7 @@
#include <boost/timer/timer.hpp>
#include <units/angle.h>
namespace scwx
{
namespace qt
{
namespace view
namespace scwx::qt::view
{
static const std::string logPrefix_ = "scwx::qt::view::level3_raster_view";
@ -125,7 +121,7 @@ void Level3RasterView::ComputeSweep()
std::shared_ptr<wsr88d::rpg::Level3Message> message;
std::chrono::system_clock::time_point requestedTime {selected_time()};
std::chrono::system_clock::time_point foundTime;
std::tie(message, foundTime) =
std::tie(message, foundTime, std::ignore) =
radarProductManager->GetLevel3Data(GetRadarProductName(), requestedTime);
// If a different time was found than what was requested, update it
@ -538,6 +534,4 @@ std::shared_ptr<Level3RasterView> Level3RasterView::Create(
return std::make_shared<Level3RasterView>(product, radarProductManager);
}
} // namespace view
} // namespace qt
} // namespace scwx
} // namespace scwx::qt::view

28
scwx-qt/source/scwx/qt/view/overlay_product_view.cpp
View file

@ -8,11 +8,7 @@
#include <boost/asio.hpp>
#include <boost/uuid/random_generator.hpp>
namespace scwx
{
namespace qt
{
namespace view
namespace scwx::qt::view
{
static const std::string logPrefix_ = "scwx::qt::view::overlay_product_view";
@ -128,6 +124,22 @@ void OverlayProductView::Impl::ConnectRadarProductManager()
}
},
Qt::QueuedConnection);
connect(radarProductManager_.get(),
&manager::RadarProductManager::ProductTimesPopulated,
self_,
[this](common::RadarProductGroup group,
const std::string& product,
std::chrono::system_clock::time_point queryTime)
{
if (group == common::RadarProductGroup::Level3 &&
product == kNst_ && queryTime == selectedTime_)
{
// If the data associated with the currently selected time is
// reloaded, update the view
Update(product);
}
});
}
void OverlayProductView::Impl::DisconnectRadarProductManager()
@ -286,7 +298,7 @@ void OverlayProductView::Impl::Update(const std::string& product)
std::shared_ptr<wsr88d::rpg::Level3Message> message;
std::chrono::system_clock::time_point requestedTime {selectedTime_};
std::chrono::system_clock::time_point foundTime;
std::tie(message, foundTime) =
std::tie(message, foundTime, std::ignore) =
radarProductManager_->GetLevel3Data(product, requestedTime);
// If a different time was found than what was requested, update it
@ -329,6 +341,4 @@ void OverlayProductView::SetAutoUpdate(bool enabled)
p->autoUpdateEnabled_ = enabled;
}
} // namespace view
} // namespace qt
} // namespace scwx
} // namespace scwx::qt::view

4
wxdata/include/scwx/provider/aws_level2_chunks_data_provider.hpp
View file

@ -46,7 +46,9 @@ public:
std::string FindLatestKey() override;
std::chrono::system_clock::time_point FindLatestTime() override;
std::vector<std::chrono::system_clock::time_point>
GetTimePointsByDate(std::chrono::system_clock::time_point date) override;
GetTimePointsByDate(std::chrono::system_clock::time_point date,
bool update) override;
bool IsDateCached(std::chrono::system_clock::time_point date) override;
std::tuple<bool, size_t, size_t>
ListObjects(std::chrono::system_clock::time_point date) override;
std::shared_ptr<wsr88d::NexradFile>

27
wxdata/include/scwx/provider/aws_nexrad_data_provider.hpp
View file

@ -2,17 +2,12 @@
#include <scwx/provider/nexrad_data_provider.hpp>
namespace Aws
{
namespace S3
namespace Aws::S3
{
class S3Client;
} // namespace S3
} // namespace Aws
} // namespace Aws::S3
namespace scwx
{
namespace provider
namespace scwx::provider
{
/**
@ -32,20 +27,23 @@ public:
AwsNexradDataProvider(AwsNexradDataProvider&&) noexcept;
AwsNexradDataProvider& operator=(AwsNexradDataProvider&&) noexcept;
size_t cache_size() const override;
[[nodiscard]] std::size_t cache_size() const override;
std::chrono::system_clock::time_point last_modified() const override;
std::chrono::seconds update_period() const override;
[[nodiscard]] std::chrono::system_clock::time_point
last_modified() const override;
[[nodiscard]] std::chrono::seconds update_period() const override;
std::string FindKey(std::chrono::system_clock::time_point time) override;
std::string FindLatestKey() override;
std::chrono::system_clock::time_point FindLatestTime() override;
std::vector<std::chrono::system_clock::time_point>
GetTimePointsByDate(std::chrono::system_clock::time_point date) override;
GetTimePointsByDate(std::chrono::system_clock::time_point date,
bool update) override;
bool IsDateCached(std::chrono::system_clock::time_point date) override;
std::tuple<bool, size_t, size_t>
ListObjects(std::chrono::system_clock::time_point date) override;
std::shared_ptr<wsr88d::NexradFile>
LoadObjectByKey(const std::string& key) override;
LoadObjectByKey(const std::string& key) override;
std::shared_ptr<wsr88d::NexradFile>
LoadObjectByTime(std::chrono::system_clock::time_point time) override;
std::pair<size_t, size_t> Refresh() override;
@ -61,5 +59,4 @@ private:
std::unique_ptr<Impl> p;
};
} // namespace provider
} // namespace scwx
} // namespace scwx::provider

29
wxdata/include/scwx/provider/nexrad_data_provider.hpp
View file

@ -7,9 +7,7 @@
#include <string>
#include <vector>
namespace scwx
{
namespace provider
namespace scwx::provider
{
class NexradDataProvider
@ -24,7 +22,7 @@ public:
NexradDataProvider(NexradDataProvider&&) noexcept;
NexradDataProvider& operator=(NexradDataProvider&&) noexcept;
virtual size_t cache_size() const = 0;
[[nodiscard]] virtual size_t cache_size() const = 0;
/**
* Gets the last modified time. This is equal to the most recent object's
@ -32,7 +30,8 @@ public:
*
* @return Last modified time
*/
virtual std::chrono::system_clock::time_point last_modified() const = 0;
[[nodiscard]] virtual std::chrono::system_clock::time_point
last_modified() const = 0;
/**
* Gets the current update period. This is equal to the difference between
@ -41,7 +40,7 @@ public:
*
* @return Update period
*/
virtual std::chrono::seconds update_period() const = 0;
[[nodiscard]] virtual std::chrono::seconds update_period() const = 0;
/**
* Finds the most recent key in the cache, no later than the time provided.
@ -116,7 +115,7 @@ public:
*
* @return NEXRAD data time point
*/
virtual std::chrono::system_clock::time_point
[[nodiscard]] virtual std::chrono::system_clock::time_point
GetTimePointByKey(const std::string& key) const = 0;
/**
@ -124,11 +123,22 @@ public:
* to the cache if required.
*
* @param date Date for which to get NEXRAD data time points
* @param update Whether or not to list and add data not present in the cache
*
* @return NEXRAD data time points
*/
virtual std::vector<std::chrono::system_clock::time_point>
GetTimePointsByDate(std::chrono::system_clock::time_point date) = 0;
GetTimePointsByDate(std::chrono::system_clock::time_point date,
bool update) = 0;
/**
* Determines if time points for the requested date are cached.
*
* @param date Date for which to query the cache
*
* @return Whether or not the requested date is cached
*/
virtual bool IsDateCached(std::chrono::system_clock::time_point date) = 0;
/**
* Requests available NEXRAD products for the current radar site, and adds
@ -148,5 +158,4 @@ private:
std::unique_ptr<Impl> p;
};
} // namespace provider
} // namespace scwx
} // namespace scwx::provider

9
wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp
View file

@ -289,11 +289,18 @@ AwsLevel2ChunksDataProvider::FindLatestTime()
std::vector<std::chrono::system_clock::time_point>
AwsLevel2ChunksDataProvider::GetTimePointsByDate(
std::chrono::system_clock::time_point /*date*/)
std::chrono::system_clock::time_point /* date */, bool /* update */)
{
return {};
}
bool AwsLevel2ChunksDataProvider::IsDateCached(
std::chrono::system_clock::time_point /* date */)
{
// No cache, default to true
return true;
}
std::chrono::system_clock::time_point
AwsLevel2ChunksDataProvider::Impl::GetScanTime(const std::string& prefix)
{

52
wxdata/source/scwx/provider/aws_nexrad_data_provider.cpp
View file

@ -15,9 +15,7 @@
#include <aws/s3/model/ListObjectsV2Request.h>
#include <fmt/chrono.h>
namespace scwx
{
namespace provider
namespace scwx::provider
{
static const std::string logPrefix_ =
@ -177,7 +175,7 @@ std::chrono::system_clock::time_point AwsNexradDataProvider::FindLatestTime()
std::vector<std::chrono::system_clock::time_point>
AwsNexradDataProvider::GetTimePointsByDate(
std::chrono::system_clock::time_point date)
std::chrono::system_clock::time_point date, bool update)
{
const auto day = std::chrono::floor<std::chrono::days>(date);
@ -188,23 +186,26 @@ AwsNexradDataProvider::GetTimePointsByDate(
std::shared_lock lock(p->objectsMutex_);
// Is the date present in the date list?
bool currentDatePresent;
bool currentDatePresent = false;
auto currentDateIterator =
std::find(p->objectDates_.cbegin(), p->objectDates_.cend(), day);
if (currentDateIterator == p->objectDates_.cend())
{
// Temporarily unlock mutex
lock.unlock();
// List objects, since the date is not present in the date list
auto [success, newObjects, totalObjects] = ListObjects(date);
if (success)
if (update)
{
p->UpdateObjectDates(date);
}
// Temporarily unlock mutex
lock.unlock();
// Re-lock mutex
lock.lock();
// List objects, since the date is not present in the date list
const auto [success, newObjects, totalObjects] = ListObjects(date);
if (success)
{
p->UpdateObjectDates(date);
}
// Re-lock mutex
lock.lock();
}
currentDatePresent = false;
}
@ -214,8 +215,8 @@ AwsNexradDataProvider::GetTimePointsByDate(
}
// Determine objects to retrieve
auto objectsBegin = p->objects_.lower_bound(day);
auto objectsEnd = p->objects_.lower_bound(day + std::chrono::days {1});
const auto objectsBegin = p->objects_.lower_bound(day);
const auto objectsEnd = p->objects_.lower_bound(day + std::chrono::days {1});
// Copy time points to destination vector
std::transform(objectsBegin,
@ -236,6 +237,20 @@ AwsNexradDataProvider::GetTimePointsByDate(
return timePoints;
}
bool AwsNexradDataProvider::IsDateCached(
std::chrono::system_clock::time_point date)
{
const auto day = std::chrono::floor<std::chrono::days>(date);
const std::shared_lock lock(p->objectsMutex_);
// Is the date present in the date list?
const auto currentDateIterator =
std::find(p->objectDates_.cbegin(), p->objectDates_.cend(), day);
return currentDateIterator != p->objectDates_.cend();
}
std::tuple<bool, size_t, size_t>
AwsNexradDataProvider::ListObjects(std::chrono::system_clock::time_point date)
{
@ -446,5 +461,4 @@ void AwsNexradDataProvider::Impl::UpdateObjectDates(
objectDates_.push_back(day);
}
} // namespace provider
} // namespace scwx
} // namespace scwx::provider