blakeridgway/supercell-wx
1
0
Fork 0
mirror of https://github.com/ciphervance/supercell-wx.git synced 2025-10-30 18:40:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

updated how the most recent scan was determined to ensure correctness

Browse source
This commit is contained in:
AdenKoperczak 2025-04-04 19:35:12 -04:00
parent add57ff26f
commit ac6d6093ec
No known key found for this signature in database
GPG key ID: 9843017036F62EE7
1 changed files with 20 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

20
wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp
View file

@ -368,16 +368,19 @@ AwsLevel2ChunksDataProvider::Impl::ListObjects()
int previousScanNumber = scanNumberMap.crbegin()->first;
const int firstScanNumber = scanNumberMap.cbegin()->first;
// Look for a gap in scan numbers. This indicates that is the latest
// scan.
// This indicates that highest number scan is the last scan
// (including if there is only 1 scan)
// NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers)
if (previousScanNumber != 999 || firstScanNumber != 1)
if (previousScanNumber != 999 || scans.size() == 1)
{
lastScanNumber = previousScanNumber;
}
else
{
// have already checked scan with highest number, so skip first
// Have already checked scan with highest number, so skip first
previousScanNumber = firstScanNumber;
bool first = true;
for (const auto& scan : scanNumberMap)
@ -397,11 +400,19 @@ AwsLevel2ChunksDataProvider::Impl::ListObjects()
}
if (lastScanNumber == -1)
{
// 999 is the last scan
if (firstScanNumber != 1)
{
lastScanNumber = previousScanNumber;
}
else
{
logger_->warn("Could not find last scan");
// TODO make sure this makes sence
return {false, 0, 0};
}
}
std::string& lastScanPrefix = scanNumberMap.at(lastScanNumber);
int secondLastScanNumber =
@ -444,7 +455,6 @@ AwsLevel2ChunksDataProvider::Impl::ListObjects()
currentScan_.hasAllFiles_ = false;
newObjects += 1;
}
logger_->error("{}", currentScan_.prefix_);
}
}
@ -510,6 +520,10 @@ bool AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord)
const int keyNumber = std::stoi(keyNumberStr);
if (keyNumber != scanRecord.nextFile_)
{
logger_->warn("Chunk found that was not in order {} {} {}",
key,
scanRecord.nextFile_,
keyNumber);
continue;
}