blakeridgway/supercell-wx
1
0
Fork 0
mirror of https://github.com/ciphervance/supercell-wx.git synced 2025-10-31 10:10:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

updated how the most recent scan was determined to ensure correctness

Browse source
This commit is contained in:
AdenKoperczak 2025-04-04 19:35:12 -04:00
parent add57ff26f
commit ac6d6093ec
No known key found for this signature in database
GPG key ID: 9843017036F62EE7
1 changed files with 20 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

26
wxdata/source/scwx/provider/aws_level2_chunks_data_provider.cpp
View file

@ -368,16 +368,19 @@ AwsLevel2ChunksDataProvider::Impl::ListObjects()
int previousScanNumber = scanNumberMap.crbegin()->first; int previousScanNumber = scanNumberMap.crbegin()->first;
const int firstScanNumber = scanNumberMap.cbegin()->first; const int firstScanNumber = scanNumberMap.cbegin()->first;
// Look for a gap in scan numbers. This indicates that is the latest
// scan.
// This indicates that highest number scan is the last scan // This indicates that highest number scan is the last scan
// (including if there is only 1 scan) // (including if there is only 1 scan)
// NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers) // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers)
if (previousScanNumber != 999 || firstScanNumber != 1) if (previousScanNumber != 999 || scans.size() == 1)
{ {
lastScanNumber = previousScanNumber; lastScanNumber = previousScanNumber;
} }
else else
{ {
// have already checked scan with highest number, so skip first // Have already checked scan with highest number, so skip first
previousScanNumber = firstScanNumber; previousScanNumber = firstScanNumber;
bool first = true; bool first = true;
for (const auto& scan : scanNumberMap) for (const auto& scan : scanNumberMap)
@ -398,9 +401,17 @@ AwsLevel2ChunksDataProvider::Impl::ListObjects()
if (lastScanNumber == -1) if (lastScanNumber == -1)
{ {
logger_->warn("Could not find last scan"); // 999 is the last scan
// TODO make sure this makes sence if (firstScanNumber != 1)
return {false, 0, 0}; {
lastScanNumber = previousScanNumber;
}
else
{
logger_->warn("Could not find last scan");
// TODO make sure this makes sence
return {false, 0, 0};
}
} }
std::string& lastScanPrefix = scanNumberMap.at(lastScanNumber); std::string& lastScanPrefix = scanNumberMap.at(lastScanNumber);
@ -444,7 +455,6 @@ AwsLevel2ChunksDataProvider::Impl::ListObjects()
currentScan_.hasAllFiles_ = false; currentScan_.hasAllFiles_ = false;
newObjects += 1; newObjects += 1;
} }
logger_->error("{}", currentScan_.prefix_);
} }
} }
@ -510,6 +520,10 @@ bool AwsLevel2ChunksDataProvider::Impl::LoadScan(Impl::ScanRecord& scanRecord)
const int keyNumber = std::stoi(keyNumberStr); const int keyNumber = std::stoi(keyNumberStr);
if (keyNumber != scanRecord.nextFile_) if (keyNumber != scanRecord.nextFile_)
{ {
logger_->warn("Chunk found that was not in order {} {} {}",
key,
scanRecord.nextFile_,
keyNumber);
continue; continue;
} }