mirror of
https://github.com/ciphervance/supercell-wx.git
synced 2025-10-30 04:10:06 +00:00
added scwx-qt/tools/update_radar_sites.py to update radar sites from NOAA HOMR data
This commit is contained in:
AdenKoperczak
parent
af25556a09
commit
507292385d
1 changed files with 281 additions and 0 deletions
281
scwx-qt/tools/update_radar_sites.py
Normal file
281
scwx-qt/tools/update_radar_sites.py
Normal file
|
|
@ -0,0 +1,281 @@
|
|||
#!/usr/bin/env python3
|
||||
import requests
|
||||
import json
|
||||
import argparse
|
||||
|
||||
NOAA_BASE = "http://www.ncdc.noaa.gov/homr/services/station"
|
||||
|
||||
# Get the noaa station data.
|
||||
# platform is what platform should be searched for
|
||||
# current (should) filter to only current stations (always is filtered)
|
||||
# icao is a ICAO identifier. Without it we search for all.
|
||||
def get_noaa_stations(platform, current, icao = None):
|
||||
params = {
|
||||
"definitions": "false",
|
||||
"phrData": "false",
|
||||
}
|
||||
if platform is not None:
|
||||
params["platform"] = platform
|
||||
if current:
|
||||
params["current"] = "true"
|
||||
if icao is not None:
|
||||
params["qid"] = f"ICAO:{icao}"
|
||||
|
||||
res = requests.get(NOAA_BASE + "/search", params = params)
|
||||
|
||||
if res.ok:
|
||||
return res.json()["stationCollection"]["stations"]
|
||||
else:
|
||||
print("NETWORK ERROR: Could not get resources from NOAA HOMR")
|
||||
print(res.text)
|
||||
exit(5)
|
||||
|
||||
# dictionary to convert NOAA types to Supercell_wx types
|
||||
NOAA_TYPE_DICT = {
|
||||
"TDWR": "tdwr",
|
||||
"NEXRAD": "wsr88d"
|
||||
}
|
||||
|
||||
# Given an list of objects, find the object with the best value for key.
|
||||
# The values that appear earlier in values are better.
|
||||
# subKey will take return the value under that key, not the full object.
|
||||
# parser (needs subKey) is a function that will have the value found by
|
||||
# subKey and return a parsed version of it (often 'float' because HOMR
|
||||
# data uses strings for floats)
|
||||
def extract_best(items, key, values, subKey = None, parser = None):
|
||||
valuesPart = enumerate(reversed(values))
|
||||
valueDict = dict([(k,v) for v,k in valuesPart])
|
||||
best = None
|
||||
bestInd = -1
|
||||
|
||||
for item in items:
|
||||
index = valueDict.get(item[key], -1)
|
||||
if bestInd is None or bestInd < index:
|
||||
bestInd = index
|
||||
best = item
|
||||
|
||||
|
||||
if subKey is None or best is None:
|
||||
return best
|
||||
|
||||
if parser is not None:
|
||||
return parser(best[subKey])
|
||||
|
||||
return best[subKey]
|
||||
|
||||
def make_noaa_stations_dict(noaaStations):
|
||||
output = {}
|
||||
|
||||
for station in noaaStations:
|
||||
stationId = extract_best(station["identifiers"], "idType", ["NEXRAD", "ICAO"], "id")
|
||||
|
||||
if stationId in output: # some stations are repeaded in non NEXRAD/TDWR locations.
|
||||
continue
|
||||
|
||||
stationDict = {}
|
||||
stationDict["lat"] = float(station["header"]["latitude_dec"])
|
||||
stationDict["lon"] = float(station["header"]["longitude_dec"])
|
||||
stationDict["elevation"] = extract_best(station["location"].get("elevations", []),
|
||||
"elevationType",
|
||||
["GROUND"],
|
||||
"elevationFeet",
|
||||
float)
|
||||
# These are some things that could be updated from the NOAA HOMR data,
|
||||
# but are not necessary in the same format, so they are disabled.
|
||||
"""
|
||||
stationDict["id"] = stationId
|
||||
stationDict["country"] = station["location"]["geoInfo"]["countries"][0]["country"]
|
||||
if "stateProvinces" in station["location"]["geoInfo"]:
|
||||
stationDict["state"] = station["location"]["geoInfo"]["stateProvinces"][0]["stateProvince"]
|
||||
else:
|
||||
stationDict["state"] = None
|
||||
stationDict["place"] = extract_best(station["names"], "nameType", ["PRINCIPAL"], "name")
|
||||
stationDict["type"] = NOAA_TYPE_DICT[station["platforms"][0]["platform"]]
|
||||
#stationDict["tz"] = station["location"]["geoInfo"]["utcOffsets"][0]["utcOffset"] # This is UTC offset, not timezone
|
||||
"""
|
||||
|
||||
output[stationId] = stationDict
|
||||
|
||||
return output
|
||||
|
||||
# Get the list of updated stations (not in place), using the noaaStationsDict
|
||||
# from make_noaa_stations_dict
|
||||
def update_stations(noaaStationsDict, previousStations):
|
||||
newStations = []
|
||||
for station in previousStations:
|
||||
newStation = station.copy()
|
||||
|
||||
if not station["id"] in noaaStationsDict:
|
||||
# may be good idea to add fallback to a ICAO search for non active
|
||||
# stations.
|
||||
print(f"WARNING: Station '{station['id']}' not found in noaa data")
|
||||
|
||||
if "elevation" not in station:
|
||||
newStation["elevation"] = None
|
||||
else:
|
||||
newStation.update(noaaStationsDict[station["id"]])
|
||||
|
||||
newStations.append(newStation)
|
||||
|
||||
return newStations
|
||||
|
||||
# Customized dump routine. Formats it as one station per row, aligning items.
|
||||
def custom_dump(stations, file):
|
||||
file.write("[\n")
|
||||
lengths = {}
|
||||
lastKey = None
|
||||
keys = None
|
||||
|
||||
# Find length for each value, and ensure all stations have the same keys.
|
||||
for station in stations:
|
||||
for key, value in station.items():
|
||||
length = len(json.dumps(value))
|
||||
if key in lengths:
|
||||
lengths[key] = max(length, lengths[key])
|
||||
else:
|
||||
lengths[key] = length
|
||||
lastKey = key
|
||||
|
||||
newKeys = list(station.keys())
|
||||
if keys is None:
|
||||
keys = newKeys
|
||||
elif keys != newKeys:
|
||||
print("DUMP ERROR: Stations did not have the same keys.")
|
||||
exit(3)
|
||||
|
||||
# Write out each station with the correct format.
|
||||
lastType = None
|
||||
for station in stations:
|
||||
# put an empty line between NEXRAD and TDWR.
|
||||
if lastType is not None and lastType != station["type"]:
|
||||
file.write("\n")
|
||||
|
||||
file.write("\t{ ")
|
||||
|
||||
for key, value in station.items():
|
||||
value = json.dumps(value)
|
||||
file.write(f'"{key}": {value}')
|
||||
|
||||
if key != lastKey:
|
||||
file.write(", ")
|
||||
file.write(" " * (lengths[key] - len(value)))
|
||||
|
||||
if station == stations[-1]:
|
||||
file.write(" }\n")
|
||||
else:
|
||||
file.write(" },\n")
|
||||
|
||||
lastType = station["type"]
|
||||
|
||||
file.write("]\n")
|
||||
|
||||
# Write coordinates out to a file. Useful for checking against map program.
|
||||
def make_coords(stations, file):
|
||||
for station in stations:
|
||||
lat = str(abs(station["lat"]))
|
||||
lat += "N" if station["lat"] > 0 else "S"
|
||||
|
||||
lon = str(abs(station["lon"]))
|
||||
lon += "E" if station["lon"] > 0 else "W"
|
||||
|
||||
file.write(f"{lat} {lon}\n")
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="""Update supercell-wx's location data for towers form NOAA's HOMR database.\n
|
||||
Recommended Arguments: -u ../res/config/radar_sites.json -t""")
|
||||
parser.add_argument("--current_file", "-u", type = str, default = None, required = False,
|
||||
help = "The 'radar_sites.json' file to update. Without this option, this will generate a new file")
|
||||
parser.add_argument("--test_updated", "-t", default = False, action = "store_true",
|
||||
help = "Read in the updated file to ensure it is valid JSON. Should be used.")
|
||||
parser.add_argument("--updated_file", "-o", type = str, default = None, required = False,
|
||||
help = "The updated 'radar_sites.json' file. The default is to overwrite the current one.")
|
||||
parser.add_argument("--coord_file", "-c", type = str, default = None, required = False,
|
||||
help = "Output an additional file with the coordinates of each site.")
|
||||
parser.add_argument("--resp_file", "-r", type = str, default = None, required = False,
|
||||
help = "Output most of the JSON from the responses.")
|
||||
parser.add_argument("--input_json", "-i", type = str, default = None, required = False,
|
||||
help = "Instead of querying NOAA, just read in a JSON file made by \"-r\".")
|
||||
parser.add_argument("--json_dump", "-j", default = False, action = "store_true",
|
||||
help = "Uses 'json.dump' instead of the custom dump function. Has worse formatting.")
|
||||
parser.add_argument("--more_radars", "-m", default = False, action = "store_true",
|
||||
help = "Get AWOS and UPPERAIR stations as well. Should NOT be used.")
|
||||
parser.add_argument("--current_only", "-C", default = False, action = "store_true",
|
||||
help = "Get only currently active stations. Does not seem to change anything.")
|
||||
|
||||
args = parser.parse_args()
|
||||
# default to updating the same file as input
|
||||
if args.updated_file is None:
|
||||
if args.current_file is None:
|
||||
parser.error("Needs 'current_file' or 'updated_file'")
|
||||
args.updated_file = args.current_file
|
||||
|
||||
previousStations = None
|
||||
if args.current_file is not None:
|
||||
print(f"Reading Current Sites from '{args.current_file}'")
|
||||
with open(args.current_file, "r") as file:
|
||||
previousStations = json.load(file)
|
||||
|
||||
if args.input_json is None:
|
||||
print("Getting NEXRAD stations")
|
||||
noaaStations = get_noaa_stations("NEXRAD", args.current_only)
|
||||
|
||||
print("Getting TDWR stations")
|
||||
noaaStations += get_noaa_stations("TDWR", args.current_only)
|
||||
|
||||
if args.more_radars: # Should not be used
|
||||
print("Getting AWOS stations")
|
||||
noaaStations += get_noaa_stations("AWOS", args.current_only)
|
||||
|
||||
print("Getting UPPERAIR stations")
|
||||
noaaStations += get_noaa_stations("UPPERAIR", args.current_only)
|
||||
else:
|
||||
with open(args.input_json, "r") as file:
|
||||
noaaStations = json.load(file)
|
||||
|
||||
if args.resp_file is not None:
|
||||
with open(args.resp_file, "w") as file:
|
||||
json.dump(noaaStations, file, indent=4)
|
||||
|
||||
print("Processing Data")
|
||||
noaaStationsDict = make_noaa_stations_dict(noaaStations)
|
||||
|
||||
if args.current_file is None:
|
||||
newStations = list(noaaStationsDict.values())
|
||||
else:
|
||||
newStations = update_stations(noaaStationsDict, previousStations)
|
||||
|
||||
print(f"Saving Updated Sites to '{args.updated_file}'")
|
||||
with open(args.updated_file, "w") as file:
|
||||
if args.json_dump:
|
||||
json.dump(newStation, file)
|
||||
else:
|
||||
custom_dump(newStations, file)
|
||||
|
||||
if args.coord_file is not None:
|
||||
print(f"Saving Coordinates to '{args.coord_file}'")
|
||||
with open(args.coord_file, "w") as file:
|
||||
make_coords(newStations, file)
|
||||
|
||||
if args.test_updated:
|
||||
failed = False
|
||||
with open(args.updated_file, "r") as file:
|
||||
try:
|
||||
data = json.load(file)
|
||||
if len(data) < len(newStations):
|
||||
print(f"TEST ERROR: Only read in {len(data)} out of {len(newStations)} items.")
|
||||
failed = True
|
||||
if json.dumps(data) != json.dumps(newStations):
|
||||
print(f"TEST ERROR: Dumps are not equal")
|
||||
failed = True
|
||||
except Exception as e:
|
||||
print(e)
|
||||
failed = True
|
||||
|
||||
if failed:
|
||||
exit(4)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue