added scwx-qt/tools/update_radar_sites.py to update radar sites from NOAA HOMR data

scwx-qt/tools/update_radar_sites.py

@@ -0,0 +1,281 @@
+#!/usr/bin/env python3
+import requests
+import json
+import argparse
+
+NOAA_BASE = "http://www.ncdc.noaa.gov/homr/services/station"
+
+# Get the noaa station data.
+# platform is what platform should be searched for
+# current (should) filter to only current stations (always is filtered)
+# icao is a ICAO identifier. Without it we search for all.
+def get_noaa_stations(platform, current, icao = None):
+    params = {
+        "definitions":  "false",
+        "phrData":      "false",
+    }
+    if platform is not None:
+        params["platform"] = platform
+    if current:
+        params["current"] = "true"
+    if icao is not None:
+        params["qid"] = f"ICAO:{icao}"
+
+    res = requests.get(NOAA_BASE + "/search", params = params)
+
+    if res.ok:
+        return res.json()["stationCollection"]["stations"]
+    else:
+        print("NETWORK ERROR: Could not get resources from NOAA HOMR")
+        print(res.text)
+        exit(5)
+
+# dictionary to convert NOAA types to Supercell_wx types
+NOAA_TYPE_DICT = {
+    "TDWR":     "tdwr",
+    "NEXRAD":  "wsr88d"
+}
+
+# Given an list of objects, find the object with the best value for key.
+# The values that appear earlier in values are better.
+# subKey will take return the value under that key, not the full object.
+# parser (needs subKey) is a function that will have the value found by
+#   subKey and return a parsed version of it (often 'float' because HOMR
+#   data uses strings for floats)
+def extract_best(items, key, values, subKey = None, parser = None):
+    valuesPart = enumerate(reversed(values))
+    valueDict = dict([(k,v) for v,k in valuesPart])
+    best = None
+    bestInd = -1
+
+    for item in items:
+        index = valueDict.get(item[key], -1)
+        if bestInd is None or bestInd < index:
+            bestInd = index
+            best = item
+
+
+    if subKey is None or best is None:
+        return best
+
+    if parser is not None:
+        return parser(best[subKey])
+
+    return best[subKey]
+
+def make_noaa_stations_dict(noaaStations):
+    output = {}
+
+    for station in noaaStations:
+        stationId = extract_best(station["identifiers"], "idType", ["NEXRAD", "ICAO"], "id")
+
+        if stationId in output: # some stations are repeaded in non NEXRAD/TDWR locations.
+            continue
+
+        stationDict = {}
+        stationDict["lat"]   = float(station["header"]["latitude_dec"])
+        stationDict["lon"]   = float(station["header"]["longitude_dec"])
+        stationDict["elevation"] = extract_best(station["location"].get("elevations", []),
+                                                "elevationType",
+                                                ["GROUND"],
+                                                "elevationFeet",
+                                                float)
+        # These are some things that could be updated from the NOAA HOMR data,
+        # but are not necessary in the same format, so they are disabled.
+        """
+        stationDict["id"] = stationId
+        stationDict["country"] = station["location"]["geoInfo"]["countries"][0]["country"]
+        if "stateProvinces" in station["location"]["geoInfo"]:
+            stationDict["state"]   = station["location"]["geoInfo"]["stateProvinces"][0]["stateProvince"]
+        else:
+            stationDict["state"] = None
+        stationDict["place"]   = extract_best(station["names"], "nameType", ["PRINCIPAL"], "name")
+        stationDict["type"]    = NOAA_TYPE_DICT[station["platforms"][0]["platform"]]
+        #stationDict["tz"]      = station["location"]["geoInfo"]["utcOffsets"][0]["utcOffset"] # This is UTC offset, not timezone
+        """
+
+        output[stationId] = stationDict
+
+    return output
+
+# Get the list of updated stations (not in place), using the noaaStationsDict
+# from make_noaa_stations_dict
+def update_stations(noaaStationsDict, previousStations):
+    newStations = []
+    for station in previousStations:
+        newStation = station.copy()
+
+        if not station["id"] in noaaStationsDict:
+            # may be good idea to add fallback to a ICAO search for non active
+            # stations.
+            print(f"WARNING: Station '{station['id']}' not found in noaa data")
+
+            if "elevation" not in station:
+                newStation["elevation"] = None
+        else:
+            newStation.update(noaaStationsDict[station["id"]])
+
+        newStations.append(newStation)
+
+    return newStations
+
+# Customized dump routine. Formats it as one station per row, aligning items.
+def custom_dump(stations, file):
+    file.write("[\n")
+    lengths = {}
+    lastKey = None
+    keys = None
+
+    # Find length for each value, and ensure all stations have the same keys.
+    for station in stations:
+        for key, value in station.items():
+            length = len(json.dumps(value))
+            if key in lengths:
+                lengths[key] = max(length, lengths[key])
+            else:
+                lengths[key] = length
+            lastKey = key
+
+        newKeys = list(station.keys())
+        if keys is None:
+            keys = newKeys
+        elif keys != newKeys:
+            print("DUMP ERROR: Stations did not have the same keys.")
+            exit(3)
+
+    # Write out each station with the correct format.
+    lastType = None
+    for station in stations:
+        # put an empty line between NEXRAD and TDWR.
+        if lastType is not None and lastType != station["type"]:
+            file.write("\n")
+
+        file.write("\t{ ")
+
+        for key, value in station.items():
+            value = json.dumps(value)
+            file.write(f'"{key}": {value}')
+
+            if key != lastKey:
+                file.write(", ")
+                file.write(" " * (lengths[key] - len(value)))
+
+        if station == stations[-1]:
+            file.write(" }\n")
+        else:
+            file.write(" },\n")
+
+        lastType = station["type"]
+
+    file.write("]\n")
+
+# Write coordinates out to a file. Useful for checking against map program.
+def make_coords(stations, file):
+    for station in stations:
+        lat = str(abs(station["lat"]))
+        lat += "N" if station["lat"] > 0 else "S"
+
+        lon = str(abs(station["lon"]))
+        lon += "E" if station["lon"] > 0 else "W"
+
+        file.write(f"{lat} {lon}\n")
+
+def main():
+    parser = argparse.ArgumentParser(
+            description="""Update supercell-wx's location data for towers form NOAA's HOMR database.\n
+            Recommended Arguments: -u ../res/config/radar_sites.json -t""")
+    parser.add_argument("--current_file", "-u", type = str, default = None, required = False,
+                        help = "The 'radar_sites.json' file to update. Without this option, this will generate a new file")
+    parser.add_argument("--test_updated", "-t", default = False, action = "store_true",
+                        help = "Read in the updated file to ensure it is valid JSON. Should be used.")
+    parser.add_argument("--updated_file", "-o", type = str, default = None, required = False,
+                        help = "The updated 'radar_sites.json' file. The default is to overwrite the current one.")
+    parser.add_argument("--coord_file", "-c", type = str, default = None, required = False,
+                        help = "Output an additional file with the coordinates of each site.")
+    parser.add_argument("--resp_file", "-r", type = str, default = None, required = False,
+                        help = "Output most of the JSON from the responses.")
+    parser.add_argument("--input_json", "-i", type = str, default = None, required = False,
+                        help = "Instead of querying NOAA, just read in a JSON file made by \"-r\".")
+    parser.add_argument("--json_dump", "-j", default = False, action = "store_true",
+                        help = "Uses 'json.dump' instead of the custom dump function. Has worse formatting.")
+    parser.add_argument("--more_radars", "-m", default = False, action = "store_true",
+                        help = "Get AWOS and UPPERAIR stations as well. Should NOT be used.")
+    parser.add_argument("--current_only", "-C", default = False, action = "store_true",
+                        help = "Get only currently active stations. Does not seem to change anything.")
+
+    args = parser.parse_args()
+    # default to updating the same file as input
+    if args.updated_file is None:
+        if args.current_file is None:
+            parser.error("Needs 'current_file' or 'updated_file'")
+        args.updated_file = args.current_file
+
+    previousStations = None
+    if args.current_file is not None:
+        print(f"Reading Current Sites from '{args.current_file}'")
+        with open(args.current_file, "r") as file:
+            previousStations = json.load(file)
+
+    if args.input_json is None:
+        print("Getting NEXRAD stations")
+        noaaStations = get_noaa_stations("NEXRAD", args.current_only)
+
+        print("Getting TDWR stations")
+        noaaStations += get_noaa_stations("TDWR", args.current_only)
+
+        if args.more_radars: # Should not be used
+            print("Getting AWOS stations")
+            noaaStations += get_noaa_stations("AWOS", args.current_only)
+
+            print("Getting UPPERAIR stations")
+            noaaStations += get_noaa_stations("UPPERAIR", args.current_only)
+    else:
+        with open(args.input_json, "r") as file:
+            noaaStations = json.load(file)
+
+    if args.resp_file is not None:
+        with open(args.resp_file, "w") as file:
+            json.dump(noaaStations, file, indent=4)
+
+    print("Processing Data")
+    noaaStationsDict = make_noaa_stations_dict(noaaStations)
+
+    if args.current_file is None:
+        newStations = list(noaaStationsDict.values())
+    else:
+        newStations = update_stations(noaaStationsDict, previousStations)
+
+    print(f"Saving Updated Sites to '{args.updated_file}'")
+    with open(args.updated_file, "w") as file:
+        if args.json_dump:
+            json.dump(newStation, file)
+        else:
+            custom_dump(newStations, file)
+
+    if args.coord_file is not None:
+        print(f"Saving Coordinates to '{args.coord_file}'")
+        with open(args.coord_file, "w") as file:
+            make_coords(newStations, file)
+
+    if args.test_updated:
+        failed = False
+        with open(args.updated_file, "r") as file:
+            try:
+                data = json.load(file)
+                if len(data) < len(newStations):
+                    print(f"TEST ERROR: Only read in {len(data)} out of {len(newStations)} items.")
+                    failed = True
+                if json.dumps(data) != json.dumps(newStations):
+                    print(f"TEST ERROR: Dumps are not equal")
+                    failed = True
+            except Exception as e:
+                print(e)
+                failed = True
+
+            if failed:
+                exit(4)
+
+if __name__ == "__main__":
+    main()
+
+