From d142624772a6130ba943ed0909547c858f9d769a Mon Sep 17 00:00:00 2001 From: Robbie Byrd Date: Sun, 12 Jul 2026 19:32:12 +0000 Subject: [PATCH 1/9] feat(weather-recon): scaffold package (issue #184 phase 1) Co-Authored-By: Claude Fable 5 --- packages/tools/weather-recon/pyproject.toml | 23 +++++++++++++++++++ .../tools/weather-recon/tests/__init__.py | 0 .../weather_recon.egg-info/PKG-INFO | 12 ++++++++++ .../weather_recon.egg-info/SOURCES.txt | 7 ++++++ .../dependency_links.txt | 1 + .../weather_recon.egg-info/requires.txt | 8 +++++++ .../weather_recon.egg-info/top_level.txt | 1 + .../weather-recon/weather_recon/__init__.py | 0 8 files changed, 52 insertions(+) create mode 100644 packages/tools/weather-recon/pyproject.toml create mode 100644 packages/tools/weather-recon/tests/__init__.py create mode 100644 packages/tools/weather-recon/weather_recon.egg-info/PKG-INFO create mode 100644 packages/tools/weather-recon/weather_recon.egg-info/SOURCES.txt create mode 100644 packages/tools/weather-recon/weather_recon.egg-info/dependency_links.txt create mode 100644 packages/tools/weather-recon/weather_recon.egg-info/requires.txt create mode 100644 packages/tools/weather-recon/weather_recon.egg-info/top_level.txt create mode 100644 packages/tools/weather-recon/weather_recon/__init__.py diff --git a/packages/tools/weather-recon/pyproject.toml b/packages/tools/weather-recon/pyproject.toml new file mode 100644 index 00000000..f36307cc --- /dev/null +++ b/packages/tools/weather-recon/pyproject.toml @@ -0,0 +1,23 @@ +[project] +name = "weather-recon" +version = "0.1.0" +description = "Load Sept 2001 weather stations/observations/forecasts into Directus (Prefect flows)" +requires-python = ">=3.12" +dependencies = [ + "prefect>=3.7,<4", + "prefect-kubernetes>=0.5", + "httpx>=0.27", +] + +[project.optional-dependencies] +dev = ["pytest>=8", "ruff>=0.4", "timezonefinder>=6"] + +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +packages = ["weather_recon"] + +[tool.ruff] +line-length = 100 diff --git a/packages/tools/weather-recon/tests/__init__.py b/packages/tools/weather-recon/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/packages/tools/weather-recon/weather_recon.egg-info/PKG-INFO b/packages/tools/weather-recon/weather_recon.egg-info/PKG-INFO new file mode 100644 index 00000000..d59069be --- /dev/null +++ b/packages/tools/weather-recon/weather_recon.egg-info/PKG-INFO @@ -0,0 +1,12 @@ +Metadata-Version: 2.4 +Name: weather-recon +Version: 0.1.0 +Summary: Load Sept 2001 weather stations/observations/forecasts into Directus (Prefect flows) +Requires-Python: >=3.12 +Requires-Dist: prefect<4,>=3.7 +Requires-Dist: prefect-kubernetes>=0.5 +Requires-Dist: httpx>=0.27 +Provides-Extra: dev +Requires-Dist: pytest>=8; extra == "dev" +Requires-Dist: ruff>=0.4; extra == "dev" +Requires-Dist: timezonefinder>=6; extra == "dev" diff --git a/packages/tools/weather-recon/weather_recon.egg-info/SOURCES.txt b/packages/tools/weather-recon/weather_recon.egg-info/SOURCES.txt new file mode 100644 index 00000000..2407d608 --- /dev/null +++ b/packages/tools/weather-recon/weather_recon.egg-info/SOURCES.txt @@ -0,0 +1,7 @@ +pyproject.toml +weather_recon/__init__.py +weather_recon.egg-info/PKG-INFO +weather_recon.egg-info/SOURCES.txt +weather_recon.egg-info/dependency_links.txt +weather_recon.egg-info/requires.txt +weather_recon.egg-info/top_level.txt \ No newline at end of file diff --git a/packages/tools/weather-recon/weather_recon.egg-info/dependency_links.txt b/packages/tools/weather-recon/weather_recon.egg-info/dependency_links.txt new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/packages/tools/weather-recon/weather_recon.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/packages/tools/weather-recon/weather_recon.egg-info/requires.txt b/packages/tools/weather-recon/weather_recon.egg-info/requires.txt new file mode 100644 index 00000000..09ac1e08 --- /dev/null +++ b/packages/tools/weather-recon/weather_recon.egg-info/requires.txt @@ -0,0 +1,8 @@ +prefect<4,>=3.7 +prefect-kubernetes>=0.5 +httpx>=0.27 + +[dev] +pytest>=8 +ruff>=0.4 +timezonefinder>=6 diff --git a/packages/tools/weather-recon/weather_recon.egg-info/top_level.txt b/packages/tools/weather-recon/weather_recon.egg-info/top_level.txt new file mode 100644 index 00000000..2a373116 --- /dev/null +++ b/packages/tools/weather-recon/weather_recon.egg-info/top_level.txt @@ -0,0 +1 @@ +weather_recon diff --git a/packages/tools/weather-recon/weather_recon/__init__.py b/packages/tools/weather-recon/weather_recon/__init__.py new file mode 100644 index 00000000..e69de29b From 686c1616ca24563f51af8e90bfe1bb1645a69562 Mon Sep 17 00:00:00 2001 From: Robbie Byrd Date: Sun, 12 Jul 2026 19:34:30 +0000 Subject: [PATCH 2/9] chore(weather-recon): gitignore build artifacts Co-Authored-By: Claude Fable 5 --- packages/tools/weather-recon/.gitignore | 3 +++ .../weather-recon/weather_recon.egg-info/PKG-INFO | 12 ------------ .../weather-recon/weather_recon.egg-info/SOURCES.txt | 7 ------- .../weather_recon.egg-info/dependency_links.txt | 1 - .../weather_recon.egg-info/requires.txt | 8 -------- .../weather_recon.egg-info/top_level.txt | 1 - 6 files changed, 3 insertions(+), 29 deletions(-) create mode 100644 packages/tools/weather-recon/.gitignore delete mode 100644 packages/tools/weather-recon/weather_recon.egg-info/PKG-INFO delete mode 100644 packages/tools/weather-recon/weather_recon.egg-info/SOURCES.txt delete mode 100644 packages/tools/weather-recon/weather_recon.egg-info/dependency_links.txt delete mode 100644 packages/tools/weather-recon/weather_recon.egg-info/requires.txt delete mode 100644 packages/tools/weather-recon/weather_recon.egg-info/top_level.txt diff --git a/packages/tools/weather-recon/.gitignore b/packages/tools/weather-recon/.gitignore new file mode 100644 index 00000000..31492377 --- /dev/null +++ b/packages/tools/weather-recon/.gitignore @@ -0,0 +1,3 @@ +.venv/ +__pycache__/ +*.egg-info/ diff --git a/packages/tools/weather-recon/weather_recon.egg-info/PKG-INFO b/packages/tools/weather-recon/weather_recon.egg-info/PKG-INFO deleted file mode 100644 index d59069be..00000000 --- a/packages/tools/weather-recon/weather_recon.egg-info/PKG-INFO +++ /dev/null @@ -1,12 +0,0 @@ -Metadata-Version: 2.4 -Name: weather-recon -Version: 0.1.0 -Summary: Load Sept 2001 weather stations/observations/forecasts into Directus (Prefect flows) -Requires-Python: >=3.12 -Requires-Dist: prefect<4,>=3.7 -Requires-Dist: prefect-kubernetes>=0.5 -Requires-Dist: httpx>=0.27 -Provides-Extra: dev -Requires-Dist: pytest>=8; extra == "dev" -Requires-Dist: ruff>=0.4; extra == "dev" -Requires-Dist: timezonefinder>=6; extra == "dev" diff --git a/packages/tools/weather-recon/weather_recon.egg-info/SOURCES.txt b/packages/tools/weather-recon/weather_recon.egg-info/SOURCES.txt deleted file mode 100644 index 2407d608..00000000 --- a/packages/tools/weather-recon/weather_recon.egg-info/SOURCES.txt +++ /dev/null @@ -1,7 +0,0 @@ -pyproject.toml -weather_recon/__init__.py -weather_recon.egg-info/PKG-INFO -weather_recon.egg-info/SOURCES.txt -weather_recon.egg-info/dependency_links.txt -weather_recon.egg-info/requires.txt -weather_recon.egg-info/top_level.txt \ No newline at end of file diff --git a/packages/tools/weather-recon/weather_recon.egg-info/dependency_links.txt b/packages/tools/weather-recon/weather_recon.egg-info/dependency_links.txt deleted file mode 100644 index 8b137891..00000000 --- a/packages/tools/weather-recon/weather_recon.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/packages/tools/weather-recon/weather_recon.egg-info/requires.txt b/packages/tools/weather-recon/weather_recon.egg-info/requires.txt deleted file mode 100644 index 09ac1e08..00000000 --- a/packages/tools/weather-recon/weather_recon.egg-info/requires.txt +++ /dev/null @@ -1,8 +0,0 @@ -prefect<4,>=3.7 -prefect-kubernetes>=0.5 -httpx>=0.27 - -[dev] -pytest>=8 -ruff>=0.4 -timezonefinder>=6 diff --git a/packages/tools/weather-recon/weather_recon.egg-info/top_level.txt b/packages/tools/weather-recon/weather_recon.egg-info/top_level.txt deleted file mode 100644 index 2a373116..00000000 --- a/packages/tools/weather-recon/weather_recon.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -weather_recon From b4a46ab0bbcde440869cf57fe12bac89ae39e990 Mon Sep 17 00:00:00 2001 From: Robbie Byrd Date: Sun, 12 Jul 2026 19:39:08 +0000 Subject: [PATCH 3/9] feat(weather-recon): curated US/CA/MX station list from ISD history Adds scripts/build_stations.py (pick_station_rows/station_record pure functions, TDD'd) and the committed data/stations.csv it generates from NOAA's isd-history.csv: 187 stations covering 2001-09-09..12 (US 152, CA 17, MX 18). 7 of the original 175 curated ICAOs lacked ISD coverage for the window; resolved by checking data/isd-history.csv for a same-city substitute: - CYQB (Quebec City) -> substituted CWQB (same airport, alternate identifier, continuous 1955-2025 coverage) - KMDT (Harrisburg) -> substituted KCXY (Harrisburg Capital City, covers 2000-2003) - KHOU (Houston Hobby) -> dropped; KIAH already covers Houston in the curated set - KBTR (Baton Rouge), KFLL (Fort Lauderdale), KGPT (Gulfport-Biloxi), KRSW (SW Florida/Fort Myers) -> dropped; no same-city ISD row with 2001-09 coverage found (each has a pre-1998/pre-2000 gap and a post-2005/2006 row, but nothing spanning 2001) data/isd-history.csv (~3 MB NOAA cache) is gitignored, not committed. Co-Authored-By: Claude Fable 5 --- packages/tools/weather-recon/.gitignore | 1 + .../tools/weather-recon/data/stations.csv | 188 ++++++++++++++++++ .../weather-recon/scripts/build_stations.py | 128 ++++++++++++ .../tests/test_build_stations.py | 45 +++++ 4 files changed, 362 insertions(+) create mode 100644 packages/tools/weather-recon/data/stations.csv create mode 100644 packages/tools/weather-recon/scripts/build_stations.py create mode 100644 packages/tools/weather-recon/tests/test_build_stations.py diff --git a/packages/tools/weather-recon/.gitignore b/packages/tools/weather-recon/.gitignore index 31492377..672afb36 100644 --- a/packages/tools/weather-recon/.gitignore +++ b/packages/tools/weather-recon/.gitignore @@ -1,3 +1,4 @@ .venv/ __pycache__/ *.egg-info/ +data/isd-history.csv diff --git a/packages/tools/weather-recon/data/stations.csv b/packages/tools/weather-recon/data/stations.csv new file mode 100644 index 00000000..840a407f --- /dev/null +++ b/packages/tools/weather-recon/data/stations.csv @@ -0,0 +1,188 @@ +station_id,name,lat,lon,elevation_m,country,tz,isd_id +CWQB,QUEBEC/JEAN LESAGE INTL QUE,46.783,-71.383,74.0,CA,America/Toronto,717140-99999 +CYEG,EDMONTON INTL,53.31,-113.58,723.3,CA,America/Edmonton,711230-99999 +CYHZ,HALIFAX INTL / STANFIELD INTL,44.881,-63.509,145.4,CA,America/Halifax,713950-99999 +CYOW,OTTAWA MACDONALD CARTIER INTL,45.323,-75.669,114.0,CA,America/Toronto,716280-99999 +CYQR,REGINA INTL,50.433,-104.667,577.3,CA,America/Regina,718630-99999 +CYQT,THUNDER BAY A,48.367,-89.333,199.0,CA,America/Toronto,717490-99999 +CYSJ,SAINT JOHN,45.317,-65.883,108.8,CA,America/Moncton,716090-99999 +CYUL,MONTREAL/TRUDEAU INT,45.467,-73.733,36.0,CA,America/Toronto,716270-94792 +CYVR,VANCOUVER INTL,49.194,-123.184,4.3,CA,America/Vancouver,718920-99999 +CYWG,WINNIPEG INTL / JAMES ARMSTRONG RICHARDSON INTL,49.91,-97.24,238.7,CA,America/Winnipeg,718520-99999 +CYXE,SASKATOON JOHN G DIEFENBAKER INTL,52.167,-106.7,503.8,CA,America/Regina,718660-99999 +CYXY,WHITEHORSE INTL / ERIK NIELSEN INTL,60.714,-135.076,707.0,CA,America/Whitehorse,719640-99999 +CYYC,CALGARY INTL,51.114,-114.02,1084.2,CA,America/Edmonton,718770-99999 +CYYJ,VICTORIA INTL,48.647,-123.426,19.2,CA,America/Vancouver,717990-99999 +CYYT,ST JOHNS INTL,47.619,-52.752,140.5,CA,America/St_Johns,718010-99999 +CYYZ,LESTER B PEARSON INTL,43.677,-79.631,173.4,CA,America/Toronto,716240-99999 +CYZF,YELLOWKNIFE,62.463,-114.44,205.7,CA,America/Edmonton,719360-99999 +KABE,LEHIGH VALLEY INTERNATIONAL AIRPORT,40.65,-75.448,117.4,US,America/New_York,725170-14737 +KABQ,ALBUQUERQUE INTL SUNPORT AIRPORT,35.042,-106.615,1618.4,US,America/Denver,723650-23050 +KALB,ALBANY INTERNATIONAL AIRPORT,42.747,-73.799,85.4,US,America/New_York,725180-14735 +KAMA,AMARILLO RICK HUSBAND INTL AIRPORT,35.22,-101.717,1099.3,US,America/Chicago,723630-23047 +KATL,HARTSFIELD-JACKSON ATLANTA INTL AP,33.63,-84.442,308.3,US,America/New_York,722190-13874 +KAUS,AUSTIN-BERGSTROM INTL AIRPORT,30.183,-97.68,146.3,US,America/Chicago,722540-13904 +KAVP,WILKES-BARRE/SCRANTON INTL AIRPORT,41.333,-75.723,289.9,US,America/New_York,725130-14777 +KBDL,BRADLEY INTERNATIONAL AIRPORT,41.937,-72.682,51.6,US,America/New_York,725080-14740 +KBFL,MEADOWS FIELD AIRPORT,35.434,-119.055,149.3,US,America/Los_Angeles,723840-23155 +KBGR,BANGOR INTL,44.8,-68.817,60.0,US,America/New_York,726070-99999 +KBHM,BIRMINGHAM INTERNATIONAL AIRPORT,33.565,-86.745,187.7,US,America/Chicago,722280-13876 +KBIL,BILLINGS LOGAN INTERNATIONAL AIRPORT,45.807,-108.546,1094.3,US,America/Denver,726770-24033 +KBIS,BISMARCK MUNICIPAL AIRPORT,46.782,-100.758,503.2,US,America/Chicago,727640-24011 +KBNA,NASHVILLE INTERNATIONAL AIRPORT,36.111,-86.688,179.0,US,America/Chicago,723270-13897 +KBOI,BOISE AIR TERMINAL/GOWEN FD AIRPORT,43.567,-116.241,860.5,US,America/Boise,726810-24131 +KBOS,GEN E L LOGAN INTERNATIONAL AIRPORT,42.361,-71.01,3.3,US,America/New_York,725090-14739 +KBRO,BROWNSVILLE/S PADRE ISLAND INTL AP,25.915,-97.423,5.7,US,America/Chicago,722500-12919 +KBTV,BURLINGTON INTERNATIONAL AIRPORT,44.468,-73.15,101.1,US,America/New_York,726170-14742 +KBUF,BUFFALO NIAGARA INTERNATIONAL AIRPOR,42.94,-78.736,216.2,US,America/New_York,725280-14733 +KBWI,BALTIMORE-WASHINGTON INTL AIRPORT,39.173,-76.684,42.0,US,America/New_York,724060-93721 +KBZN,GALLATIN FIELD AIRPORT,45.788,-111.162,1350.7,US,America/Denver,726797-24132 +KCAE,COLUMBIA METROPOLITAN AIRPORT,33.942,-81.118,68.3,US,America/New_York,723100-13883 +KCHA,LOVELL FIELD AIRPORT,35.034,-85.2,204.0,US,America/New_York,723240-13882 +KCHS,CHARLESTON AFB/INTERNATIONAL AIRPORT,32.899,-80.041,11.8,US,America/New_York,722080-13880 +KCID,EASTERN IOWA REGIONAL AIRPORT,41.883,-91.725,256.0,US,America/Chicago,725450-14990 +KCLE,CLEVELAND-HOPKINS INTERNATIONAL AP,41.406,-81.852,236.8,US,America/New_York,725240-14820 +KCLT,CHARLOTTE/DOUGLAS INTERNATIONAL AP,35.223,-80.954,222.6,US,America/New_York,723140-13881 +KCMH,PORT COLUMBUS INTERNATIONAL AIRPORT,39.991,-82.877,246.8,US,America/New_York,724280-14821 +KCOS,CITY OF COLORADO SPRINGS MUNICIPAL AP,38.809,-104.689,1884.2,US,America/Denver,724660-93037 +KCOU,COLUMBIA REGIONAL AIRPORT,38.817,-92.215,272.2,US,America/Chicago,724450-03945 +KCPR,NATRONA COUNTY INTERNATIONAL AP,42.898,-106.474,1621.1,US,America/Denver,725690-24089 +KCRP,CORPUS CHRISTI INTERNATIONAL AIRPORT,27.773,-97.513,12.5,US,America/Chicago,722510-12924 +KCRW,YEAGER AIRPORT,38.38,-81.591,278.1,US,America/New_York,724140-13866 +KCVG,CINCINNATI/NORTHERN KENTUCKY INTL AP,39.044,-84.672,262.4,US,America/New_York,724210-93814 +KCXY,CAPITAL CITY AIRPORT,40.218,-76.855,102.2,US,America/New_York,725118-14751 +KCYS,CHEYENNE AIRPORT,41.158,-104.808,1864.9,US,America/Denver,725640-24018 +KDAL,DALLAS LOVE FIELD AIRPORT,32.838,-96.836,147.6,US,America/Chicago,722580-13960 +KDAY,J.M.COX DAYTON INTERNATIONAL AIRPORT,39.906,-84.219,302.6,US,America/New_York,724290-93815 +KDCA,RONALD REAGAN WASHINGTON NATL AP,38.847,-77.035,4.1,US,America/New_York,724050-13743 +KDEN,DENVER INTERNATIONAL AIRPORT,39.847,-104.656,1647.2,US,America/Denver,725650-03017 +KDFW,DALLAS/FT WORTH INTERNATIONAL AP,32.897,-97.022,165.7,US,America/Chicago,722590-03927 +KDLH,DULUTH INTERNATIONAL AIRPORT,46.844,-92.187,433.4,US,America/Chicago,727450-14913 +KDSM,DES MOINES INTERNATIONAL AIRPORT,41.534,-93.653,286.3,US,America/Chicago,725460-14933 +KDTW,DETROIT METRO WAYNE COUNTY AIRPORT,42.231,-83.331,191.9,US,America/Detroit,725370-94847 +KELP,EL PASO INTERNATIONAL AIRPORT,31.812,-106.377,1202.1,US,America/Denver,722700-23044 +KELY,ELY AIRPORT/YELLAND FIELD/AIRPORT,39.295,-114.847,1904.9,US,America/Los_Angeles,724860-23154 +KEUG,MAHLON SWEET FIELD AIRPORT,44.133,-123.216,109.0,US,America/Los_Angeles,726930-24221 +KEWR,NEWARK LIBERTY INTERNATIONAL AP,40.683,-74.169,2.0,US,America/New_York,725020-14734 +KEYW,KEY WEST INTERNATIONAL AIRPORT,24.557,-81.755,0.3,US,America/New_York,722010-12836 +KFAR,HECTOR INTERNATIONAL AIRPORT,46.924,-96.812,272.8,US,America/Chicago,727530-14914 +KFAT,FRESNO YOSEMITE INTERNATIONAL AIRPORT,36.78,-119.72,101.9,US,America/Los_Angeles,723890-93193 +KFLG,FLAGSTAFF AIRPORT,35.133,-111.667,2135.0,US,America/Phoenix,723750-99999 +KFSD,JOE FOSS FIELD AIRPORT,43.578,-96.754,432.4,US,America/Chicago,726510-14944 +KFSM,FORT SMITH REGIONAL AIRPORT,35.333,-94.365,136.7,US,America/Chicago,723440-13964 +KFWA,FORT WAYNE INTERNATIONAL AIRPORT,40.972,-85.206,243.0,US,America/Indiana/Indianapolis,725330-14827 +KGEG,SPOKANE INTERNATIONAL AIRPORT,47.622,-117.528,717.7,US,America/Los_Angeles,727850-24157 +KGJT,GRAND JUNCTION REGIONAL AIRPORT,39.134,-108.541,1470.4,US,America/Denver,724760-23066 +KGRB,AUSTIN STRAUBEL INTERNATIONAL AP,44.48,-88.137,208.6,US,America/Chicago,726450-14898 +KGRR,GERALD R FORD INTERNATIONAL AIRPORT,42.882,-85.523,240.3,US,America/Detroit,726350-94860 +KGSO,PIEDMONT TRIAD INTERNATIONAL AIRPORT,36.097,-79.943,275.0,US,America/New_York,723170-13723 +KGTF,GREAT FALLS INTERNATIONAL AIRPORT,47.473,-111.383,1117.7,US,America/Denver,727750-24143 +KHLN,HELENA REGIONAL AIRPORT,46.604,-111.989,1178.1,US,America/Denver,727720-24144 +KHSV,HUNTSVILLE INTL/C.T.JONES FIELD AIRPORT,34.644,-86.786,190.0,US,America/Chicago,723230-03856 +KIAD,WASHINGTON DULLES INTERNATIONAL AP,38.935,-77.447,89.5,US,America/New_York,724030-93738 +KIAH,G BUSH INTERCONTINENTAL AP/HOUSTON AP,29.984,-95.361,27.5,US,America/Chicago,722430-12960 +KICT,WICHITA EISENHOWER NATIONAL,37.648,-97.43,401.7,US,America/Chicago,724500-03928 +KIDA,IDAHO FALLS REGIONAL ARPT,43.52,-112.068,1442.7,US,America/Boise,725785-24145 +KIND,INDIANAPOLIS INTERNATIONAL AIRPORT,39.725,-86.282,241.3,US,America/Indiana/Indianapolis,724380-93819 +KJAN,JACKSON INTERNATIONAL AIRPORT,32.32,-90.078,90.2,US,America/Chicago,722350-03940 +KJAX,JACKSONVILLE INTERNATIONAL AIRPORT,30.495,-81.694,7.2,US,America/New_York,722060-13889 +KJFK,JOHN F KENNEDY INTERNATIONAL AIRPORT,40.639,-73.764,2.7,US,America/New_York,744860-94789 +KLAN,CAPITAL CITY AIRPORT,42.776,-84.6,261.2,US,America/Detroit,725390-14836 +KLAS,MCCARRAN INTERNATIONAL AIRPORT,36.072,-115.163,662.8,US,America/Los_Angeles,723860-23169 +KLAX,LOS ANGELES INTERNATIONAL AIRPORT,33.938,-118.387,29.7,US,America/Los_Angeles,722950-23174 +KLBB,LUBBOCK INTERNATIONAL AIRPORT,33.666,-101.823,993.2,US,America/Chicago,722670-23042 +KLCH,LAKE CHARLES REGIONAL AIRPORT,30.126,-93.228,2.0,US,America/Chicago,722400-03937 +KLEX,BLUE GRASS AIRPORT,38.034,-84.611,293.2,US,America/New_York,724220-93820 +KLGA,LA GUARDIA AIRPORT,40.779,-73.88,3.0,US,America/New_York,725030-14732 +KLIT,ADAMS FIELD AIRPORT,34.727,-92.236,76.4,US,America/Chicago,723403-13963 +KLNK,LINCOLN MUNICIPAL AIRPORT,40.848,-96.765,356.7,US,America/Chicago,725510-14939 +KMAF,MIDLAND INTERNATIONAL AIRPORT,31.948,-102.209,872.4,US,America/Chicago,722650-23023 +KMCI,KANSAS CITY INTERNATIONAL AIRPORT,39.297,-94.731,307.4,US,America/Chicago,724460-03947 +KMCO,ORLANDO INTERNATIONAL AIRPORT,28.418,-81.324,27.1,US,America/New_York,722050-12815 +KMDW,CHICAGO MIDWAY INTL ARPT,41.784,-87.755,185.8,US,America/Chicago,725340-14819 +KMEM,MEMPHIS INTERNATIONAL AIRPORT,35.056,-89.986,76.6,US,America/Chicago,723340-13893 +KMFR,ROGUE VALLEY INTL-MEDFORD AIRPORT,42.375,-122.877,400.3,US,America/Los_Angeles,725970-24225 +KMGM,MONTGOMERY RGNL (DANNELLY FD) AP,32.3,-86.407,66.2,US,America/Chicago,722260-13895 +KMHT,MANCHESTER AIRPORT,42.93,-71.436,69.8,US,America/New_York,743945-14710 +KMIA,MIAMI INTERNATIONAL AIRPORT,25.788,-80.317,1.4,US,America/New_York,722020-12839 +KMKE,GENERAL MITCHELL INTERNATIONAL AP,42.955,-87.905,203.3,US,America/Chicago,726400-14839 +KMLI,QUAD CITY INTERNATIONAL AIRPORT,41.448,-90.524,175.5,US,America/Chicago,725440-14923 +KMOB,MOBILE REGIONAL AIRPORT,30.688,-88.246,65.5,US,America/Chicago,722230-13894 +KMRY,MONTEREY PENINSULA AIRPORT,36.59,-121.849,48.8,US,America/Los_Angeles,999999-23259 +KMSN,DANE CO REGIONAL-TRUAX FIELD AIRPORT,43.141,-89.345,261.7,US,America/Chicago,726410-14837 +KMSO,MISSOULA INTERNATIONAL AIRPORT,46.921,-114.094,973.8,US,America/Denver,727730-24153 +KMSP,MINNEAPOLIS-ST PAUL INTERNATIONAL AP,44.885,-93.231,254.5,US,America/Chicago,726580-14922 +KMSY,LOUIS ARMSTRONG NEW ORLEANS INTL AP,29.998,-90.278,-1.0,US,America/Chicago,722310-12916 +KOAK,METRO OAKLAND INTL AIRPORT,37.718,-122.233,1.5,US,America/Los_Angeles,724930-23230 +KOKC,WILL ROGERS WORLD AIRPORT,35.388,-97.6,389.9,US,America/Chicago,723530-13967 +KOMA,EPPLEY AIRFIELD AIRPORT,41.312,-95.902,298.8,US,America/Chicago,725500-14942 +KORD,CHICAGO O'HARE INTERNATIONAL AIRPORT,41.96,-87.932,204.8,US,America/Chicago,725300-94846 +KORF,NORFOLK INTERNATIONAL AIRPORT,36.904,-76.193,3.3,US,America/New_York,723080-13737 +KPBI,PALM BEACH INTERNATIONAL AIRPORT,26.685,-80.099,3.5,US,America/New_York,722030-12844 +KPDX,PORTLAND INTERNATIONAL AIRPORT,45.596,-122.609,6.7,US,America/Los_Angeles,726980-24229 +KPHL,PHILADELPHIA INTERNATIONAL AIRPORT,39.873,-75.227,2.2,US,America/New_York,724080-13739 +KPHX,PHOENIX SKY HARBOR INTL AIRPORT,33.428,-112.004,339.2,US,America/Phoenix,722780-23183 +KPIA,GREATER PEORIA REGIONAL AIRPORT,40.667,-89.684,199.7,US,America/Chicago,725320-14842 +KPIH,POCATELLO REGIONAL AIRPORT,42.92,-112.572,1356.6,US,America/Boise,725780-24156 +KPIT,PITTSBURGH INTERNATIONAL AIRPORT,40.485,-80.214,341.0,US,America/New_York,725200-94823 +KPUB,PUEBLO MEMORIAL AIRPORT,38.289,-104.506,1424.5,US,America/Denver,724640-93058 +KPVD,THEODORE F GREEN STATE AIRPORT,41.723,-71.432,15.4,US,America/New_York,725070-14765 +KPWM,PORTLAND INTERNATIONAL JETPORT AP,43.642,-70.304,13.5,US,America/New_York,726060-14764 +KRAP,RAPID CITY REGIONAL AIRPORT,44.046,-103.054,963.8,US,America/Denver,726620-24090 +KRDD,REDDING MUNICIPAL AIRPORT,40.515,-122.298,152.6,US,America/Los_Angeles,725920-24257 +KRDU,RALEIGH-DURHAM INTERNATIONAL AP,35.892,-78.782,120.5,US,America/New_York,723060-13722 +KRFD,GREATER ROCKFORD AIRPORT,42.193,-89.093,221.0,US,America/Chicago,725430-94822 +KRIC,RICHMOND INTERNATIONAL AIRPORT,37.512,-77.323,50.7,US,America/New_York,724010-13740 +KRNO,RENO/TAHOE INTERNATIONAL AIRPORT,39.508,-119.768,1342.5,US,America/Los_Angeles,724880-23185 +KROC,GREATER ROCHESTER INTERNATIONAL AP,43.117,-77.675,164.5,US,America/New_York,725290-14768 +KROW,ROSWELL INTERNATIONAL AIR CENTER AP,33.307,-104.508,1104.6,US,America/Denver,722680-23009 +KRST,ROCHESTER INTERNATIONAL AIRPORT,43.904,-92.492,397.9,US,America/Chicago,726440-14925 +KSAN,SAN DIEGO INTERNATIONAL AIRPORT,32.734,-117.183,4.6,US,America/Los_Angeles,722900-23188 +KSAT,SAN ANTONIO INTERNATIONAL AIRPORT,29.544,-98.484,243.6,US,America/Chicago,722530-12921 +KSAV,SAVANNAH/HILTON HEAD INTL AIRPORT,32.131,-81.202,8.7,US,America/New_York,722070-03822 +KSBA,SANTA BARBARA MUNICIPAL AIRPORT,34.424,-119.842,2.5,US,America/Los_Angeles,723925-23190 +KSBN,SOUTH BEND REGIONAL AIRPORT,41.707,-86.316,235.4,US,America/Indiana/Indianapolis,725350-14848 +KSDF,LOUISVILLE INTL-STANDIFORD FIELD AP,38.177,-85.731,146.3,US,America/Kentucky/Louisville,724230-93821 +KSEA,SEATTLE-TACOMA INTERNATIONAL AIRPORT,47.445,-122.314,112.5,US,America/Los_Angeles,727930-24233 +KSFO,SAN FRANCISCO INTERNATIONAL AIRPORT,37.62,-122.366,3.2,US,America/Los_Angeles,724940-23234 +KSGF,SPRINGFIELD-BRANSON REGIONAL AIRPORT,37.24,-93.39,384.7,US,America/Chicago,724400-13995 +KSHV,SHREVEPORT REGIONAL AIRPORT,32.447,-93.824,69.6,US,America/Chicago,722480-13957 +KSJC,N Y. MINETA SN JO INTL APT,37.359,-121.924,15.0,US,America/Los_Angeles,724945-23293 +KSLC,SALT LAKE CITY INTERNATIONAL AIRPORT,40.771,-111.965,1288.4,US,America/Denver,725720-24127 +KSMF,SACRAMENTO INTL AIRPORT,38.701,-121.595,7.5,US,America/Los_Angeles,724839-93225 +KSPI,ABRAHAM LINCOLN CAPITAL AIRPORT,39.845,-89.684,179.7,US,America/Chicago,724390-93822 +KSTL,LAMBERT-ST LOUIS INTERNATIONAL AP,38.752,-90.373,162.0,US,America/Chicago,724340-13994 +KSYR,SYRACUSE HANCOCK INTERNATIONAL AP,43.111,-76.104,124.9,US,America/New_York,725190-14771 +KTLH,TALLAHASSEE REGIONAL AIRPORT,30.394,-84.351,17.1,US,America/New_York,722140-93805 +KTOL,TOLEDO EXPRESS AIRPORT,41.587,-83.805,205.5,US,America/New_York,725360-94830 +KTPA,TAMPA INTERNATIONAL AIRPORT,27.963,-82.54,1.8,US,America/New_York,722110-12842 +KTUL,TULSA INTERNATIONAL AIRPORT,36.199,-95.878,194.9,US,America/Chicago,723560-13968 +KTUS,TUCSON INTERNATIONAL AIRPORT,32.132,-110.956,777.6,US,America/Phoenix,722740-23160 +KTYS,MC GHEE TYSON AIRPORT,35.818,-83.986,295.8,US,America/New_York,723260-13891 +KYKM,YAKIMA AIR TERMINAL/MCALSR FIELD AP,46.564,-120.535,321.0,US,America/Los_Angeles,727810-24243 +KYUM,YUMA INTL AIRPORT,32.65,-114.6,63.0,US,America/Phoenix,722800-99999 +MMAA,GENERAL JUAN N ALVAREZ INTL / ACAPULCO INTL,16.757,-99.754,4.0,MX,America/Mexico_City,768056-99999 +MMCS,ABRAHAM GONZALEZ INTL / CIUDAD JUAREZ INTL,31.636,-106.429,1189.9,MX,America/Ciudad_Juarez,760753-99999 +MMCU,GENERAL R FIERRO VILLALOBOS INTL / CHIHUAHUA INTL,28.703,-105.965,1360.0,MX,America/Chihuahua,762253-99999 +MMGL,DON MIGUEL HIDALGO Y COSTILLA INTL / GUADALAJARA INTL,20.522,-103.311,1528.9,MX,America/Mexico_City,766133-99999 +MMHO,GENERAL IGNACIO P GARCIA INTL / HERMOSILLO INTL,29.096,-111.048,191.1,MX,America/Hermosillo,761600-99999 +MMLO,GUANAJUATO INTL / LEON INTL,20.983,-101.483,1815.4,MX,America/Mexico_City,765773-99999 +MMMD,LICENCIADO MANUEL CRESCENCIO REJON INT / MERIDA INTL,20.937,-89.658,11.6,MX,America/Merida,766440-99999 +MMMX,LICENCIADO BENITO JUAREZ INTL / MEXICO CITY INTL,19.436,-99.072,2229.9,MX,America/Mexico_City,766793-99999 +MMMY,GENERAL MARIANO ESCOBEDO INTL / MONTERREY INTL,25.778,-100.107,389.5,MX,America/Monterrey,763943-99999 +MMMZ,GENERAL RAFAEL BUELNA INTL / MAZATLAN INTL,23.161,-106.266,11.6,MX,America/Mazatlan,764593-99999 +MMOX,XOXOCOTLAN INTL / OAXACA INTL,17.0,-96.733,1520.6,MX,America/Mexico_City,767755-99999 +MMPR,LICENCIADO GUSTAVO DIAZ ORDAZ INTL / PUERTO VALLARTA INTL,20.68,-105.254,7.0,MX,America/Mexico_City,766013-99999 +MMSD,LOS CABOS INTL / SAN JOSE DEL CABO INTL,23.152,-109.721,114.0,MX,America/Mazatlan,764056-99999 +MMTJ,GENERAL ABELARDO L RODRIGUEZ INTL / TIJUANA INTL,32.541,-116.97,149.0,MX,America/Tijuana,760013-99999 +MMTM,GENERAL FRANCISCO JAVIER MINA INTL / TAMPICO INTL,22.296,-97.866,24.4,MX,America/Monterrey,765493-99999 +MMUN,CANCUN INTL,21.037,-86.877,6.1,MX,America/Cancun,765950-99999 +MMVA,C P A CARLOS ROVIROSA INTL / VILLAHERMOSA INTL,17.983,-92.817,14.0,MX,America/Mexico_City,767433-99999 +MMVR,GENERAL HERIBERTO JARA INTL / VERACRUZ INTL,19.146,-96.187,27.4,MX,America/Mexico_City,766913-99999 +PAFA,FAIRBANKS INTERNATIONAL,64.803,-147.876,131.1,US,America/Anchorage,702610-26411 +PAJN,JUNEAU INTERNATIONAL,58.354,-134.556,5.9,US,America/Juneau,703810-25309 +PANC,TED STEVENS ANCHORAGE INTL,61.169,-150.028,38.0,US,America/Anchorage,702730-26451 +PHLI,LIHUE AIRPORT,21.98,-159.339,31.9,US,Pacific/Honolulu,911650-22536 +PHNL,HONOLULU INTERNATIONAL AIRPORT,21.324,-157.939,1.9,US,Pacific/Honolulu,911820-22521 +PHOG,KAHULUI AIRPORT,20.889,-156.435,14.0,US,Pacific/Honolulu,911900-22516 +PHTO,HILO INTERNATIONAL AIRPORT,19.719,-155.049,8.8,US,Pacific/Honolulu,912850-21504 diff --git a/packages/tools/weather-recon/scripts/build_stations.py b/packages/tools/weather-recon/scripts/build_stations.py new file mode 100644 index 00000000..e5f9636b --- /dev/null +++ b/packages/tools/weather-recon/scripts/build_stations.py @@ -0,0 +1,128 @@ +""" +Build data/stations.csv from NOAA's isd-history.csv and the curated ICAO list. + +Dev-run only (output is committed): + + python scripts/build_stations.py # uses cached data/isd-history.csv if present + python scripts/build_stations.py --refresh # re-download the master list + +Reports any curated ICAO with no ISD row covering 2001-09-09..12 — trim or +substitute those and re-run until the report is empty. +""" + +import argparse +import csv +import sys +from pathlib import Path + +ISD_HISTORY_URL = "https://www.ncei.noaa.gov/pub/data/noaa/isd-history.csv" +ROOT = Path(__file__).resolve().parents[1] +CACHE = ROOT / "data" / "isd-history.csv" +OUT = ROOT / "data" / "stations.csv" +FIELDS = ["station_id", "name", "lat", "lon", "elevation_m", "country", "tz", "isd_id"] + +# Curated station list: major-city METAR sites across the US, Canada and +# Mexico with 2001 coverage. ~175 sites; every US state, ~13 CA / ~15 MX. +CURATED_ICAOS = { + # US — Northeast + "KBOS", "KPWM", "KBGR", "KBTV", "KMHT", "KPVD", "KBDL", "KALB", "KSYR", + "KROC", "KBUF", "KJFK", "KLGA", "KEWR", "KPHL", "KABE", "KAVP", "KCXY", + "KPIT", "KDCA", "KIAD", "KBWI", + # US — Southeast + "KRIC", "KORF", "KRDU", "KGSO", "KCLT", "KCAE", "KCHS", "KSAV", "KATL", + "KTLH", "KJAX", "KMCO", "KTPA", "KMIA", "KPBI", "KEYW", + "KBHM", "KMGM", "KMOB", "KHSV", "KBNA", "KMEM", "KTYS", "KCHA", "KSDF", + "KLEX", "KCRW", "KJAN", + # US — Midwest + "KCLE", "KCMH", "KCVG", "KDAY", "KTOL", "KIND", "KFWA", "KSBN", "KDTW", + "KGRR", "KLAN", "KORD", "KMDW", "KRFD", "KPIA", "KSPI", "KMLI", "KMKE", + "KMSN", "KGRB", "KMSP", "KDLH", "KRST", "KFAR", "KBIS", "KFSD", "KRAP", + "KDSM", "KCID", "KOMA", "KLNK", "KICT", "KSTL", "KMCI", "KSGF", "KCOU", + # US — South-central + "KOKC", "KTUL", "KLIT", "KFSM", "KDFW", "KDAL", "KIAH", "KAUS", + "KSAT", "KELP", "KLBB", "KAMA", "KMAF", "KCRP", "KBRO", "KSHV", + "KMSY", "KLCH", + # US — Mountain + "KDEN", "KCOS", "KPUB", "KGJT", "KCYS", "KCPR", "KBIL", "KGTF", "KBZN", + "KMSO", "KHLN", "KBOI", "KIDA", "KPIH", "KSLC", "KLAS", "KRNO", "KELY", + "KPHX", "KTUS", "KFLG", "KYUM", "KABQ", "KROW", + # US — Pacific + "KSEA", "KGEG", "KYKM", "KPDX", "KEUG", "KMFR", "KSFO", "KOAK", "KSJC", + "KSMF", "KFAT", "KBFL", "KLAX", "KSAN", "KSBA", "KMRY", "KRDD", + # US — Alaska & Hawaii + "PANC", "PAFA", "PAJN", "PHNL", "PHOG", "PHTO", "PHLI", + # Canada + "CYVR", "CYYJ", "CYYC", "CYEG", "CYXE", "CYQR", "CYWG", "CYQT", "CYYZ", + "CYOW", "CYUL", "CWQB", "CYHZ", "CYSJ", "CYYT", "CYXY", "CYZF", + # Mexico + "MMMX", "MMGL", "MMMY", "MMTJ", "MMHO", "MMCS", "MMCU", "MMMZ", "MMSD", + "MMPR", "MMLO", "MMAA", "MMVR", "MMMD", "MMUN", "MMOX", "MMVA", "MMTM", +} + + +def pick_station_rows(rows, icaos, start="20010909", end="20010912"): + """Per ICAO in `icaos`, the isd-history row covering [start, end] with the + latest END. ICAOs with no covering row are simply absent from the result.""" + picked = {} + for row in rows: + icao = (row.get("ICAO") or "").strip() + if icao not in icaos: + continue + if not (row["BEGIN"] <= start and row["END"] >= end): + continue + if icao not in picked or row["END"] > picked[icao]["END"]: + picked[icao] = row + return picked + + +def station_record(row, tz): + """One stations.csv record from an isd-history row.""" + elev = row.get("ELEV(M)", "").strip() + return { + "station_id": row["ICAO"].strip(), + "name": row["STATION NAME"].strip(), + "lat": float(row["LAT"]), + "lon": float(row["LON"]), + "elevation_m": float(elev) if elev else None, + "country": row["CTRY"].strip(), + "tz": tz, + "isd_id": f'{row["USAF"]}-{row["WBAN"]}', + } + + +def main(refresh=False): + import httpx + from timezonefinder import TimezoneFinder + + if refresh or not CACHE.is_file(): + print(f"downloading {ISD_HISTORY_URL} ...") + CACHE.parent.mkdir(parents=True, exist_ok=True) + CACHE.write_bytes(httpx.get(ISD_HISTORY_URL, timeout=120, + follow_redirects=True).raise_for_status().content) + with CACHE.open(newline="", encoding="utf-8") as f: + picked = pick_station_rows(csv.DictReader(f), CURATED_ICAOS) + + missing = sorted(CURATED_ICAOS - set(picked)) + tf = TimezoneFinder() + records = [] + for icao in sorted(picked): + row = picked[icao] + tz = tf.timezone_at(lng=float(row["LON"]), lat=float(row["LAT"])) + records.append(station_record(row, tz)) + + with OUT.open("w", newline="", encoding="utf-8") as f: + w = csv.DictWriter(f, fieldnames=FIELDS) + w.writeheader() + w.writerows(records) + print(f"wrote {len(records)} stations to {OUT}") + if missing: + print(f"WARNING: {len(missing)} curated ICAOs lack 2001-09 ISD coverage: " + f"{', '.join(missing)}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--refresh", action="store_true") + sys.exit(main(refresh=ap.parse_args().refresh)) diff --git a/packages/tools/weather-recon/tests/test_build_stations.py b/packages/tools/weather-recon/tests/test_build_stations.py new file mode 100644 index 00000000..4fdc94ba --- /dev/null +++ b/packages/tools/weather-recon/tests/test_build_stations.py @@ -0,0 +1,45 @@ +# tests/test_build_stations.py +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +from build_stations import pick_station_rows, station_record # noqa: E402 + + +def _row(icao, begin, end, usaf="725300", wban="94846", lat="41.995", + lon="-87.934", elev="200.6", ctry="US", name="CHICAGO O'HARE"): + return {"USAF": usaf, "WBAN": wban, "STATION NAME": name, "CTRY": ctry, + "STATE": "IL", "ICAO": icao, "LAT": lat, "LON": lon, + "ELEV(M)": elev, "BEGIN": begin, "END": end} + + +def test_pick_keeps_only_rows_covering_the_window(): + rows = [_row("KORD", "19730101", "19991231"), # ends before window + _row("KORD", "20000101", "20051231"), # covers window + _row("KMSY", "20011001", "20051231")] # begins after window + picked = pick_station_rows(rows, {"KORD", "KMSY"}) + assert set(picked) == {"KORD"} + assert picked["KORD"]["BEGIN"] == "20000101" + + +def test_pick_prefers_latest_end_when_multiple_rows_cover(): + rows = [_row("KORD", "19730101", "20020101"), + _row("KORD", "20000101", "20251231")] + assert pick_station_rows(rows, {"KORD"})["KORD"]["END"] == "20251231" + + +def test_pick_ignores_icaos_not_in_curated_set(): + assert pick_station_rows([_row("KJFK", "19700101", "20251231")], {"KORD"}) == {} + + +def test_station_record_shapes_and_types(): + rec = station_record(_row("KORD", "20000101", "20251231"), tz="America/Chicago") + assert rec == {"station_id": "KORD", "name": "CHICAGO O'HARE", + "lat": 41.995, "lon": -87.934, "elevation_m": 200.6, + "country": "US", "tz": "America/Chicago", "isd_id": "725300-94846"} + + +def test_station_record_missing_elevation_becomes_none(): + rec = station_record(_row("KORD", "20000101", "20251231", elev=""), + tz="America/Chicago") + assert rec["elevation_m"] is None From a3e16f80936132a850482478f7bd505440a23caf Mon Sep 17 00:00:00 2001 From: Robbie Byrd Date: Sun, 12 Jul 2026 19:46:15 +0000 Subject: [PATCH 4/9] fix(weather-recon): substitute KFMY for KRSW (2001-09 coverage exists) Task review caught that the KRSW drop was based on a wrong data read: data/isd-history.csv row 722106-12835 "PAGE FIELD AIRPORT" (KFMY, 26.585,-81.861) covers 19430101-20250826, i.e. full 2001-09 coverage at Fort Myers. The original grep for "FORT MYERS" only matched the retired 999999-12835 row whose name includes the city; the covering row's name field ("PAGE FIELD AIRPORT") does not, so it was missed. Adds KFMY to CURATED_ICAOS as the KRSW substitute (like CYQB->CWQB / KMDT->KCXY), regenerates data/stations.csv (187 -> 188 stations: US 153, CA 17, MX 18), and updates the stale curated-list header comment to the real counts. Co-Authored-By: Claude Fable 5 --- packages/tools/weather-recon/data/stations.csv | 1 + packages/tools/weather-recon/scripts/build_stations.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/tools/weather-recon/data/stations.csv b/packages/tools/weather-recon/data/stations.csv index 840a407f..7b25126e 100644 --- a/packages/tools/weather-recon/data/stations.csv +++ b/packages/tools/weather-recon/data/stations.csv @@ -68,6 +68,7 @@ KEYW,KEY WEST INTERNATIONAL AIRPORT,24.557,-81.755,0.3,US,America/New_York,72201 KFAR,HECTOR INTERNATIONAL AIRPORT,46.924,-96.812,272.8,US,America/Chicago,727530-14914 KFAT,FRESNO YOSEMITE INTERNATIONAL AIRPORT,36.78,-119.72,101.9,US,America/Los_Angeles,723890-93193 KFLG,FLAGSTAFF AIRPORT,35.133,-111.667,2135.0,US,America/Phoenix,723750-99999 +KFMY,PAGE FIELD AIRPORT,26.585,-81.861,3.8,US,America/New_York,722106-12835 KFSD,JOE FOSS FIELD AIRPORT,43.578,-96.754,432.4,US,America/Chicago,726510-14944 KFSM,FORT SMITH REGIONAL AIRPORT,35.333,-94.365,136.7,US,America/Chicago,723440-13964 KFWA,FORT WAYNE INTERNATIONAL AIRPORT,40.972,-85.206,243.0,US,America/Indiana/Indianapolis,725330-14827 diff --git a/packages/tools/weather-recon/scripts/build_stations.py b/packages/tools/weather-recon/scripts/build_stations.py index e5f9636b..1f0dc8e1 100644 --- a/packages/tools/weather-recon/scripts/build_stations.py +++ b/packages/tools/weather-recon/scripts/build_stations.py @@ -22,7 +22,7 @@ FIELDS = ["station_id", "name", "lat", "lon", "elevation_m", "country", "tz", "isd_id"] # Curated station list: major-city METAR sites across the US, Canada and -# Mexico with 2001 coverage. ~175 sites; every US state, ~13 CA / ~15 MX. +# Mexico with 2001 coverage. 188 sites; every US state, 17 CA / 18 MX. CURATED_ICAOS = { # US — Northeast "KBOS", "KPWM", "KBGR", "KBTV", "KMHT", "KPVD", "KBDL", "KALB", "KSYR", @@ -31,6 +31,7 @@ # US — Southeast "KRIC", "KORF", "KRDU", "KGSO", "KCLT", "KCAE", "KCHS", "KSAV", "KATL", "KTLH", "KJAX", "KMCO", "KTPA", "KMIA", "KPBI", "KEYW", + "KFMY", # KRSW substitute: Fort Myers Page Field (KRSW has no 2001-09 rows) "KBHM", "KMGM", "KMOB", "KHSV", "KBNA", "KMEM", "KTYS", "KCHA", "KSDF", "KLEX", "KCRW", "KJAN", # US — Midwest From f383be9bf76d2dbb8ff78cf770206f254581eea5 Mon Sep 17 00:00:00 2001 From: Robbie Byrd Date: Sun, 12 Jul 2026 19:49:56 +0000 Subject: [PATCH 5/9] feat(weather-recon): stations.csv loader with validation TDD implementation of load_stations() function that validates and parses stations.csv into typed dicts. Includes 10 test cases covering valid rows, type conversion, empty elevation handling, and comprehensive error validation (lat/lon ranges, country codes, required fields, duplicates). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01W179bqEPFP92a3JrsPkD5c --- .../weather-recon/tests/test_stations.py | 52 +++++++++++++++++++ .../weather-recon/weather_recon/stations.py | 42 +++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 packages/tools/weather-recon/tests/test_stations.py create mode 100644 packages/tools/weather-recon/weather_recon/stations.py diff --git a/packages/tools/weather-recon/tests/test_stations.py b/packages/tools/weather-recon/tests/test_stations.py new file mode 100644 index 00000000..08615aad --- /dev/null +++ b/packages/tools/weather-recon/tests/test_stations.py @@ -0,0 +1,52 @@ +import pytest + +from weather_recon.stations import load_stations + +HEADER = "station_id,name,lat,lon,elevation_m,country,tz,isd_id\n" +GOOD = 'KORD,CHICAGO OHARE,41.995,-87.934,200.6,US,America/Chicago,725300-94846\n' + + +def _write(tmp_path, body): + p = tmp_path / "stations.csv" + p.write_text(HEADER + body, encoding="utf-8") + return p + + +def test_loads_and_types_a_valid_row(tmp_path): + rows = load_stations(_write(tmp_path, GOOD)) + assert rows == [{"station_id": "KORD", "name": "CHICAGO OHARE", "lat": 41.995, + "lon": -87.934, "elevation_m": 200.6, "country": "US", + "tz": "America/Chicago", "isd_id": "725300-94846"}] + + +def test_empty_elevation_is_none(tmp_path): + rows = load_stations(_write( + tmp_path, 'CYYZ,TORONTO PEARSON,43.68,-79.63,,CA,America/Toronto,712650-99999\n')) + assert rows[0]["elevation_m"] is None + + +@pytest.mark.parametrize("bad,msg", [ + ('KORD,X,91.0,-87.9,1,US,America/Chicago,1-2\n', "lat"), + ('KORD,X,41.9,-187.9,1,US,America/Chicago,1-2\n', "lon"), + ('KORD,X,41.9,-87.9,1,FR,Europe/Paris,1-2\n', "country"), + ('KORD,X,41.9,-87.9,1,US,,1-2\n', "tz"), + ('KORD,X,41.9,-87.9,1,US,America/Chicago,\n', "isd_id"), + (GOOD + GOOD, "duplicate"), +]) +def test_rejects_invalid_rows(tmp_path, bad, msg): + with pytest.raises(ValueError, match=msg): + load_stations(_write(tmp_path, bad)) + + +def test_rejects_missing_columns(tmp_path): + p = tmp_path / "stations.csv" + p.write_text("station_id,name\nKORD,X\n", encoding="utf-8") + with pytest.raises(ValueError, match="column"): + load_stations(p) + + +def test_committed_csv_is_valid(): + from pathlib import Path + rows = load_stations(Path(__file__).resolve().parents[1] / "data" / "stations.csv") + assert len(rows) >= 150 + assert {r["country"] for r in rows} == {"US", "CA", "MX"} diff --git a/packages/tools/weather-recon/weather_recon/stations.py b/packages/tools/weather-recon/weather_recon/stations.py new file mode 100644 index 00000000..a10b3dab --- /dev/null +++ b/packages/tools/weather-recon/weather_recon/stations.py @@ -0,0 +1,42 @@ +"""Load and validate the curated station reference CSV (data/stations.csv).""" + +import csv + +COLUMNS = ["station_id", "name", "lat", "lon", "elevation_m", "country", "tz", "isd_id"] +COUNTRIES = {"US", "CA", "MX"} + + +def load_stations(path): + """Parse stations.csv into typed dicts; raise ValueError on any bad row.""" + with open(path, newline="", encoding="utf-8") as f: + reader = csv.DictReader(f) + if reader.fieldnames != COLUMNS: + raise ValueError(f"{path}: expected columns {COLUMNS}, got {reader.fieldnames}") + rows, seen = [], set() + for i, raw in enumerate(reader, start=2): + where = f"{path}:{i} ({raw.get('station_id')!r})" + sid = (raw["station_id"] or "").strip() + if not sid: + raise ValueError(f"{where}: empty station_id") + if sid in seen: + raise ValueError(f"{where}: duplicate station_id") + seen.add(sid) + lat, lon = float(raw["lat"]), float(raw["lon"]) + if not -90 <= lat <= 90: + raise ValueError(f"{where}: lat {lat} out of range") + if not -180 <= lon <= 180: + raise ValueError(f"{where}: lon {lon} out of range") + if raw["country"] not in COUNTRIES: + raise ValueError(f"{where}: country {raw['country']!r} not in {COUNTRIES}") + for req in ("name", "tz", "isd_id"): + if not (raw[req] or "").strip(): + raise ValueError(f"{where}: empty {req}") + elev = (raw["elevation_m"] or "").strip() + rows.append({"station_id": sid, "name": raw["name"].strip(), + "lat": lat, "lon": lon, + "elevation_m": float(elev) if elev else None, + "country": raw["country"], "tz": raw["tz"].strip(), + "isd_id": raw["isd_id"].strip()}) + if not rows: + raise ValueError(f"{path}: no station rows") + return rows From 58cb0523386a92a6343671410edd3be3bb527c19 Mon Sep 17 00:00:00 2001 From: Robbie Byrd Date: Sun, 12 Jul 2026 19:53:43 +0000 Subject: [PATCH 6/9] feat(weather-recon): Directus client + weather collection specs Implement DirectusClient with weather collections (weather_stations, weather_observations, weather_forecasts). Ported from flight_recon with three changes: new module docstring, weather-specific COLLECTIONS dict, and delete_all method (vs flight_date windowed delete_window). Co-Authored-By: Claude Fable 5 --- .../weather-recon/tests/test_schema_fields.py | 44 ++++ .../weather-recon/weather_recon/directus.py | 194 ++++++++++++++++++ 2 files changed, 238 insertions(+) create mode 100644 packages/tools/weather-recon/tests/test_schema_fields.py create mode 100644 packages/tools/weather-recon/weather_recon/directus.py diff --git a/packages/tools/weather-recon/tests/test_schema_fields.py b/packages/tools/weather-recon/tests/test_schema_fields.py new file mode 100644 index 00000000..1ddfa7f1 --- /dev/null +++ b/packages/tools/weather-recon/tests/test_schema_fields.py @@ -0,0 +1,44 @@ +from weather_recon.directus import COLLECTIONS + + +def test_expected_collections(): + assert set(COLLECTIONS) == {"weather_stations", "weather_observations", + "weather_forecasts"} + + +def test_every_collection_has_exactly_one_primary_key(): + for name, spec in COLLECTIONS.items(): + pks = [f["field"] for f in spec["fields"] + if f.get("schema", {}).get("is_primary_key")] + assert len(pks) == 1, f"{name}: PKs {pks}" + + +def test_no_duplicate_field_names(): + for name, spec in COLLECTIONS.items(): + fields = [f["field"] for f in spec["fields"]] + assert len(fields) == len(set(fields)), f"{name}" + + +def test_json_fields_carry_cast_json_special(): + # Repo law (flight_recon/directus.py:37): json fields without cast-json + # fail collection creation with an opaque 400. + for name, spec in COLLECTIONS.items(): + for f in spec["fields"]: + if f["type"] == "json": + assert "cast-json" in f.get("meta", {}).get("special", []), \ + f"{name}.{f['field']}" + + +def test_stations_pk_is_the_icao_string(): + pk = next(f for f in COLLECTIONS["weather_stations"]["fields"] + if f.get("schema", {}).get("is_primary_key")) + assert pk["field"] == "station_id" and pk["type"] == "string" + + +def test_observation_and_forecast_time_fields_are_timestamps(): + obs = {f["field"]: f for f in COLLECTIONS["weather_observations"]["fields"]} + fc = {f["field"]: f for f in COLLECTIONS["weather_forecasts"]["fields"]} + assert obs["observed_at"]["type"] == "timestamp" + assert obs["observed_at"]["schema"] == {"is_nullable": False} + assert fc["issued_at"]["type"] == "timestamp" + assert fc["issued_at"]["schema"] == {"is_nullable": False} diff --git a/packages/tools/weather-recon/weather_recon/directus.py b/packages/tools/weather-recon/weather_recon/directus.py new file mode 100644 index 00000000..ea776758 --- /dev/null +++ b/packages/tools/weather-recon/weather_recon/directus.py @@ -0,0 +1,194 @@ +""" +Directus REST client for the weather pipeline (issue #184). + +Ported from flight_recon.directus (same server, same gotchas): +- json-typed fields MUST carry the cast-json special (none yet in these + collections, but keep _JSON_META for the Phase 2 fields that will). +- insert_many is bounded by row count AND serialized bytes (1 MB payload cap). + +Idempotency: weather_stations is a ~175-row reference table keyed by ICAO — +re-loads do delete_all + insert. Phase 2 flows will add windowed deletes for +observations/forecasts when they land. +""" + +import json +import logging +import time + +import httpx + +log = logging.getLogger(__name__) + +INSERT_CHUNK = 2000 # row-count bound per request +MAX_BATCH_BYTES = 700_000 # size bound: Directus MAX_PAYLOAD_SIZE defaults to 1 MB +DELETE_MAX_PASSES = 50 # QUERY_LIMIT_MAX may cap a bulk delete; loop until empty + +# json-typed fields MUST carry the cast-json special: without it Directus +# rejects the whole collection payload with an opaque 400 ("collection/field +# required"). Discovered empirically against directus:latest, 2026-07-07. +_JSON_META = {"special": ["cast-json"], "interface": "input-code"} + +COLLECTIONS = { + "weather_stations": { + "meta": {"icon": "cloud", "note": "Curated US/CA/MX METAR stations " + "(reference table, ICAO-keyed)", + "accountability": None}, + "fields": [ + {"field": "station_id", "type": "string", + "schema": {"is_primary_key": True}}, + {"field": "name", "type": "string", "schema": {"is_nullable": False}}, + {"field": "lat", "type": "float", "schema": {"is_nullable": False}}, + {"field": "lon", "type": "float", "schema": {"is_nullable": False}}, + {"field": "elevation_m", "type": "float"}, + {"field": "country", "type": "string", "schema": {"is_nullable": False}}, + {"field": "tz", "type": "string", "schema": {"is_nullable": False}}, + {"field": "isd_id", "type": "string", "schema": {"is_nullable": False}}, + ], + }, + "weather_observations": { + "meta": {"icon": "thermostat", "note": "Hourly METAR/ISD surface observations, " + "2001-09-09..12 (loaded by Phase 2)", + "accountability": None}, + "fields": [ + {"field": "id", "type": "integer", + "schema": {"is_primary_key": True, "has_auto_increment": True}}, + {"field": "station_id", "type": "string", "schema": {"is_nullable": False}}, + {"field": "observed_at", "type": "timestamp", + "schema": {"is_nullable": False}}, + {"field": "temp_c", "type": "float"}, + {"field": "dewpoint_c", "type": "float"}, + {"field": "wind_dir_deg", "type": "integer"}, + {"field": "wind_speed_kt", "type": "float"}, + {"field": "gust_kt", "type": "float"}, + {"field": "pressure_hpa", "type": "float"}, + {"field": "sky_condition", "type": "string"}, + {"field": "present_weather", "type": "string"}, + {"field": "visibility_km", "type": "float"}, + {"field": "raw_metar", "type": "text"}, + {"field": "run_id", "type": "string"}, + ], + }, + "weather_forecasts": { + "meta": {"icon": "wb_sunny", "note": "Archived NWS forecast text products " + "(ZFP/AFD), 2001-09 (loaded by Phase 2)", + "accountability": None}, + "fields": [ + {"field": "id", "type": "integer", + "schema": {"is_primary_key": True, "has_auto_increment": True}}, + {"field": "wfo", "type": "string"}, + {"field": "zone", "type": "string"}, + {"field": "product_type", "type": "string"}, + {"field": "issued_at", "type": "timestamp", "schema": {"is_nullable": False}}, + {"field": "raw_text", "type": "text", "schema": {"is_nullable": False}}, + {"field": "run_id", "type": "string"}, + ], + }, +} + + +def _check(r): + """raise_for_status, but keep Directus's error body — it names the invalid + field, which the bare status line never does.""" + if r.status_code >= 400: + raise httpx.HTTPStatusError( + f"Directus {r.request.method} {r.request.url.path} -> " + f"{r.status_code}: {r.text[:500]}", + request=r.request, response=r) + return r + + +class DirectusClient: + def __init__(self, base_url, token, timeout=60.0): + self._http = httpx.Client( + base_url=base_url.rstrip("/"), + headers={"Authorization": f"Bearer {token}"}, + timeout=timeout, + ) + + def close(self): + self._http.close() + + # ------------------------------------------------------------ schema + def collection_exists(self, name): + # Directus answers 403 (not 404) for unknown collections on some + # policies, so treat any non-200 as "missing". + return self._http.get(f"/collections/{name}").status_code == 200 + + def existing_fields(self, collection): + r = self._http.get(f"/fields/{collection}") + _check(r) + return {f["field"] for f in r.json()["data"]} + + def ensure_collection(self, name): + """Create `name` (and any missing fields) per COLLECTIONS. Returns + a list of human-readable actions taken (empty = already in shape).""" + spec = COLLECTIONS[name] + actions = [] + if not self.collection_exists(name): + r = self._http.post("/collections", json={ + "collection": name, + "meta": spec["meta"], + "schema": {}, + "fields": spec["fields"], + }) + _check(r) + actions.append(f"created collection {name} with {len(spec['fields'])} fields") + return actions + have = self.existing_fields(name) + for f in spec["fields"]: + if f["field"] not in have: + r = self._http.post(f"/fields/{name}", json=f) + _check(r) + actions.append(f"added field {name}.{f['field']}") + return actions + + # ------------------------------------------------------------ items + def count(self, collection, flt): + r = self._http.get(f"/items/{collection}", params={ + "aggregate[count]": "*", "filter": json.dumps(flt)}) + _check(r) + return int(r.json()["data"][0]["count"]) + + def delete_all(self, collection): + """Delete every row (reference-table reload). Returns rows deleted.""" + flt = {"station_id": {"_nnull": True}} + total = self.count(collection, flt) + if total == 0: + return 0 + log.warning("deleting all %d rows from %s before reload", total, collection) + for _ in range(DELETE_MAX_PASSES): + r = self._http.request("DELETE", f"/items/{collection}", + json={"query": {"filter": flt, "limit": -1}}) + _check(r) + if self.count(collection, flt) == 0: + return total + time.sleep(0.5) + raise RuntimeError(f"{collection}: rows remain after {DELETE_MAX_PASSES} " + f"delete passes") + + def insert_many(self, collection, rows, chunk=INSERT_CHUNK, max_bytes=MAX_BATCH_BYTES): + """Chunked POST /items/{collection}. Returns rows inserted. + + Batches are bounded by BOTH row count and serialized size — rows with + big json fields (a transcontinental track's geometry is ~6-10 KB) + blow past Directus's 1 MB payload cap long before `chunk` rows.""" + i = 0 + while i < len(rows): + batch, size = [], 2 # brackets + while i < len(rows) and len(batch) < chunk: + row_bytes = len(json.dumps(rows[i], default=str)) + 1 + if batch and size + row_bytes > max_bytes: + break + batch.append(rows[i]) + size += row_bytes + i += 1 + r = self._http.post(f"/items/{collection}", json=batch, + params={"fields": "id"}) + _check(r) + log.info("%s: inserted %d/%d", collection, i, len(rows)) + return len(rows) + + def insert_one(self, collection, row): + r = self._http.post(f"/items/{collection}", json=row) + _check(r) + return r.json()["data"] From 15ae88c5bcdcc48a17d3fc77666b64f800bc19c2 Mon Sep 17 00:00:00 2001 From: Robbie Byrd Date: Sun, 12 Jul 2026 19:57:38 +0000 Subject: [PATCH 7/9] fix(weather-recon): delete_all filters on collection PK, not station_id weather_forecasts has no station_id field; derive the _nnull delete filter from the collection's primary key instead. Co-Authored-By: Claude Fable 5 --- packages/tools/weather-recon/tests/test_schema_fields.py | 9 +++++++++ packages/tools/weather-recon/weather_recon/directus.py | 4 +++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/tools/weather-recon/tests/test_schema_fields.py b/packages/tools/weather-recon/tests/test_schema_fields.py index 1ddfa7f1..cf6770c1 100644 --- a/packages/tools/weather-recon/tests/test_schema_fields.py +++ b/packages/tools/weather-recon/tests/test_schema_fields.py @@ -35,6 +35,15 @@ def test_stations_pk_is_the_icao_string(): assert pk["field"] == "station_id" and pk["type"] == "string" +def test_delete_all_pk_filter_derivable_for_every_collection(): + # delete_all filters on the collection PK; every collection must yield one + from weather_recon.directus import COLLECTIONS + for name, spec in COLLECTIONS.items(): + pk = next(f["field"] for f in spec["fields"] + if f.get("schema", {}).get("is_primary_key")) + assert pk + + def test_observation_and_forecast_time_fields_are_timestamps(): obs = {f["field"]: f for f in COLLECTIONS["weather_observations"]["fields"]} fc = {f["field"]: f for f in COLLECTIONS["weather_forecasts"]["fields"]} diff --git a/packages/tools/weather-recon/weather_recon/directus.py b/packages/tools/weather-recon/weather_recon/directus.py index ea776758..fb100b2a 100644 --- a/packages/tools/weather-recon/weather_recon/directus.py +++ b/packages/tools/weather-recon/weather_recon/directus.py @@ -151,7 +151,9 @@ def count(self, collection, flt): def delete_all(self, collection): """Delete every row (reference-table reload). Returns rows deleted.""" - flt = {"station_id": {"_nnull": True}} + pk = next(f["field"] for f in COLLECTIONS[collection]["fields"] + if f.get("schema", {}).get("is_primary_key")) + flt = {pk: {"_nnull": True}} total = self.count(collection, flt) if total == 0: return 0 From e34bc52cca6737f14f866e53b171ef96503737cc Mon Sep 17 00:00:00 2001 From: Robbie Byrd Date: Sun, 12 Jul 2026 20:00:04 +0000 Subject: [PATCH 8/9] feat(weather-recon): load-weather-stations Prefect flow Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01W179bqEPFP92a3JrsPkD5c --- .../tools/weather-recon/weather_recon/flow.py | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 packages/tools/weather-recon/weather_recon/flow.py diff --git a/packages/tools/weather-recon/weather_recon/flow.py b/packages/tools/weather-recon/weather_recon/flow.py new file mode 100644 index 00000000..e97ff044 --- /dev/null +++ b/packages/tools/weather-recon/weather_recon/flow.py @@ -0,0 +1,82 @@ +""" +Prefect flow: validate the curated station list and load it into Directus, +creating the three weather collections on the way. + +$DIRECTUS_API_TOKEN must be in the environment (weather-recon-secrets k8s +Secret in cluster, kubectl-extracted for local runs — never a parameter, +never logged). directus_url overrides $DIRECTUS_URL. +""" + +import os + +from prefect import flow, get_run_logger, task + +from weather_recon.directus import COLLECTIONS, DirectusClient +from weather_recon.stations import load_stations + +# NB: scalar retry_delay_seconds — list-valued delays 422 on this server (see +# packages/tools/video-grabber/CLAUDE.md). +NETWORK_RETRIES = dict(retries=4, retry_delay_seconds=15, retry_jitter_factor=0.3) + + +def _client(directus_url): + url = directus_url or os.environ["DIRECTUS_URL"] + token = os.environ["DIRECTUS_API_TOKEN"] + return DirectusClient(url, token) + + +@task +def validate_stations(stations_path): + log = get_run_logger() + rows = load_stations(stations_path) + by_country = {c: sum(1 for r in rows if r["country"] == c) for c in ("US", "CA", "MX")} + log.info("stations ok: %d rows (%s)", len(rows), by_country) + return rows + + +@task(**NETWORK_RETRIES) +def ensure_schema(directus_url): + log = get_run_logger() + client = _client(directus_url) + try: + for name in COLLECTIONS: + for action in client.ensure_collection(name): + log.warning("schema change: %s", action) + log.info("schema ensured for %s", list(COLLECTIONS)) + finally: + client.close() + + +@task(**NETWORK_RETRIES) +def load_stations_into_directus(rows, directus_url): + log = get_run_logger() + client = _client(directus_url) + try: + deleted = client.delete_all("weather_stations") + if deleted: + log.warning("reference-table reload: deleted %d existing stations", deleted) + inserted = client.insert_many("weather_stations", rows) + log.info("weather_stations: inserted %d rows", inserted) + return inserted + finally: + client.close() + + +@flow(name="load-weather-stations", log_prints=True) +def load_weather_stations( + stations_path: str = "/app/data/stations.csv", + directus_url: str | None = None, +): + log = get_run_logger() + rows = validate_stations(stations_path) + ensure_schema(directus_url) + n = load_stations_into_directus(rows, directus_url) + log.info("done: %d stations in Directus", n) + return {"stations_loaded": n} + + +if __name__ == "__main__": + # Local/dev execution without a deployment: python -m weather_recon.flow + import sys + path = sys.argv[1] if len(sys.argv) > 1 else "data/stations.csv" + load_weather_stations(stations_path=path) From d6b334b6122360c07e4bf90babf72ca000a6be8c Mon Sep 17 00:00:00 2001 From: Robbie Byrd Date: Sun, 12 Jul 2026 20:04:30 +0000 Subject: [PATCH 9/9] feat(weather-recon): Dockerfile, deploy registration, README Co-Authored-By: Claude Fable 5 --- packages/tools/weather-recon/Dockerfile | 17 ++++++++ packages/tools/weather-recon/README.md | 37 ++++++++++++++++++ .../weather-recon/weather_recon/deploy.py | 39 +++++++++++++++++++ 3 files changed, 93 insertions(+) create mode 100644 packages/tools/weather-recon/Dockerfile create mode 100644 packages/tools/weather-recon/README.md create mode 100644 packages/tools/weather-recon/weather_recon/deploy.py diff --git a/packages/tools/weather-recon/Dockerfile b/packages/tools/weather-recon/Dockerfile new file mode 100644 index 00000000..54749a54 --- /dev/null +++ b/packages/tools/weather-recon/Dockerfile @@ -0,0 +1,17 @@ +# One image serves both roles: the Prefect worker and the flow-run Jobs it spawns. +FROM python:3.12-slim + +ENV PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1 + +WORKDIR /app + +# Deps first (stable layer), pinned to the in-cluster Prefect server version. +RUN pip install "prefect==3.7.4" "prefect-kubernetes>=0.5" "httpx>=0.27" + +COPY pyproject.toml ./ +COPY weather_recon ./weather_recon +COPY data/stations.csv ./data/stations.csv +RUN pip install --no-deps . + +# Flow-run jobs override this; the worker Deployment uses it as-is. +CMD ["prefect", "worker", "start", "--pool", "weather-recon-k8s", "--type", "kubernetes"] diff --git a/packages/tools/weather-recon/README.md b/packages/tools/weather-recon/README.md new file mode 100644 index 00000000..d13b0e0d --- /dev/null +++ b/packages/tools/weather-recon/README.md @@ -0,0 +1,37 @@ +# weather-recon + +Loads September 2001 weather data into Directus for the rt911 Weather app +(issue #184; design spec: `plans/weather-app-design.md`). Phase 1 ships the +curated station reference table and the three collections; observation / +radar / forecast / almanac flows follow in Phase 2. + +## Layout + +- `scripts/build_stations.py` — dev-run: NOAA `isd-history.csv` + curated ICAO + list → `data/stations.csv` (committed). Re-run only to change the curation. +- `weather_recon/flow.py` — `load-weather-stations`: validates the CSV, ensures + the `weather_stations` / `weather_observations` / `weather_forecasts` + collections, reloads `weather_stations` (delete-all + insert; ICAO-keyed). +- `weather_recon/directus.py` — REST client + collection specs (ported from + flight-recon; same 1 MB payload batching and cast-json rules). + +## Running locally (no work pool) + + pip install -e ".[dev]" + export DIRECTUS_URL=... # e.g. port-forwarded rt911-api + export DIRECTUS_API_TOKEN=... # from the rt911 namespace secret + python -m weather_recon.flow data/stations.csv + +## Cluster deployment (mirrors flight-recon; see its k8s/ for templates) + +1. Build + import the image into k3s containerd: + `docker build -t weather-recon:0.1.0 . && docker save weather-recon:0.1.0 | sudo k3s ctr images import -` +2. Infra repo (GitOps): `weather-recon-k8s` work pool, worker Deployment + running this image, and a `weather-recon-secrets` Secret providing + `DIRECTUS_API_TOKEN` — copy the flight-recon manifests. +3. Register the deployment: see `weather_recon/deploy.py` docstring. + +## Tests + + pytest tests/ -v + ruff check weather_recon/ tests/ scripts/ diff --git a/packages/tools/weather-recon/weather_recon/deploy.py b/packages/tools/weather-recon/weather_recon/deploy.py new file mode 100644 index 00000000..aec3c326 --- /dev/null +++ b/packages/tools/weather-recon/weather_recon/deploy.py @@ -0,0 +1,39 @@ +""" +Register the load-weather-stations deployment on the weather-recon-k8s work pool. + +Run from the package root, with PREFECT_API_URL pointing at the in-cluster server: + + PREFECT_API_URL=http://prefect-server.video-grabber.svc.cluster.local:4200/api \ + python -m weather_recon.deploy + +Env knobs: + WEATHER_RECON_IMAGE image ref baked into job pods (default weather-recon:0.1.0) +""" + +import os + +from weather_recon.flow import load_weather_stations + +IMAGE = os.environ.get("WEATHER_RECON_IMAGE", "weather-recon:0.1.0") + +if __name__ == "__main__": + deployment_id = load_weather_stations.deploy( + name="load-weather-stations-k8s", + work_pool_name="weather-recon-k8s", + image=IMAGE, + build=False, # image is built/loaded out-of-band (see README) + push=False, + description="Curated US/CA/MX METAR stations -> Directus weather_stations " + "(+ ensures weather_observations / weather_forecasts collections)", + tags=["weather-recon"], + parameters={"stations_path": "/app/data/stations.csv"}, + job_variables={ + "namespace": "video-grabber", + "image_pull_policy": "IfNotPresent", # image is imported into k3s containerd + "finished_job_ttl": 3600, + "env": { + "DIRECTUS_URL": "http://rt911-api.rt911.svc.cluster.local:8055", + }, + }, + ) + print(f"registered deployment {deployment_id}")