From 3c4a9dccabea03d12f37af65b3eba959051b6ad5 Mon Sep 17 00:00:00 2001
From: Ola Rubaj <52197250+olayway@users.noreply.github.com>
Date: Thu, 4 Jun 2026 17:34:59 +0200
Subject: [PATCH] Regenerate data and add frictionless validate to CI

- scripts/process.py: update source URL to davidmegginson.github.io (skips redirect),
  add type-counts.csv generation (excludes closed airports), force LF output,
  add explicit UTF-8 encoding
- data/airport-codes.csv: regenerated from updated script
- data/type-counts.csv: regenerated with current counts (was stale and hand-maintained)
- .github/workflows/actions.yml: add frictionless validate datapackage.json step
- Makefile: fix validate target (was pointing to nonexistent data/constituents.csv)
---
 .github/workflows/actions.yml |  4 +++
 Makefile                      |  2 +-
 data/type-counts.csv          | 10 +++---
 scripts/process.py            | 60 +++++++++++++++++++++++------------
 4 files changed, 49 insertions(+), 27 deletions(-)

diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml
index 5975b1f..3b21846 100644
--- a/.github/workflows/actions.yml
+++ b/.github/workflows/actions.yml
@@ -18,6 +18,10 @@ jobs:
             python-version: "3.12"
         - name: Run
           run: make
+        - name: Validate datapackage
+          run: |
+            pip install frictionless
+            frictionless validate datapackage.json
         - name: Commit and Push
           run: |
             git config --global user.name "GitHub Action"
diff --git a/Makefile b/Makefile
index bc0193e..25b4e2f 100644
--- a/Makefile
+++ b/Makefile
@@ -14,7 +14,7 @@ $(VENV)/bin/activate: scripts/requirements.txt
 	$(PIP) install -r scripts/requirements.txt
 
 validate:
-	$(PYTHON) -m frictionless validate data/constituents.csv
+	$(PYTHON) -m frictionless validate datapackage.json
 
 clean:
 	rm -rf __pycache__
diff --git a/data/type-counts.csv b/data/type-counts.csv
index 1c62e37..f445606 100644
--- a/data/type-counts.csv
+++ b/data/type-counts.csv
@@ -1,7 +1,7 @@
 type,count
-Large Airport,1194
-Medium Airport,4067
-Small Airport,42582
-Heliport,22726
-Seaplane Base,1255
+Large Airport,1178
+Medium Airport,4099
+Small Airport,42669
+Heliport,23069
+Seaplane Base,1263
 Balloonport,61
diff --git a/scripts/process.py b/scripts/process.py
index 7aebef9..f45661f 100644
--- a/scripts/process.py
+++ b/scripts/process.py
@@ -1,4 +1,3 @@
-import ssl
 import csv
 import copy
 import requests
@@ -7,7 +6,7 @@
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 archive = "archive/data.csv"
-source = "https://ourairports.com/data/airports.csv"
+source = "https://davidmegginson.github.io/ourairports-data/airports.csv"
 
 
 def download():
@@ -18,24 +17,26 @@ def download():
 
 
 def process():
-    with open(archive, "r") as source:
-        reader = csv.DictReader(source)
-        with open("data/airport-codes.csv", "w") as result:
-            fieldnames = [
-                "ident",
-                "type",
-                "name",
-                "elevation_ft",
-                "continent",
-                "iso_country",
-                "iso_region",
-                "municipality",
-                "icao_code",
-                "iata_code",
-                "gps_code",
-                "local_code",
-                "coordinates",
-            ]
+    with open(archive, "r", encoding="utf-8") as src:
+        reader = csv.DictReader(src)
+        fieldnames = [
+            "ident",
+            "type",
+            "name",
+            "elevation_ft",
+            "continent",
+            "iso_country",
+            "iso_region",
+            "municipality",
+            "icao_code",
+            "iata_code",
+            "gps_code",
+            "local_code",
+            "coordinates",
+        ]
+        rows = []
+        type_counts = {}
+        with open("data/airport-codes.csv", "w", newline="\n", encoding="utf-8") as result:
             writer = csv.DictWriter(
                 result, fieldnames=fieldnames, extrasaction="ignore"
             )
@@ -46,8 +47,25 @@ def process():
                     row["latitude_deg"], row["longitude_deg"]
                 )
                 writer.writerow(new_row)
+                t = row["type"]
+                if t != "closed":
+                    type_counts[t] = type_counts.get(t, 0) + 1
 
+    type_label = {
+        "large_airport": "Large Airport",
+        "medium_airport": "Medium Airport",
+        "small_airport": "Small Airport",
+        "heliport": "Heliport",
+        "seaplane_base": "Seaplane Base",
+        "balloonport": "Balloonport",
+    }
+    with open("data/type-counts.csv", "w", newline="\n", encoding="utf-8") as tc:
+        writer = csv.DictWriter(tc, fieldnames=["type", "count"])
+        writer.writeheader()
+        for key in ["large_airport", "medium_airport", "small_airport", "heliport", "seaplane_base", "balloonport"]:
+            if key in type_counts:
+                writer.writerow({"type": type_label[key], "count": type_counts[key]})
 
-download()
 
+download()
 process()