Regenerate data and add frictionless validate to CI

.github/workflows/actions.yml

@@ -18,6 +18,10 @@ jobs:
             python-version: "3.12"
         - name: Run
           run: make
+        - name: Validate datapackage
+          run: |
+            pip install frictionless
+            frictionless validate datapackage.json
         - name: Commit and Push
           run: |
             git config --global user.name "GitHub Action"

Makefile

@@ -14,7 +14,7 @@ $(VENV)/bin/activate: scripts/requirements.txt
 	$(PIP) install -r scripts/requirements.txt
 
 validate:
-	$(PYTHON) -m frictionless validate data/constituents.csv
+	$(PYTHON) -m frictionless validate datapackage.json
 
 clean:
 	rm -rf __pycache__

data/type-counts.csv

@@ -1,7 +1,7 @@
 type,count
-Large Airport,1194
-Medium Airport,4067
-Small Airport,42582
-Heliport,22726
-Seaplane Base,1255
+Large Airport,1178
+Medium Airport,4099
+Small Airport,42669
+Heliport,23069
+Seaplane Base,1263
 Balloonport,61

scripts/process.py

@@ -1,4 +1,3 @@
-import ssl
 import csv
 import copy
 import requests
@@ -7,7 +6,7 @@
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 archive = "archive/data.csv"
-source = "https://ourairports.com/data/airports.csv"
+source = "https://davidmegginson.github.io/ourairports-data/airports.csv"
 
 
 def download():
@@ -18,24 +17,26 @@ def download():
 
 
 def process():
-    with open(archive, "r") as source:
-        reader = csv.DictReader(source)
-        with open("data/airport-codes.csv", "w") as result:
-            fieldnames = [
-                "ident",
-                "type",
-                "name",
-                "elevation_ft",
-                "continent",
-                "iso_country",
-                "iso_region",
-                "municipality",
-                "icao_code",
-                "iata_code",
-                "gps_code",
-                "local_code",
-                "coordinates",
-            ]
+    with open(archive, "r", encoding="utf-8") as src:
+        reader = csv.DictReader(src)
+        fieldnames = [
+            "ident",
+            "type",
+            "name",
+            "elevation_ft",
+            "continent",
+            "iso_country",
+            "iso_region",
+            "municipality",
+            "icao_code",
+            "iata_code",
+            "gps_code",
+            "local_code",
+            "coordinates",
+        ]
+        rows = []
+        type_counts = {}
+        with open("data/airport-codes.csv", "w", newline="\n", encoding="utf-8") as result:
             writer = csv.DictWriter(
                 result, fieldnames=fieldnames, extrasaction="ignore"
             )
@@ -46,8 +47,25 @@ def process():
                     row["latitude_deg"], row["longitude_deg"]
                 )
                 writer.writerow(new_row)
+                t = row["type"]
+                if t != "closed":
+                    type_counts[t] = type_counts.get(t, 0) + 1
 
+    type_label = {
+        "large_airport": "Large Airport",
+        "medium_airport": "Medium Airport",
+        "small_airport": "Small Airport",
+        "heliport": "Heliport",
+        "seaplane_base": "Seaplane Base",
+        "balloonport": "Balloonport",
+    }
+    with open("data/type-counts.csv", "w", newline="\n", encoding="utf-8") as tc:
+        writer = csv.DictWriter(tc, fieldnames=["type", "count"])
+        writer.writeheader()
+        for key in ["large_airport", "medium_airport", "small_airport", "heliport", "seaplane_base", "balloonport"]:
+            if key in type_counts:
+                writer.writerow({"type": type_label[key], "count": type_counts[key]})
 
-download()
 
+download()
 process()