mirror of
https://github.com/PlaneQuery/OpenAirframes.git
synced 2026-06-19 03:00:05 +02:00
Compare commits
13 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 ggman12
|
921cbefb6e | ||
|
Clément Wehrung
|
b1dd01c27e | ||
|
Clément Wehrung
|
2282e1197f | ||
|
Clément Wehrung
|
ea9c095f91 | ||
|
Clément Wehrung
|
4eb2b9ce0b | ||
|
Clément Wehrung
|
23ef72100f | ||
|
Clément Wehrung
|
bfb22670ba | ||
|
Clem
|
c7a3d9e056 | ||
|
Clément Wehrung
|
0d683d3510 | ||
|
Clem
|
8f11a1d05a | ||
|
ggman12
|
85a3db4dd0 | ||
|
ggman12
|
42f95208c6 | ||
|
ggman12
|
16a0a5fec8 |
+13
-7
@@ -1,4 +1,4 @@
|
||||
name: FAA daily snapshot + release
|
||||
name: planequery-aircraft Daily Release
|
||||
|
||||
on:
|
||||
schedule:
|
||||
@@ -29,19 +29,25 @@ jobs:
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
|
||||
- name: Run snapshot script
|
||||
- name: Run daily release script
|
||||
run: |
|
||||
python src/snapshot_faa.py
|
||||
python src/create_daily_planequery_aircraft_release.py
|
||||
ls -lah data/faa_releasable
|
||||
ls -lah data/planequery_aircraft
|
||||
|
||||
- name: Prepare release metadata
|
||||
id: meta
|
||||
run: |
|
||||
DATE=$(date -u +"%Y-%m-%d")
|
||||
TAG="faa-${DATE}"
|
||||
TAG="planequery-aircraft-${DATE}"
|
||||
# Find the CSV file in data/planequery_aircraft matching the pattern
|
||||
CSV_FILE=$(ls data/planequery_aircraft/planequery_aircraft_*_${DATE}.csv | head -1)
|
||||
CSV_BASENAME=$(basename "$CSV_FILE")
|
||||
echo "date=$DATE" >> "$GITHUB_OUTPUT"
|
||||
echo "tag=$TAG" >> "$GITHUB_OUTPUT"
|
||||
echo "name=FAA ReleasableAircraft snapshot ($DATE)" >> "$GITHUB_OUTPUT"
|
||||
echo "csv_file=$CSV_FILE" >> "$GITHUB_OUTPUT"
|
||||
echo "csv_basename=$CSV_BASENAME" >> "$GITHUB_OUTPUT"
|
||||
echo "name=planequery-aircraft snapshot ($DATE)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Create GitHub Release and upload assets
|
||||
uses: softprops/action-gh-release@v2
|
||||
@@ -52,10 +58,10 @@ jobs:
|
||||
Automated daily snapshot generated at 06:00 UTC for ${{ steps.meta.outputs.date }}.
|
||||
|
||||
Assets:
|
||||
- ReleasableAircraft_${{ steps.meta.outputs.date }}.csv
|
||||
- ${{ steps.meta.outputs.csv_basename }}
|
||||
- ReleasableAircraft_${{ steps.meta.outputs.date }}.zip
|
||||
files: |
|
||||
data/faa_releasable/ReleasableAircraft_${{ steps.meta.outputs.date }}.csv
|
||||
${{ steps.meta.outputs.csv_file }}
|
||||
data/faa_releasable/ReleasableAircraft_${{ steps.meta.outputs.date }}.zip
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -0,0 +1,171 @@
|
||||
name: Process Historical FAA Data
|
||||
|
||||
on:
|
||||
workflow_dispatch: # Manual trigger
|
||||
|
||||
jobs:
|
||||
generate-matrix:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
matrix: ${{ steps.set-matrix.outputs.matrix }}
|
||||
steps:
|
||||
- name: Generate date ranges
|
||||
id: set-matrix
|
||||
run: |
|
||||
python3 << 'EOF'
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
start = datetime(2023, 8, 16)
|
||||
end = datetime(2026, 1, 1)
|
||||
|
||||
ranges = []
|
||||
current = start
|
||||
|
||||
# Process in 4-day chunks
|
||||
while current < end:
|
||||
chunk_end = current + timedelta(days=4)
|
||||
# Don't go past the end date
|
||||
if chunk_end > end:
|
||||
chunk_end = end
|
||||
|
||||
ranges.append({
|
||||
"since": current.strftime("%Y-%m-%d"),
|
||||
"until": chunk_end.strftime("%Y-%m-%d")
|
||||
})
|
||||
|
||||
current = chunk_end
|
||||
|
||||
print(f"::set-output name=matrix::{json.dumps(ranges)}")
|
||||
EOF
|
||||
|
||||
clone-faa-repo:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Cache FAA repository
|
||||
id: cache-faa-repo
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: data/scrape-faa-releasable-aircraft
|
||||
key: faa-repo-v1
|
||||
|
||||
- name: Clone FAA repository
|
||||
if: steps.cache-faa-repo.outputs.cache-hit != 'true'
|
||||
run: |
|
||||
mkdir -p data
|
||||
git clone https://github.com/simonw/scrape-faa-releasable-aircraft data/scrape-faa-releasable-aircraft
|
||||
echo "Repository cloned successfully"
|
||||
|
||||
process-chunk:
|
||||
needs: [generate-matrix, clone-faa-repo]
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
max-parallel: 5 # Process 5 chunks at a time
|
||||
matrix:
|
||||
range: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Restore FAA repository cache
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: data/scrape-faa-releasable-aircraft
|
||||
key: faa-repo-v1
|
||||
fail-on-cache-miss: true
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
|
||||
- name: Process chunk ${{ matrix.range.since }} to ${{ matrix.range.until }}
|
||||
run: |
|
||||
python src/get_historical_faa.py "${{ matrix.range.since }}" "${{ matrix.range.until }}"
|
||||
|
||||
- name: Upload CSV artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: csv-${{ matrix.range.since }}-to-${{ matrix.range.until }}
|
||||
path: data/faa_releasable_historical/*.csv
|
||||
retention-days: 1
|
||||
|
||||
create-release:
|
||||
needs: process-chunk
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: artifacts
|
||||
|
||||
- name: Prepare release files
|
||||
run: |
|
||||
mkdir -p release-files
|
||||
find artifacts -name "*.csv" -exec cp {} release-files/ \;
|
||||
ls -lh release-files/
|
||||
|
||||
- name: Create Release
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
tag_name: historical-faa-${{ github.run_number }}
|
||||
name: Historical FAA Data Release ${{ github.run_number }}
|
||||
body: |
|
||||
Automated release of historical FAA aircraft data
|
||||
Processing period: 2023-08-16 to 2026-01-01
|
||||
Generated: ${{ github.event.repository.updated_at }}
|
||||
files: release-files/*.csv
|
||||
draft: false
|
||||
prerelease: false
|
||||
|
||||
concatenate-and-release:
|
||||
needs: process-chunk
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: artifacts
|
||||
|
||||
- name: Prepare CSVs for concatenation
|
||||
run: |
|
||||
mkdir -p data/faa_releasable_historical
|
||||
find artifacts -name "*.csv" -exec cp {} data/faa_releasable_historical/ \;
|
||||
ls -lh data/faa_releasable_historical/
|
||||
|
||||
- name: Concatenate all CSVs
|
||||
run: |
|
||||
python scripts/concat_csvs.py
|
||||
|
||||
- name: Create Combined Release
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
tag_name: historical-faa-combined-${{ github.run_number }}
|
||||
name: Historical FAA Data Combined Release ${{ github.run_number }}
|
||||
body: |
|
||||
Combined historical FAA aircraft data (all chunks concatenated)
|
||||
Processing period: 2023-08-16 to 2026-01-01
|
||||
Generated: ${{ github.event.repository.updated_at }}
|
||||
files: data/planequery_aircraft/*.csv
|
||||
draft: false
|
||||
prerelease: false
|
||||
@@ -1 +1 @@
|
||||
Downloads [`https://registry.faa.gov/database/ReleasableAircraft.zip`](https://registry.faa.gov/database/ReleasableAircraft.zip). Creates a daily GitHub Release at 06:00 UTC containing the unaltered `ReleasableAircraft.zip` and a derived CSV file with all data. The FAA database updates daily at 05:30 UTC.
|
||||
Downloads [`https://registry.faa.gov/database/ReleasableAircraft.zip`](https://registry.faa.gov/database/ReleasableAircraft.zip). Creates a daily GitHub Release at 06:00 UTC containing the unaltered `ReleasableAircraft.zip` and a derived CSV file with all data from FAA database since 2023-08-16. The FAA database updates daily at 05:30 UTC.
|
||||
@@ -0,0 +1 @@
|
||||
ecosystem.config.cjs
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Clem
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
Executable
+229
@@ -0,0 +1,229 @@
|
||||
# ✈️ AF-KLM Fleet Catalog
|
||||
|
||||
Open source, community-maintained catalog of **Air France** and **KLM** fleets with real-time tracking of aircraft properties, WiFi connectivity, and historical changes.
|
||||
|
||||
---
|
||||
|
||||
## 📊 Fleet Overview
|
||||
|
||||
| Airline | Total | 📶 WiFi | 🛜 High-Speed | % Starlink |
|
||||
|---------|-------|---------|---------------|------------|
|
||||
| 🇫🇷 Air France | 220 | 220 (100%) | 41 | **19%** |
|
||||
| 🇳🇱 KLM | 117 | 94 (80%) | 0 | **0%** |
|
||||
| **Combined** | **337** | **314 (93%)** | **41** | **12%** |
|
||||
|
||||
|
||||
> 🛜 **High-Speed** = Starlink satellite internet (50+ Mbps)
|
||||
> 📶 **WiFi** = Any WiFi connectivity (low-speed or high-speed)
|
||||
|
||||
*Last updated: 2026-02-04*
|
||||
|
||||
---
|
||||
|
||||
## 🛫 Fleet Breakdown
|
||||
|
||||
### 🇫🇷 Air France (AF)
|
||||
|
||||
| Aircraft Type | Count |
|
||||
|---------------|-------|
|
||||
| A220-300 PASSENGER | 46 |
|
||||
| 777-300ER | 43 |
|
||||
| A350-900 | 41 |
|
||||
| A320 | 29 |
|
||||
| 777-200-200ER | 18 |
|
||||
| A321 | 12 |
|
||||
| 787-9 | 10 |
|
||||
| A330-200 | 8 |
|
||||
| A320 (SHARKLETS) | 6 |
|
||||
| A318 | 4 |
|
||||
| A319 | 3 |
|
||||
| **Total** | **220** |
|
||||
|
||||
### 🇳🇱 KLM (KL)
|
||||
|
||||
| Aircraft Type | Count |
|
||||
|---------------|-------|
|
||||
| 737-800 | 29 |
|
||||
| 777-300ER | 15 |
|
||||
| 787-10 | 15 |
|
||||
| 777-200-200ER | 14 |
|
||||
| A321NEO | 12 |
|
||||
| 787-9 | 12 |
|
||||
| A330-200 | 5 |
|
||||
| A330-300 | 5 |
|
||||
| 737-900 | 5 |
|
||||
| 737-700 | 5 |
|
||||
| **Total** | **117** |
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
## 📋 Detailed Configuration
|
||||
|
||||
### 🇫🇷 Air France — Detailed Configuration
|
||||
|
||||
| Aircraft | Config | Seats | Count | 🛜 Starlink |
|
||||
|----------|--------|-------|-------|-------------|
|
||||
| 777-200-200ER | `J028W032Y268` | 328 | 18 | - |
|
||||
| 777-300ER | `J014W028Y430` | 472 | 12 | - |
|
||||
| 777-300ER | `J048W048Y273` | 369 | 8 | - |
|
||||
| 777-300ER | `P004J058W028Y206` | 296 | 14 | 1/14 (7%) |
|
||||
| 777-300ER | `P004J060W044Y204` | 312 | 9 | 1/9 (11%) |
|
||||
| 787-9 | `J030W021Y228` | 279 | 10 | - |
|
||||
| A220-300 PASSENGER | `Y148` | 148 | 46 | 12/46 (26%) |
|
||||
| A318 | `Y131` | 131 | 4 | - |
|
||||
| A319 | `C072Y071` | 143 | 2 | - |
|
||||
| A319 | `Y142` | 142 | 1 | - |
|
||||
| A320 | `C108Y066` | 174 | 22 | 2/22 (9%) |
|
||||
| A320 | `Y178` | 178 | 7 | - |
|
||||
| A320 (SHARKLETS) | `C108Y066` | 174 | 6 | - |
|
||||
| A321 | `C082Y130` | 212 | 8 | - |
|
||||
| A321 | `Y212` | 212 | 4 | - |
|
||||
| A330-200 | `J036W021Y167` | 224 | 8 | 1/8 (13%) |
|
||||
| A350-900 | `J034W024Y266` | 324 | 20 | 10/20 (50%) |
|
||||
| A350-900 | `J048W032Y210` | 290 | 1 | 1/1 (100%) |
|
||||
| A350-900 | `J048W032Y212` | 292 | 20 | 13/20 (65%) |
|
||||
|
||||
### 🇳🇱 KLM — Detailed Configuration
|
||||
|
||||
| Aircraft | Config | Seats | Count | 🛜 Starlink |
|
||||
|----------|--------|-------|-------|-------------|
|
||||
| 737-700 | `C036M106` | 142 | 5 | - |
|
||||
| 737-800 | `C036M150` | 186 | 29 | - |
|
||||
| 737-900 | `C056M132` | 188 | 5 | - |
|
||||
| 777-200-200ER | `C035W024M229` | 288 | 12 | - |
|
||||
| 777-200-200ER | `C035W032M219` | 286 | 2 | - |
|
||||
| 777-300ER | `C035W024M322` | 381 | 15 | - |
|
||||
| 787-10 | `C038W028M252` | 318 | 15 | - |
|
||||
| 787-9 | `C030W021M224` | 275 | 12 | - |
|
||||
| A321NEO | `C030M197` | 227 | 12 | - |
|
||||
| A330-200 | `C018M246` | 264 | 5 | - |
|
||||
| A330-300 | `C030M262` | 292 | 5 | - |
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Quick Start
|
||||
|
||||
### Update the Catalog
|
||||
|
||||
```bash
|
||||
# Set your API key
|
||||
export AFKLM_API_KEY=your_api_key_here
|
||||
|
||||
# Update Air France
|
||||
node fleet-update.js --airline AF
|
||||
|
||||
# Update KLM
|
||||
node fleet-update.js --airline KL
|
||||
|
||||
# Preview changes without saving
|
||||
node fleet-update.js --airline KL --dry-run
|
||||
|
||||
# Regenerate this README with latest stats
|
||||
node generate-readme.js
|
||||
```
|
||||
|
||||
### Using the Data
|
||||
|
||||
```javascript
|
||||
// Load Air France fleet
|
||||
const response = await fetch('https://raw.githubusercontent.com/.../airlines/AF.json');
|
||||
const fleet = await response.json();
|
||||
|
||||
// Find all Starlink aircraft
|
||||
const starlink = fleet.aircraft.filter(a => a.connectivity.wifi === 'high-speed');
|
||||
console.log(`${starlink.length} aircraft with Starlink`);
|
||||
|
||||
// Get aircraft by type
|
||||
const a350s = fleet.aircraft.filter(a => a.aircraft_type.full_name?.includes('A350'));
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📁 Data Structure
|
||||
|
||||
```
|
||||
af-klm/
|
||||
├── airlines/
|
||||
│ ├── AF.json # Air France fleet
|
||||
│ └── KL.json # KLM fleet
|
||||
├── schema/
|
||||
│ └── aircraft.schema.json
|
||||
├── fleet-update.js # Update script
|
||||
└── generate-readme.js # This stats generator
|
||||
```
|
||||
|
||||
### Aircraft Schema
|
||||
|
||||
```json
|
||||
{
|
||||
"registration": "F-HTYA",
|
||||
"aircraft_type": {
|
||||
"iata_code": "359",
|
||||
"manufacturer": "Airbus",
|
||||
"model": "A350",
|
||||
"full_name": "AIRBUS A350-900"
|
||||
},
|
||||
"cabin": {
|
||||
"physical_configuration": "J034W024Y266",
|
||||
"total_seats": 324,
|
||||
"classes": { "business": 34, "premium_economy": 24, "economy": 266 }
|
||||
},
|
||||
"connectivity": {
|
||||
"wifi": "high-speed",
|
||||
"wifi_provider": "Starlink",
|
||||
"satellite": true
|
||||
},
|
||||
"tracking": {
|
||||
"first_seen": "2025-01-15",
|
||||
"last_seen": "2026-02-04",
|
||||
"total_flights": 1250
|
||||
},
|
||||
"history": [
|
||||
{
|
||||
"timestamp": "2026-01-20",
|
||||
"property": "connectivity.wifi",
|
||||
"old_value": "low-speed",
|
||||
"new_value": "high-speed",
|
||||
"source": "airline_api"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🤝 Contributing
|
||||
|
||||
### Daily Updates
|
||||
|
||||
Community members are encouraged to run the update script daily:
|
||||
|
||||
1. Fork this repo
|
||||
2. Set your `AFKLM_API_KEY`
|
||||
3. Run `node fleet-update.js --airline AF` and `--airline KL`
|
||||
4. Run `node generate-readme.js` to update stats
|
||||
5. Submit a PR
|
||||
|
||||
### API Key
|
||||
|
||||
Get a free API key at [developer.airfranceklm.com](https://developer.airfranceklm.com)
|
||||
|
||||
---
|
||||
|
||||
## 📋 Schema Version
|
||||
|
||||
Current: **1.0.0**
|
||||
|
||||
---
|
||||
|
||||
## 📄 License
|
||||
|
||||
Under MIT License
|
||||
|
||||
---
|
||||
|
||||
Made with ✈️ by the aviation community
|
||||
Executable
+9913
File diff suppressed because it is too large
Load Diff
Executable
+5322
File diff suppressed because it is too large
Load Diff
Executable
+116
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Weekly Fleet Update Cron Job
|
||||
*
|
||||
* Updates AF and KL fleet data, regenerates README, and pushes to GitHub.
|
||||
*
|
||||
* Usage:
|
||||
* node cron-update.js # Run once
|
||||
* pm2 start cron-update.js --cron "0 6 * * 0" --no-autorestart # Every Sunday 6am
|
||||
*
|
||||
* Environment:
|
||||
* AFKLM_API_KEY - API key for Air France/KLM API
|
||||
*/
|
||||
|
||||
import { execSync, spawn } from 'child_process';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
function log(msg) {
|
||||
console.log(`[${new Date().toISOString()}] ${msg}`);
|
||||
}
|
||||
|
||||
function exec(cmd) {
|
||||
log(`> ${cmd}`);
|
||||
try {
|
||||
const result = execSync(cmd, { cwd: __dirname, encoding: 'utf-8' });
|
||||
if (result.trim()) console.log(result.trim());
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error(`Error: ${error.stderr || error.message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function runUpdate(airline) {
|
||||
return new Promise((resolve) => {
|
||||
log(`Updating ${airline} fleet...`);
|
||||
|
||||
const child = spawn('node', ['fleet-update.js', '--airline', airline], {
|
||||
cwd: __dirname,
|
||||
env: process.env,
|
||||
stdio: 'inherit',
|
||||
});
|
||||
|
||||
child.on('close', (code) => {
|
||||
if (code === 0) {
|
||||
log(`✅ ${airline} complete`);
|
||||
resolve(true);
|
||||
} else {
|
||||
log(`❌ ${airline} failed (code ${code})`);
|
||||
resolve(false);
|
||||
}
|
||||
});
|
||||
|
||||
child.on('error', (err) => {
|
||||
log(`❌ ${airline} error: ${err.message}`);
|
||||
resolve(false);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
log('🚀 Weekly fleet update starting...\n');
|
||||
|
||||
// Check API key
|
||||
if (!process.env.AFKLM_API_KEY && !process.env.AFKLM_API_KEYS) {
|
||||
log('❌ No API key found. Set AFKLM_API_KEY environment variable.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Update each airline
|
||||
for (const airline of ['AF', 'KL']) {
|
||||
await runUpdate(airline);
|
||||
}
|
||||
|
||||
// Regenerate README
|
||||
log('\n📊 Regenerating README...');
|
||||
exec('node generate-readme.js');
|
||||
|
||||
// Check for changes
|
||||
log('\n📝 Checking for changes...');
|
||||
|
||||
try {
|
||||
const status = execSync('git status --porcelain', { cwd: __dirname, encoding: 'utf-8' });
|
||||
|
||||
if (!status.trim()) {
|
||||
log('✅ No changes to commit');
|
||||
return;
|
||||
}
|
||||
|
||||
log(`Changes:\n${status}`);
|
||||
|
||||
// Git add, commit, push
|
||||
log('\n📤 Pushing to GitHub...');
|
||||
exec('git add -A');
|
||||
|
||||
const date = new Date().toISOString().split('T')[0];
|
||||
exec(`git commit -m "Auto-update fleet data - ${date}"`);
|
||||
exec('git push origin main');
|
||||
|
||||
log('\n✅ Successfully pushed to GitHub!');
|
||||
} catch (error) {
|
||||
log(`Git error: ${error.message}`);
|
||||
}
|
||||
|
||||
log('\n🏁 Done!');
|
||||
}
|
||||
|
||||
main().catch(error => {
|
||||
log(`❌ Fatal error: ${error.message}`);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
+488
@@ -0,0 +1,488 @@
|
||||
# Open Source Airline Fleet Catalog - Schema Proposal
|
||||
|
||||
> **Author:** Clément Wehrung
|
||||
> **Date:** February 4, 2026
|
||||
> **Status:** Draft for Discussion
|
||||
> **Implementation:** See `fleet-catalog/` directory
|
||||
|
||||
## Overview
|
||||
|
||||
This document proposes a standardized JSON schema for an open source catalog of airline fleets. The goal is to track aircraft properties (WiFi, cabin configuration, IFE, etc.) across multiple airlines with a consistent format and change history.
|
||||
|
||||
## Design Principles
|
||||
|
||||
1. **One JSON file per airline** - Easy to maintain, review PRs, and avoid merge conflicts
|
||||
2. **Standardized enums** - Consistent values across all airlines (e.g., WiFi types)
|
||||
3. **History tracking** - Record property changes over time with timestamps
|
||||
4. **Extensible** - Room for airline-specific fields without breaking the schema
|
||||
5. **Machine-readable** - JSON Schema validation for data quality
|
||||
|
||||
## Current Implementation
|
||||
|
||||
The schema has been implemented with Air France data exported from the fleet database:
|
||||
- **220 aircraft** with full property data
|
||||
- **History tracking** for WiFi upgrades, seat config changes, etc.
|
||||
- **ICAO24 hex codes** for ADS-B tracking correlation
|
||||
|
||||
---
|
||||
|
||||
## Proposed Directory Structure
|
||||
|
||||
```
|
||||
fleet-catalog/
|
||||
├── schema/
|
||||
│ └── aircraft.schema.json # JSON Schema for validation
|
||||
├── airlines/
|
||||
│ ├── AF.json # Air France
|
||||
│ ├── BA.json # British Airways
|
||||
│ ├── DL.json # Delta
|
||||
│ ├── LH.json # Lufthansa
|
||||
│ └── ...
|
||||
├── reference/
|
||||
│ ├── aircraft-types.json # ICAO/IATA aircraft type codes
|
||||
│ ├── wifi-providers.json # Known WiFi providers & capabilities
|
||||
│ └── cabin-class-codes.json # Cabin class code mappings
|
||||
└── README.md
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Schema Definition
|
||||
|
||||
### Root Object (Airline File)
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": "1.0.0",
|
||||
"airline": {
|
||||
"iata_code": "AF",
|
||||
"icao_code": "AFR",
|
||||
"name": "Air France",
|
||||
"country": "FR"
|
||||
},
|
||||
"generated_at": "2026-02-04T18:32:20.803Z",
|
||||
"aircraft": [...]
|
||||
}
|
||||
```
|
||||
|
||||
### Aircraft Object
|
||||
|
||||
```json
|
||||
{
|
||||
"registration": "FHPND",
|
||||
"icao24": "39bda3",
|
||||
|
||||
"aircraft_type": {
|
||||
"iata_code": "223",
|
||||
"icao_code": "A223",
|
||||
"manufacturer": "Airbus",
|
||||
"model": "A220",
|
||||
"variant": "300",
|
||||
"full_name": "AIRBUS A220-300 PASSENGER"
|
||||
},
|
||||
|
||||
"operator": {
|
||||
"sub_fleet_code": "CA",
|
||||
"cabin_crew_employer": "AF",
|
||||
"cockpit_crew_employer": "AF"
|
||||
},
|
||||
|
||||
"cabin": {
|
||||
"physical_configuration": "Y148",
|
||||
"operational_configuration": "C008Y135",
|
||||
"saleable_configuration": null,
|
||||
"total_seats": 148,
|
||||
"classes": {
|
||||
"first": 0,
|
||||
"business": 0,
|
||||
"premium_economy": 0,
|
||||
"economy": 148
|
||||
},
|
||||
"freight_configuration": "PP000LL000"
|
||||
},
|
||||
|
||||
"connectivity": {
|
||||
"wifi": "high-speed",
|
||||
"wifi_provider": "Starlink",
|
||||
"satellite": true,
|
||||
"live_tv": false,
|
||||
"power_outlets": true,
|
||||
"usb_ports": true
|
||||
},
|
||||
|
||||
"ife": {
|
||||
"type": "streaming",
|
||||
"personal_screens": false
|
||||
},
|
||||
|
||||
"status": "active",
|
||||
|
||||
"tracking": {
|
||||
"first_seen": "2025-12-20",
|
||||
"last_seen": "2026-02-05",
|
||||
"total_flights": 3214
|
||||
},
|
||||
|
||||
"metadata": {
|
||||
"delivery_date": null,
|
||||
"msn": null,
|
||||
"line_number": null,
|
||||
"production_site": null,
|
||||
"engine_type": null,
|
||||
"aircraft_name": null,
|
||||
"livery": null,
|
||||
"comments": null
|
||||
},
|
||||
|
||||
"history": [...]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Standardized Enums
|
||||
|
||||
### `connectivity.wifi`
|
||||
|
||||
| Value | Description | Examples |
|
||||
|-------|-------------|----------|
|
||||
| `"none"` | No WiFi available | — |
|
||||
| `"low-speed"` | Basic WiFi, typically < 10 Mbps | Gogo ATG, old Ku-band systems |
|
||||
| `"high-speed"` | Fast WiFi, typically > 50 Mbps | Starlink, Viasat Ka-band, Gogo 2Ku |
|
||||
|
||||
### `connectivity.wifi_provider`
|
||||
|
||||
Suggested standardized provider names:
|
||||
|
||||
| Provider | Notes |
|
||||
|----------|-------|
|
||||
| `"Starlink"` | SpaceX LEO constellation |
|
||||
| `"Viasat"` | Ka-band GEO satellites |
|
||||
| `"Gogo 2Ku"` | Dual Ku-band antennas |
|
||||
| `"Gogo ATG"` | Air-to-ground (US only) |
|
||||
| `"Panasonic Ku"` | Ku-band system |
|
||||
| `"Inmarsat GX"` | Global Xpress Ka-band |
|
||||
| `"Anuvu"` | Formerly Global Eagle |
|
||||
|
||||
### `ife.type`
|
||||
|
||||
| Value | Description |
|
||||
|-------|-------------|
|
||||
| `"none"` | No IFE system |
|
||||
| `"overhead"` | Shared overhead screens only |
|
||||
| `"seatback"` | Personal seatback screens |
|
||||
| `"streaming"` | BYOD streaming to personal devices |
|
||||
| `"hybrid"` | Both seatback screens and streaming |
|
||||
|
||||
### `status`
|
||||
|
||||
| Value | Description |
|
||||
|-------|-------------|
|
||||
| `"active"` | Currently in service |
|
||||
| `"stored"` | Temporarily stored/parked |
|
||||
| `"maintenance"` | In heavy maintenance |
|
||||
| `"retired"` | Permanently removed from fleet |
|
||||
|
||||
### Cabin Class Codes
|
||||
|
||||
Standard codes used in `configuration_raw`:
|
||||
|
||||
| Code | Class | Notes |
|
||||
|------|-------|-------|
|
||||
| `F` | First Class | Traditional first |
|
||||
| `P` | First Class | Premium first (e.g., La Première) |
|
||||
| `J` | Business Cla ss | Standard code |
|
||||
| `C` | Business Class | Alternative code |
|
||||
| `W` | Premium Economy | |
|
||||
| `Y` | Economy | |
|
||||
|
||||
---
|
||||
|
||||
## History Tracking
|
||||
|
||||
Each time a property changes, append an entry to the `history` array:
|
||||
|
||||
```json
|
||||
{
|
||||
"history": [
|
||||
{
|
||||
"timestamp": "2026-01-15T14:30:00.000Z",
|
||||
"property": "connectivity.wifi",
|
||||
"old_value": "low-speed",
|
||||
"new_value": "high-speed",
|
||||
"source": "flight_api"
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-01-15T14:30:00.000Z",
|
||||
"property": "connectivity.wifi_provider",
|
||||
"old_value": "Gogo",
|
||||
"new_value": "Starlink",
|
||||
"source": "flight_api"
|
||||
},
|
||||
{
|
||||
"timestamp": "2025-06-01T00:00:00.000Z",
|
||||
"property": "cabin.configuration_raw",
|
||||
"old_value": "Y146",
|
||||
"new_value": "Y148",
|
||||
"source": "manual"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### History Fields
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `timestamp` | ISO 8601 | When the change was detected |
|
||||
| `property` | string | Dot-notation path to the changed field |
|
||||
| `old_value` | any | Previous value (or `null` if new) |
|
||||
| `new_value` | any | New value |
|
||||
| `source` | string | How the change was detected |
|
||||
|
||||
### Source Values
|
||||
|
||||
| Value | Description |
|
||||
|-------|-------------|
|
||||
| `"flight_api"` | Detected via flight data API |
|
||||
| `"airline_api"` | From airline's official API |
|
||||
| `"manual"` | Manual update/correction |
|
||||
| `"seatguru"` | SeatGuru or similar source |
|
||||
| `"community"` | Community contribution |
|
||||
|
||||
---
|
||||
|
||||
## Example: Air France A220-300
|
||||
|
||||
```json
|
||||
{
|
||||
"registration": "FHPND",
|
||||
|
||||
"aircraft_type": {
|
||||
"icao_code": "A223",
|
||||
"iata_code": "223",
|
||||
"manufacturer": "Airbus",
|
||||
"model": "A220-300",
|
||||
"variant": null
|
||||
},
|
||||
|
||||
"cabin": {
|
||||
"configuration_raw": "Y148",
|
||||
"total_seats": 148,
|
||||
"classes": {
|
||||
"first": 0,
|
||||
"business": 0,
|
||||
"premium_economy": 0,
|
||||
"economy": 148
|
||||
}
|
||||
},
|
||||
|
||||
"connectivity": {
|
||||
"wifi": "high-speed",
|
||||
"wifi_provider": "Starlink",
|
||||
"live_tv": false,
|
||||
"power_outlets": true,
|
||||
"usb_ports": true
|
||||
},
|
||||
|
||||
"ife": {
|
||||
"type": "streaming",
|
||||
"personal_screens": false
|
||||
},
|
||||
|
||||
"status": "active",
|
||||
|
||||
"tracking": {
|
||||
"first_seen": "2025-12-20",
|
||||
"last_seen": "2026-02-05",
|
||||
"total_flights": 3214
|
||||
},
|
||||
|
||||
"history": [
|
||||
{
|
||||
"timestamp": "2026-01-15T14:30:00.000Z",
|
||||
"property": "connectivity.wifi",
|
||||
"old_value": "low-speed",
|
||||
"new_value": "high-speed",
|
||||
"source": "flight_api"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Example: Air France 777-300ER (Multi-Class)
|
||||
|
||||
```json
|
||||
{
|
||||
"registration": "FGSQA",
|
||||
|
||||
"aircraft_type": {
|
||||
"icao_code": "B77W",
|
||||
"iata_code": "77W",
|
||||
"manufacturer": "Boeing",
|
||||
"model": "777-300ER",
|
||||
"variant": null
|
||||
},
|
||||
|
||||
"cabin": {
|
||||
"configuration_raw": "P004J058W028Y206",
|
||||
"total_seats": 296,
|
||||
"classes": {
|
||||
"first": 4,
|
||||
"business": 58,
|
||||
"premium_economy": 28,
|
||||
"economy": 206
|
||||
}
|
||||
},
|
||||
|
||||
"connectivity": {
|
||||
"wifi": "high-speed",
|
||||
"wifi_provider": "Starlink",
|
||||
"live_tv": true,
|
||||
"power_outlets": true,
|
||||
"usb_ports": true
|
||||
},
|
||||
|
||||
"ife": {
|
||||
"type": "seatback",
|
||||
"personal_screens": true
|
||||
},
|
||||
|
||||
"status": "active",
|
||||
|
||||
"tracking": {
|
||||
"first_seen": "2025-12-20",
|
||||
"last_seen": "2026-02-05",
|
||||
"total_flights": 1137
|
||||
},
|
||||
|
||||
"history": []
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Migration from Current Format
|
||||
|
||||
For existing data (e.g., Air France tracking), here's the field mapping:
|
||||
|
||||
| Current Field | New Path | Transformation |
|
||||
|--------------|----------|----------------|
|
||||
| `registration` | `registration` | Keep as-is (no dash) |
|
||||
| `type_code` | `aircraft_type.iata_code` | Direct mapping |
|
||||
| `type_name` | `aircraft_type.*` | Parse into manufacturer/model |
|
||||
| `owner_airline_code` | Top-level `airline.iata_code` | Move to file level |
|
||||
| `owner_airline_name` | Top-level `airline.name` | Move to file level |
|
||||
| `wifi_enabled` | `connectivity.wifi` | Combine with `high_speed_wifi` |
|
||||
| `high_speed_wifi` | `connectivity.wifi` | `Y` → `"high-speed"`, else `"low-speed"` |
|
||||
| `physical_pax_configuration` | `cabin.configuration_raw` | Direct mapping |
|
||||
| — | `cabin.classes` | Parse from configuration |
|
||||
| `first_seen_date` | `tracking.first_seen` | Direct mapping |
|
||||
| `last_seen_date` | `tracking.last_seen` | Direct mapping |
|
||||
| `total_flights_tracked` | `tracking.total_flights` | Direct mapping |
|
||||
|
||||
### WiFi Conversion Logic
|
||||
|
||||
```javascript
|
||||
function convertWifi(wifi_enabled, high_speed_wifi) {
|
||||
if (wifi_enabled !== 'Y') return 'none';
|
||||
if (high_speed_wifi === 'Y') return 'high-speed';
|
||||
return 'low-speed';
|
||||
}
|
||||
```
|
||||
|
||||
### Cabin Configuration Parser
|
||||
|
||||
```javascript
|
||||
function parseCabinConfig(config) {
|
||||
// "P004J058W028Y206" → { first: 4, business: 58, premium_economy: 28, economy: 206 }
|
||||
const mapping = { P: 'first', F: 'first', J: 'business', C: 'business', W: 'premium_economy', Y: 'economy' };
|
||||
const classes = { first: 0, business: 0, premium_economy: 0, economy: 0 };
|
||||
const regex = /([PFJCWY])(\d{3})/g;
|
||||
let match;
|
||||
while ((match = regex.exec(config)) !== null) {
|
||||
const classKey = mapping[match[1]];
|
||||
classes[classKey] += parseInt(match[2], 10);
|
||||
}
|
||||
return classes;
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Metadata Fields (for PlaneSpotters-style data)
|
||||
|
||||
These fields capture additional data often found on PlaneSpotters.net:
|
||||
|
||||
```json
|
||||
{
|
||||
"metadata": {
|
||||
"delivery_date": "2022-03-15",
|
||||
"msn": "55012",
|
||||
"line_number": "1234",
|
||||
"production_site": "Mirabel",
|
||||
"engine_type": "PW1500G",
|
||||
"aircraft_name": "Fort-de-France",
|
||||
"livery": "standard",
|
||||
"comments": "Olympic Games 2024 special livery"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Metadata Field Descriptions
|
||||
|
||||
| Field | Description | Example |
|
||||
|-------|-------------|---------|
|
||||
| `delivery_date` | Date aircraft was delivered to airline | `2022-03-15` |
|
||||
| `msn` | Manufacturer Serial Number | `55012` |
|
||||
| `line_number` | Production line number | `1234` |
|
||||
| `production_site` | Factory location | `Toulouse`, `Hamburg`, `Mirabel`, `Charleston` |
|
||||
| `engine_type` | Engine model | `Trent XWB-84`, `GE90-115B`, `PW1500G` |
|
||||
| `aircraft_name` | Given name (if any) | `Fort-de-France`, `Château de Versailles` |
|
||||
| `livery` | Special paint scheme | `standard`, `SkyTeam`, `Olympic 2024` |
|
||||
| `comments` | Additional notes | Free text |
|
||||
|
||||
### Production Sites Reference
|
||||
|
||||
| Manufacturer | Sites |
|
||||
|--------------|-------|
|
||||
| Airbus | Toulouse (France), Hamburg (Germany), Tianjin (China), Mobile (USA) |
|
||||
| Boeing | Everett (USA), Renton (USA), Charleston (USA) |
|
||||
| Airbus Canada | Mirabel (Canada) |
|
||||
| Embraer | São José dos Campos (Brazil) |
|
||||
|
||||
---
|
||||
|
||||
## Validation
|
||||
|
||||
A JSON Schema file should be maintained at `schema/aircraft.schema.json` for:
|
||||
- CI validation on PRs
|
||||
- Editor autocomplete
|
||||
- Documentation generation
|
||||
|
||||
---
|
||||
|
||||
## Open Questions
|
||||
|
||||
1. **Registration format:** ✅ Decided: Strip dashes (`FHPND` not `F-HPND`)
|
||||
2. **ICAO24 hex codes:** ✅ Decided: Yes, include for ADS-B correlation
|
||||
3. **Frequency of updates:** Real-time vs. daily snapshots?
|
||||
4. **Historical snapshots:** Keep full point-in-time snapshots or just deltas?
|
||||
5. **API access:** Should we provide a read-only API for querying?
|
||||
6. **PlaneSpotters integration:** How to merge MSN, delivery dates, aircraft names?
|
||||
|
||||
---
|
||||
|
||||
## Implementation Status
|
||||
|
||||
- [x] Finalize schema based on feedback
|
||||
- [x] Create JSON Schema for validation (`schema/aircraft.schema.json`)
|
||||
- [x] Migrate Air France data to new format (`airlines/AF.json`)
|
||||
- [x] Set up repo structure
|
||||
- [x] Document contribution guidelines (`README.md`)
|
||||
- [ ] Add CI for schema validation
|
||||
- [ ] Add more airlines (KLM, Delta, etc.)
|
||||
- [ ] Integrate PlaneSpotters metadata (MSN, delivery dates, names)
|
||||
|
||||
Executable
+669
@@ -0,0 +1,669 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Air France / KLM Fleet Catalog Updater
|
||||
*
|
||||
* Standalone script to update AF.json or KL.json without a database.
|
||||
* Fetches flights from the Air France/KLM API and updates the catalog.
|
||||
*
|
||||
* Usage:
|
||||
* node fleet-update.js --airline AF # Update Air France
|
||||
* node fleet-update.js --airline KL # Update KLM
|
||||
* node fleet-update.js --airline KL --bootstrap # Build from scratch (7 days)
|
||||
* node fleet-update.js --airline KL --dry-run # Preview changes
|
||||
*
|
||||
* Environment:
|
||||
* AFKLM_API_KEY - Single API key for Air France/KLM API
|
||||
* AFKLM_API_KEYS - Comma-separated API keys (for rotation)
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
// Airline metadata
|
||||
const AIRLINES = {
|
||||
AF: {
|
||||
code: 'AF',
|
||||
name: 'Air France',
|
||||
country: 'France',
|
||||
registrationPrefix: 'F-',
|
||||
},
|
||||
KL: {
|
||||
code: 'KL',
|
||||
name: 'KLM Royal Dutch Airlines',
|
||||
country: 'Netherlands',
|
||||
registrationPrefix: 'PH-',
|
||||
},
|
||||
};
|
||||
|
||||
// Configuration (loaded dynamically)
|
||||
let CONFIG = {
|
||||
apiKeys: [],
|
||||
baseUrl: 'https://api.airfranceklm.com/opendata',
|
||||
pageSize: 100,
|
||||
requestDelay: 5000,
|
||||
catalogPath: null,
|
||||
airlineCode: null,
|
||||
};
|
||||
|
||||
// Track API usage
|
||||
let currentKeyIndex = 0;
|
||||
let lastRequestTime = 0;
|
||||
let totalRequests = 0;
|
||||
|
||||
// ============================================================================
|
||||
// API Functions
|
||||
// ============================================================================
|
||||
|
||||
function getApiKey() {
|
||||
return CONFIG.apiKeys[currentKeyIndex];
|
||||
}
|
||||
|
||||
function rotateKey() {
|
||||
currentKeyIndex = (currentKeyIndex + 1) % CONFIG.apiKeys.length;
|
||||
return getApiKey();
|
||||
}
|
||||
|
||||
async function throttle() {
|
||||
const now = Date.now();
|
||||
const timeSince = now - lastRequestTime;
|
||||
if (timeSince < CONFIG.requestDelay) {
|
||||
await new Promise(r => setTimeout(r, CONFIG.requestDelay - timeSince));
|
||||
}
|
||||
lastRequestTime = Date.now();
|
||||
}
|
||||
|
||||
async function apiRequest(endpoint, params = {}, retryCount = 0) {
|
||||
await throttle();
|
||||
totalRequests++;
|
||||
|
||||
const url = new URL(`${CONFIG.baseUrl}${endpoint}`);
|
||||
Object.entries(params).forEach(([key, value]) => {
|
||||
if (value !== undefined && value !== null) {
|
||||
url.searchParams.append(key, value);
|
||||
}
|
||||
});
|
||||
|
||||
// Rotate key before each request
|
||||
if (CONFIG.apiKeys.length > 1 && retryCount === 0) {
|
||||
rotateKey();
|
||||
}
|
||||
|
||||
const response = await fetch(url.toString(), {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'API-Key': getApiKey(),
|
||||
'Accept': 'application/hal+json',
|
||||
'Accept-Language': 'en-GB',
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
// Retry on rate limit (silently rotate key)
|
||||
if ((response.status === 429 || response.status === 403) && retryCount < CONFIG.apiKeys.length - 1) {
|
||||
rotateKey();
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
return apiRequest(endpoint, params, retryCount + 1);
|
||||
}
|
||||
throw new Error(`API Error: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Data Extraction
|
||||
// ============================================================================
|
||||
|
||||
function extractAircraftFromFlight(flight, airlineCode) {
|
||||
const leg = flight.flightLegs?.[0];
|
||||
if (!leg?.aircraft?.registration) return null;
|
||||
|
||||
const aircraft = leg.aircraft;
|
||||
|
||||
// Filter by owner airline
|
||||
if (aircraft.ownerAirlineCode !== airlineCode) return null;
|
||||
|
||||
return {
|
||||
registration: aircraft.registration,
|
||||
typeCode: aircraft.typeCode || null,
|
||||
typeName: aircraft.typeName || null,
|
||||
subFleetCode: aircraft.subFleetCodeId || null,
|
||||
ownerAirlineCode: aircraft.ownerAirlineCode || null,
|
||||
ownerAirlineName: aircraft.ownerAirlineName || null,
|
||||
cabinCrewEmployer: aircraft.cabinCrewEmployer || null,
|
||||
cockpitCrewEmployer: aircraft.cockpitCrewEmployer || null,
|
||||
wifiEnabled: aircraft.wifiEnabled || null,
|
||||
highSpeedWifi: aircraft.highSpeedWifi || null,
|
||||
satelliteConnectivity: aircraft.satelliteConnectivityOnBoard || null,
|
||||
physicalPaxConfiguration: aircraft.physicalPaxConfiguration || null,
|
||||
};
|
||||
}
|
||||
|
||||
function parseCabinConfig(config) {
|
||||
if (!config) return { first: 0, business: 0, premium_economy: 0, economy: 0 };
|
||||
|
||||
// P/F = First, J/C = Business, W/S = Premium Economy, Y/M = Economy
|
||||
const mapping = {
|
||||
P: 'first', F: 'first',
|
||||
J: 'business', C: 'business',
|
||||
W: 'premium_economy', S: 'premium_economy',
|
||||
Y: 'economy', M: 'economy'
|
||||
};
|
||||
const classes = { first: 0, business: 0, premium_economy: 0, economy: 0 };
|
||||
|
||||
const regex = /([PFJCWSYM])(\d{2,3})/g;
|
||||
let match;
|
||||
while ((match = regex.exec(config)) !== null) {
|
||||
const classKey = mapping[match[1]];
|
||||
if (classKey) classes[classKey] += parseInt(match[2], 10);
|
||||
}
|
||||
|
||||
return classes;
|
||||
}
|
||||
|
||||
function convertWifi(wifiEnabled, highSpeedWifi) {
|
||||
if (wifiEnabled !== 'Y') return 'none';
|
||||
if (highSpeedWifi === 'Y') return 'high-speed';
|
||||
return 'low-speed';
|
||||
}
|
||||
|
||||
function transformToSchema(raw, firstSeenDate) {
|
||||
const cabinClasses = parseCabinConfig(raw.physicalPaxConfiguration);
|
||||
|
||||
return {
|
||||
registration: raw.registration,
|
||||
icao24: null,
|
||||
|
||||
aircraft_type: {
|
||||
iata_code: raw.typeCode,
|
||||
icao_code: null,
|
||||
manufacturer: guessManufacturer(raw.typeName),
|
||||
model: guessModel(raw.typeName),
|
||||
variant: guessVariant(raw.typeName),
|
||||
full_name: raw.typeName,
|
||||
},
|
||||
|
||||
operator: {
|
||||
sub_fleet_code: raw.subFleetCode,
|
||||
cabin_crew_employer: raw.cabinCrewEmployer,
|
||||
cockpit_crew_employer: raw.cockpitCrewEmployer,
|
||||
},
|
||||
|
||||
cabin: {
|
||||
physical_configuration: raw.physicalPaxConfiguration,
|
||||
saleable_configuration: null,
|
||||
total_seats: Object.values(cabinClasses).reduce((a, b) => a + b, 0) || null,
|
||||
classes: cabinClasses,
|
||||
freight_configuration: null,
|
||||
},
|
||||
|
||||
connectivity: {
|
||||
wifi: convertWifi(raw.wifiEnabled, raw.highSpeedWifi),
|
||||
wifi_provider: raw.highSpeedWifi === 'Y' ? 'Starlink' : null,
|
||||
satellite: raw.satelliteConnectivity === 'Y',
|
||||
},
|
||||
|
||||
status: 'active',
|
||||
|
||||
tracking: {
|
||||
first_seen: firstSeenDate,
|
||||
last_seen: firstSeenDate,
|
||||
total_flights: 1,
|
||||
},
|
||||
|
||||
metadata: {
|
||||
created_at: new Date().toISOString(),
|
||||
updated_at: new Date().toISOString(),
|
||||
},
|
||||
|
||||
history: [],
|
||||
};
|
||||
}
|
||||
|
||||
function guessManufacturer(typeName) {
|
||||
if (!typeName) return null;
|
||||
if (typeName.toUpperCase().includes('AIRBUS')) return 'Airbus';
|
||||
if (typeName.toUpperCase().includes('BOEING')) return 'Boeing';
|
||||
if (typeName.toUpperCase().includes('EMBRAER')) return 'Embraer';
|
||||
return null;
|
||||
}
|
||||
|
||||
function guessModel(typeName) {
|
||||
if (!typeName) return null;
|
||||
const match = typeName.match(/A(\d{3})|(\d{3})/);
|
||||
if (match) return match[1] ? `A${match[1]}` : match[2];
|
||||
return null;
|
||||
}
|
||||
|
||||
function guessVariant(typeName) {
|
||||
if (!typeName) return null;
|
||||
const match = typeName.match(/-(\d+)/);
|
||||
return match ? match[1] : null;
|
||||
}
|
||||
|
||||
function formatDate(date) {
|
||||
return date.toISOString().split('T')[0];
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Fetch Flights
|
||||
// ============================================================================
|
||||
|
||||
async function fetchFlightsForDate(dateStr, airlineCode) {
|
||||
const dayStart = `${dateStr}T00:00:00Z`;
|
||||
const dayEnd = `${dateStr}T23:59:59Z`;
|
||||
|
||||
const allFlights = [];
|
||||
let pageNumber = 0;
|
||||
let hasMore = true;
|
||||
|
||||
while (hasMore) {
|
||||
try {
|
||||
const response = await apiRequest('/flightstatus', {
|
||||
startRange: dayStart,
|
||||
endRange: dayEnd,
|
||||
movementType: 'D',
|
||||
timeOriginType: 'S',
|
||||
timeType: 'U',
|
||||
pageSize: CONFIG.pageSize,
|
||||
pageNumber,
|
||||
operatingAirlineCode: airlineCode,
|
||||
});
|
||||
|
||||
const flights = response.operationalFlights || [];
|
||||
allFlights.push(...flights);
|
||||
|
||||
const page = response.page || {};
|
||||
const totalPages = page.totalPages || 1;
|
||||
|
||||
process.stdout.write(`\r ${dateStr}: Page ${pageNumber + 1}/${totalPages} (${allFlights.length} flights)`);
|
||||
|
||||
hasMore = pageNumber < (totalPages - 1);
|
||||
pageNumber++;
|
||||
|
||||
if (pageNumber > 100) break;
|
||||
} catch (error) {
|
||||
if (error.message.includes('403') || error.message.includes('429')) {
|
||||
console.log(`\n ⚠️ API rate limit reached after ${pageNumber} pages`);
|
||||
break;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
process.stdout.write('\n');
|
||||
return allFlights;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Update Logic
|
||||
// ============================================================================
|
||||
|
||||
function detectChanges(existing, newData, dateStr) {
|
||||
const changes = [];
|
||||
|
||||
if (existing.connectivity?.wifi !== newData.connectivity?.wifi) {
|
||||
changes.push({
|
||||
timestamp: dateStr,
|
||||
property: 'connectivity.wifi',
|
||||
old_value: existing.connectivity?.wifi,
|
||||
new_value: newData.connectivity?.wifi,
|
||||
source: 'airline_api',
|
||||
});
|
||||
}
|
||||
|
||||
if (existing.connectivity?.wifi_provider !== newData.connectivity?.wifi_provider) {
|
||||
changes.push({
|
||||
timestamp: dateStr,
|
||||
property: 'connectivity.wifi_provider',
|
||||
old_value: existing.connectivity?.wifi_provider,
|
||||
new_value: newData.connectivity?.wifi_provider,
|
||||
source: 'airline_api',
|
||||
});
|
||||
}
|
||||
|
||||
if (existing.cabin?.physical_configuration !== newData.cabin?.physical_configuration) {
|
||||
changes.push({
|
||||
timestamp: dateStr,
|
||||
property: 'cabin.physical_configuration',
|
||||
old_value: existing.cabin?.physical_configuration,
|
||||
new_value: newData.cabin?.physical_configuration,
|
||||
source: 'airline_api',
|
||||
});
|
||||
}
|
||||
|
||||
if (existing.operator?.sub_fleet_code !== newData.operator?.sub_fleet_code) {
|
||||
changes.push({
|
||||
timestamp: dateStr,
|
||||
property: 'operator.sub_fleet_code',
|
||||
old_value: existing.operator?.sub_fleet_code,
|
||||
new_value: newData.operator?.sub_fleet_code,
|
||||
source: 'airline_api',
|
||||
});
|
||||
}
|
||||
|
||||
return changes;
|
||||
}
|
||||
|
||||
function mergeAircraft(existing, newData, changes, dateStr) {
|
||||
existing.connectivity = newData.connectivity;
|
||||
existing.cabin.physical_configuration = newData.cabin.physical_configuration;
|
||||
existing.cabin.total_seats = newData.cabin.total_seats;
|
||||
existing.cabin.classes = newData.cabin.classes;
|
||||
existing.operator = newData.operator;
|
||||
existing.aircraft_type = newData.aircraft_type;
|
||||
|
||||
existing.tracking.last_seen = dateStr;
|
||||
existing.tracking.total_flights = (existing.tracking.total_flights || 0) + 1;
|
||||
existing.metadata.updated_at = new Date().toISOString();
|
||||
|
||||
if (changes.length > 0) {
|
||||
const existingKeys = new Set(
|
||||
existing.history.map(h => `${h.timestamp}|${h.property}|${h.old_value}|${h.new_value}`)
|
||||
);
|
||||
|
||||
for (const change of changes) {
|
||||
const key = `${change.timestamp}|${change.property}|${change.old_value}|${change.new_value}`;
|
||||
if (!existingKeys.has(key)) {
|
||||
existing.history.push(change);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return existing;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Main
|
||||
// ============================================================================
|
||||
|
||||
function printHelp() {
|
||||
console.log(`
|
||||
✈️ Air France / KLM Fleet Catalog Updater
|
||||
|
||||
Usage:
|
||||
node fleet-update.js --airline <CODE> [options]
|
||||
|
||||
Required:
|
||||
--airline <CODE> Airline code: AF (Air France) or KL (KLM)
|
||||
|
||||
Options:
|
||||
--dry-run Preview changes without saving
|
||||
--date <YYYY-MM-DD> Use specific date instead of today
|
||||
--bootstrap Build catalog from scratch (crawl last 7 days)
|
||||
--days <N> Number of days for bootstrap (default: 7)
|
||||
--verbose Show detailed output
|
||||
--output-changes Export changes to changes.json
|
||||
--stale-days <N> Days threshold for stale aircraft (default: 30)
|
||||
--help Show this help message
|
||||
|
||||
Environment:
|
||||
AFKLM_API_KEY Single API key
|
||||
AFKLM_API_KEYS Comma-separated API keys (for rotation)
|
||||
|
||||
Examples:
|
||||
node fleet-update.js --airline AF # Update Air France
|
||||
node fleet-update.js --airline KL --bootstrap # Build KLM catalog
|
||||
node fleet-update.js --airline KL --dry-run # Preview KLM changes
|
||||
`);
|
||||
}
|
||||
|
||||
function getDateRange(startDate, days) {
|
||||
const dates = [];
|
||||
for (let i = days - 1; i >= 0; i--) {
|
||||
const d = new Date(startDate);
|
||||
d.setDate(d.getDate() - i);
|
||||
dates.push(formatDate(d));
|
||||
}
|
||||
return dates;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
if (args.includes('--help') || args.includes('-h')) {
|
||||
printHelp();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Parse arguments
|
||||
const airlineArg = args.find((_, i) => args[i - 1] === '--airline');
|
||||
const dryRun = args.includes('--dry-run');
|
||||
const verbose = args.includes('--verbose') || args.includes('-v');
|
||||
const outputChanges = args.includes('--output-changes');
|
||||
const bootstrap = args.includes('--bootstrap');
|
||||
const dateArg = args.find((_, i) => args[i - 1] === '--date');
|
||||
const daysArg = args.find((_, i) => args[i - 1] === '--days');
|
||||
const staleDaysArg = args.find((_, i) => args[i - 1] === '--stale-days');
|
||||
|
||||
const staleDays = parseInt(staleDaysArg || '30', 10);
|
||||
const bootstrapDays = parseInt(daysArg || '7', 10);
|
||||
|
||||
// Validate airline
|
||||
if (!airlineArg || !AIRLINES[airlineArg]) {
|
||||
console.error('❌ Error: --airline is required (AF or KL)');
|
||||
printHelp();
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const airlineCode = airlineArg.toUpperCase();
|
||||
const airline = AIRLINES[airlineCode];
|
||||
|
||||
// Load API keys from environment
|
||||
const apiKeys = (process.env.AFKLM_API_KEYS || process.env.AFKLM_API_KEY || '').split(',').filter(k => k);
|
||||
if (apiKeys.length === 0) {
|
||||
console.error('❌ Error: No API key found. Set AFKLM_API_KEY or AFKLM_API_KEYS environment variable.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Configure
|
||||
CONFIG.apiKeys = apiKeys;
|
||||
CONFIG.airlineCode = airlineCode;
|
||||
CONFIG.catalogPath = path.join(__dirname, 'airlines', `${airlineCode}.json`);
|
||||
|
||||
console.log(`\n✈️ ${airline.name} Fleet Catalog Updater\n`);
|
||||
console.log(` 🔑 API keys loaded: ${apiKeys.length}`);
|
||||
|
||||
if (dryRun) {
|
||||
console.log(' 🔍 DRY RUN - no changes will be saved\n');
|
||||
}
|
||||
|
||||
// Load or create catalog
|
||||
let catalog;
|
||||
const catalogExists = fs.existsSync(CONFIG.catalogPath);
|
||||
|
||||
if (catalogExists && !bootstrap) {
|
||||
console.log(`📂 Loading ${CONFIG.catalogPath}...`);
|
||||
const content = fs.readFileSync(CONFIG.catalogPath, 'utf-8');
|
||||
catalog = JSON.parse(content);
|
||||
console.log(` Found ${catalog.aircraft_count} aircraft\n`);
|
||||
} else {
|
||||
if (bootstrap) {
|
||||
console.log(`🚀 Bootstrap mode: Creating new catalog for ${airline.name}\n`);
|
||||
} else {
|
||||
console.log(`📂 No existing catalog found, creating new one\n`);
|
||||
}
|
||||
catalog = {
|
||||
schema_version: '1.0.0',
|
||||
airline: {
|
||||
iata_code: airlineCode,
|
||||
name: airline.name,
|
||||
country: airline.country,
|
||||
},
|
||||
generated_at: new Date().toISOString(),
|
||||
aircraft_count: 0,
|
||||
aircraft: [],
|
||||
};
|
||||
}
|
||||
|
||||
// Build lookup
|
||||
const aircraftByReg = new Map();
|
||||
catalog.aircraft.forEach(a => aircraftByReg.set(a.registration, a));
|
||||
|
||||
// Determine dates to process
|
||||
let datesToProcess;
|
||||
if (bootstrap) {
|
||||
datesToProcess = getDateRange(new Date(), bootstrapDays);
|
||||
console.log(`📅 Crawling ${bootstrapDays} days: ${datesToProcess[0]} → ${datesToProcess[datesToProcess.length - 1]}\n`);
|
||||
} else {
|
||||
const targetDate = dateArg || formatDate(new Date());
|
||||
datesToProcess = [targetDate];
|
||||
console.log(`📅 Processing: ${targetDate}\n`);
|
||||
}
|
||||
|
||||
// Process each date
|
||||
let totalNew = 0;
|
||||
let totalUpdated = 0;
|
||||
let totalSeen = 0;
|
||||
const allChanges = [];
|
||||
const seenAircraftAll = new Map();
|
||||
|
||||
for (const dateStr of datesToProcess) {
|
||||
console.log(`📡 Fetching ${airlineCode} flights for ${dateStr}...`);
|
||||
|
||||
const flights = await fetchFlightsForDate(dateStr, airlineCode);
|
||||
|
||||
// Extract aircraft
|
||||
const seenToday = new Map();
|
||||
for (const flight of flights) {
|
||||
const extracted = extractAircraftFromFlight(flight, airlineCode);
|
||||
if (extracted && extracted.registration) {
|
||||
seenToday.set(extracted.registration, extracted);
|
||||
seenAircraftAll.set(extracted.registration, { data: extracted, date: dateStr });
|
||||
}
|
||||
}
|
||||
|
||||
console.log(` ✈️ ${seenToday.size} unique ${airlineCode} aircraft\n`);
|
||||
|
||||
// Process
|
||||
for (const [reg, rawData] of seenToday) {
|
||||
const newData = transformToSchema(rawData, dateStr);
|
||||
const existing = aircraftByReg.get(reg);
|
||||
|
||||
if (!existing) {
|
||||
totalNew++;
|
||||
if (verbose || bootstrap) {
|
||||
console.log(` ➕ NEW: ${reg} (${rawData.typeName || 'Unknown'})`);
|
||||
}
|
||||
|
||||
if (!dryRun) {
|
||||
catalog.aircraft.push(newData);
|
||||
aircraftByReg.set(reg, newData);
|
||||
}
|
||||
} else {
|
||||
const changes = detectChanges(existing, newData, dateStr);
|
||||
|
||||
if (changes.length > 0) {
|
||||
totalUpdated++;
|
||||
if (verbose) {
|
||||
console.log(` 🔄 UPDATED: ${reg}`);
|
||||
changes.forEach(c => console.log(` ${c.property}: ${c.old_value} → ${c.new_value}`));
|
||||
}
|
||||
allChanges.push(...changes.map(c => ({ registration: reg, ...c })));
|
||||
|
||||
if (!dryRun) {
|
||||
mergeAircraft(existing, newData, changes, dateStr);
|
||||
}
|
||||
} else {
|
||||
totalSeen++;
|
||||
if (!dryRun) {
|
||||
existing.tracking.last_seen = dateStr;
|
||||
existing.tracking.total_flights = (existing.tracking.total_flights || 0) + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Summary
|
||||
console.log('\n' + '═'.repeat(50));
|
||||
console.log('📊 Summary');
|
||||
console.log('═'.repeat(50));
|
||||
console.log(` New aircraft: ${totalNew}`);
|
||||
console.log(` Updated aircraft: ${totalUpdated}`);
|
||||
console.log(` Seen (no change): ${totalSeen}`);
|
||||
console.log(` Total in catalog: ${catalog.aircraft.length}`);
|
||||
console.log(` Total changes: ${allChanges.length}`);
|
||||
console.log(` API requests: ${totalRequests}`);
|
||||
|
||||
// Stale aircraft
|
||||
if (!bootstrap) {
|
||||
const notSeen = catalog.aircraft.filter(a => !seenAircraftAll.has(a.registration));
|
||||
const todayDate = new Date();
|
||||
const staleThreshold = new Date(todayDate.getTime() - staleDays * 24 * 60 * 60 * 1000);
|
||||
const staleAircraft = notSeen.filter(a => {
|
||||
if (!a.tracking?.last_seen) return true;
|
||||
return new Date(a.tracking.last_seen) < staleThreshold;
|
||||
});
|
||||
|
||||
if (staleAircraft.length > 0) {
|
||||
console.log(`\n⚠️ Stale aircraft (not seen in ${staleDays}+ days): ${staleAircraft.length}`);
|
||||
staleAircraft.slice(0, 5).forEach(a => {
|
||||
console.log(` - ${a.registration} (last: ${a.tracking?.last_seen || 'never'})`);
|
||||
});
|
||||
if (staleAircraft.length > 5) console.log(` ... and ${staleAircraft.length - 5} more`);
|
||||
}
|
||||
}
|
||||
|
||||
// WiFi stats
|
||||
const wifiStats = { none: 0, 'low-speed': 0, 'high-speed': 0 };
|
||||
catalog.aircraft.forEach(a => {
|
||||
const wifi = a.connectivity?.wifi || 'none';
|
||||
wifiStats[wifi] = (wifiStats[wifi] || 0) + 1;
|
||||
});
|
||||
const total = catalog.aircraft.length;
|
||||
console.log('\n📶 Fleet WiFi Status:');
|
||||
console.log(` High-speed (Starlink): ${wifiStats['high-speed']} (${total ? Math.round(wifiStats['high-speed'] / total * 100) : 0}%)`);
|
||||
console.log(` Low-speed: ${wifiStats['low-speed']} (${total ? Math.round(wifiStats['low-speed'] / total * 100) : 0}%)`);
|
||||
console.log(` None: ${wifiStats['none']} (${total ? Math.round(wifiStats['none'] / total * 100) : 0}%)`);
|
||||
|
||||
// Export changes
|
||||
if (outputChanges && allChanges.length > 0) {
|
||||
const changesPath = path.join(__dirname, `${airlineCode.toLowerCase()}-changes.json`);
|
||||
fs.writeFileSync(changesPath, JSON.stringify({
|
||||
generated_at: new Date().toISOString(),
|
||||
airline: airlineCode,
|
||||
changes: allChanges,
|
||||
}, null, 2));
|
||||
console.log(`\n📝 Changes exported to ${changesPath}`);
|
||||
}
|
||||
|
||||
// Save
|
||||
if (!dryRun && (totalNew > 0 || totalUpdated > 0 || totalSeen > 0)) {
|
||||
catalog.generated_at = new Date().toISOString();
|
||||
catalog.aircraft_count = catalog.aircraft.length;
|
||||
|
||||
catalog.aircraft.sort((a, b) => {
|
||||
const typeCompare = (a.aircraft_type?.iata_code || '').localeCompare(b.aircraft_type?.iata_code || '');
|
||||
if (typeCompare !== 0) return typeCompare;
|
||||
return a.registration.localeCompare(b.registration);
|
||||
});
|
||||
|
||||
// Ensure directory exists
|
||||
const dir = path.dirname(CONFIG.catalogPath);
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
|
||||
console.log(`\n💾 Saving to ${CONFIG.catalogPath}...`);
|
||||
fs.writeFileSync(CONFIG.catalogPath, JSON.stringify(catalog, null, 2));
|
||||
console.log('✅ Done!');
|
||||
} else if (dryRun) {
|
||||
console.log('\n🔍 Dry run complete - no changes saved');
|
||||
} else {
|
||||
console.log('\n✅ No changes to save');
|
||||
}
|
||||
|
||||
console.log();
|
||||
}
|
||||
|
||||
main().catch(error => {
|
||||
console.error(`\n❌ Error: ${error.message}`);
|
||||
if (process.env.DEBUG) console.error(error.stack);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
Executable
+393
@@ -0,0 +1,393 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Generate README with fleet statistics
|
||||
*
|
||||
* Automatically updates README.md with current fleet data from JSON files.
|
||||
* Run this after updating fleet data to keep stats in sync.
|
||||
*
|
||||
* Usage:
|
||||
* node generate-readme.js
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
// Airline display info
|
||||
const AIRLINE_INFO = {
|
||||
AF: { name: 'Air France', flag: '🇫🇷', country: 'France' },
|
||||
KL: { name: 'KLM', flag: '🇳🇱', country: 'Netherlands' },
|
||||
};
|
||||
|
||||
// Load all airline data
|
||||
function loadAirlines() {
|
||||
const airlinesDir = path.join(__dirname, 'airlines');
|
||||
const files = fs.readdirSync(airlinesDir).filter(f => f.endsWith('.json'));
|
||||
|
||||
const airlines = {};
|
||||
for (const file of files) {
|
||||
const code = file.replace('.json', '');
|
||||
const content = fs.readFileSync(path.join(airlinesDir, file), 'utf-8');
|
||||
airlines[code] = JSON.parse(content);
|
||||
}
|
||||
return airlines;
|
||||
}
|
||||
|
||||
// Get fleet breakdown by type
|
||||
function getFleetBreakdown(aircraft) {
|
||||
const breakdown = {};
|
||||
|
||||
for (const a of aircraft) {
|
||||
const typeName = a.aircraft_type?.full_name || 'Unknown';
|
||||
// Simplify type name
|
||||
let simpleType = typeName
|
||||
.replace('AIRBUS ', '')
|
||||
.replace('BOEING ', '')
|
||||
.replace(' (WINGLETS) PASSENGER/BBJ1', '')
|
||||
.replace(' (WINGLETS) PASSENGER/BBJ2', '')
|
||||
.replace(' (WINGLETS) PASSENGER/BBJ3', '')
|
||||
.replace('/200 ER', '-200ER')
|
||||
.replace('-200/200 ER', '-200ER')
|
||||
.trim();
|
||||
|
||||
breakdown[simpleType] = (breakdown[simpleType] || 0) + 1;
|
||||
}
|
||||
|
||||
// Sort by count descending
|
||||
return Object.entries(breakdown)
|
||||
.sort((a, b) => b[1] - a[1]);
|
||||
}
|
||||
|
||||
// Get WiFi stats
|
||||
function getWifiStats(aircraft) {
|
||||
const stats = { none: 0, 'low-speed': 0, 'high-speed': 0 };
|
||||
|
||||
for (const a of aircraft) {
|
||||
const wifi = a.connectivity?.wifi || 'none';
|
||||
stats[wifi] = (stats[wifi] || 0) + 1;
|
||||
}
|
||||
|
||||
const total = aircraft.length;
|
||||
return {
|
||||
total,
|
||||
none: stats.none,
|
||||
lowSpeed: stats['low-speed'],
|
||||
highSpeed: stats['high-speed'],
|
||||
nonePercent: total ? Math.round(stats.none / total * 100) : 0,
|
||||
lowSpeedPercent: total ? Math.round(stats['low-speed'] / total * 100) : 0,
|
||||
highSpeedPercent: total ? Math.round(stats['high-speed'] / total * 100) : 0,
|
||||
};
|
||||
}
|
||||
|
||||
// Generate markdown table for fleet breakdown
|
||||
function generateFleetTable(airlines) {
|
||||
let md = '';
|
||||
|
||||
for (const [code, data] of Object.entries(airlines)) {
|
||||
const info = AIRLINE_INFO[code] || { name: code, flag: '✈️' };
|
||||
const breakdown = getFleetBreakdown(data.aircraft);
|
||||
const wifi = getWifiStats(data.aircraft);
|
||||
|
||||
md += `### ${info.flag} ${info.name} (${code})\n\n`;
|
||||
md += `| Aircraft Type | Count |\n`;
|
||||
md += `|---------------|-------|\n`;
|
||||
|
||||
for (const [type, count] of breakdown) {
|
||||
md += `| ${type} | ${count} |\n`;
|
||||
}
|
||||
|
||||
md += `| **Total** | **${wifi.total}** |\n\n`;
|
||||
}
|
||||
|
||||
return md;
|
||||
}
|
||||
|
||||
// Get detailed breakdown by type and config
|
||||
function getDetailedBreakdown(aircraft) {
|
||||
const breakdown = {};
|
||||
|
||||
for (const a of aircraft) {
|
||||
const typeName = a.aircraft_type?.full_name || 'Unknown';
|
||||
// Simplify type name
|
||||
let simpleType = typeName
|
||||
.replace('AIRBUS ', '')
|
||||
.replace('BOEING ', '')
|
||||
.replace(' (WINGLETS) PASSENGER/BBJ1', '')
|
||||
.replace(' (WINGLETS) PASSENGER/BBJ2', '')
|
||||
.replace(' (WINGLETS) PASSENGER/BBJ3', '')
|
||||
.replace('/200 ER', '-200ER')
|
||||
.replace('-200/200 ER', '-200ER')
|
||||
.trim();
|
||||
|
||||
const config = a.cabin?.physical_configuration || '-';
|
||||
const wifi = a.connectivity?.wifi || 'none';
|
||||
const seats = a.cabin?.total_seats || 0;
|
||||
|
||||
const key = `${simpleType}|||${config}`;
|
||||
|
||||
if (!breakdown[key]) {
|
||||
breakdown[key] = {
|
||||
type: simpleType,
|
||||
config,
|
||||
seats,
|
||||
wifi,
|
||||
count: 0,
|
||||
highSpeed: 0,
|
||||
};
|
||||
}
|
||||
|
||||
breakdown[key].count++;
|
||||
if (wifi === 'high-speed') {
|
||||
breakdown[key].highSpeed++;
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by type name, then by config (to group similar aircraft together)
|
||||
return Object.values(breakdown)
|
||||
.sort((a, b) => {
|
||||
const typeCompare = a.type.localeCompare(b.type);
|
||||
if (typeCompare !== 0) return typeCompare;
|
||||
return a.config.localeCompare(b.config);
|
||||
});
|
||||
}
|
||||
|
||||
// Generate detailed fleet table per airline
|
||||
function generateDetailedFleetTable(airlines) {
|
||||
let md = '';
|
||||
|
||||
for (const [code, data] of Object.entries(airlines)) {
|
||||
const info = AIRLINE_INFO[code] || { name: code, flag: '✈️' };
|
||||
const breakdown = getDetailedBreakdown(data.aircraft);
|
||||
|
||||
md += `### ${info.flag} ${info.name} — Detailed Configuration\n\n`;
|
||||
md += `| Aircraft | Config | Seats | Count | 🛜 Starlink |\n`;
|
||||
md += `|----------|--------|-------|-------|-------------|\n`;
|
||||
|
||||
for (const item of breakdown) {
|
||||
const starlinkInfo = item.highSpeed > 0
|
||||
? `${item.highSpeed}/${item.count} (${Math.round(item.highSpeed / item.count * 100)}%)`
|
||||
: '-';
|
||||
md += `| ${item.type} | \`${item.config}\` | ${item.seats || '-'} | ${item.count} | ${starlinkInfo} |\n`;
|
||||
}
|
||||
|
||||
md += `\n`;
|
||||
}
|
||||
|
||||
return md;
|
||||
}
|
||||
|
||||
// Generate WiFi summary table
|
||||
function generateWifiSummary(airlines) {
|
||||
let md = '| Airline | Total | 📶 WiFi | 🛜 High-Speed | % Starlink |\n';
|
||||
md += '|---------|-------|---------|---------------|------------|\n';
|
||||
|
||||
let grandTotal = 0;
|
||||
let grandWifi = 0;
|
||||
let grandHighSpeed = 0;
|
||||
|
||||
for (const [code, data] of Object.entries(airlines)) {
|
||||
const info = AIRLINE_INFO[code] || { name: code, flag: '✈️' };
|
||||
const wifi = getWifiStats(data.aircraft);
|
||||
|
||||
const wifiTotal = wifi.lowSpeed + wifi.highSpeed;
|
||||
const wifiPercent = wifi.total ? Math.round(wifiTotal / wifi.total * 100) : 0;
|
||||
|
||||
md += `| ${info.flag} ${info.name} | ${wifi.total} | ${wifiTotal} (${wifiPercent}%) | ${wifi.highSpeed} | **${wifi.highSpeedPercent}%** |\n`;
|
||||
|
||||
grandTotal += wifi.total;
|
||||
grandWifi += wifiTotal;
|
||||
grandHighSpeed += wifi.highSpeed;
|
||||
}
|
||||
|
||||
const grandWifiPercent = grandTotal ? Math.round(grandWifi / grandTotal * 100) : 0;
|
||||
const grandHighSpeedPercent = grandTotal ? Math.round(grandHighSpeed / grandTotal * 100) : 0;
|
||||
|
||||
md += `| **Combined** | **${grandTotal}** | **${grandWifi} (${grandWifiPercent}%)** | **${grandHighSpeed}** | **${grandHighSpeedPercent}%** |\n`;
|
||||
|
||||
return md;
|
||||
}
|
||||
|
||||
// Generate the full README
|
||||
function generateReadme(airlines) {
|
||||
const now = new Date().toISOString().split('T')[0];
|
||||
|
||||
return `# ✈️ AF-KLM Fleet Catalog
|
||||
|
||||
Open source, community-maintained catalog of **Air France** and **KLM** fleets with real-time tracking of aircraft properties, WiFi connectivity, and historical changes.
|
||||
|
||||
---
|
||||
|
||||
## 📊 Fleet Overview
|
||||
|
||||
${generateWifiSummary(airlines)}
|
||||
|
||||
> 🛜 **High-Speed** = Starlink satellite internet (50+ Mbps)
|
||||
> 📶 **WiFi** = Any WiFi connectivity (low-speed or high-speed)
|
||||
|
||||
*Last updated: ${now}*
|
||||
|
||||
---
|
||||
|
||||
## 🛫 Fleet Breakdown
|
||||
|
||||
${generateFleetTable(airlines)}
|
||||
|
||||
---
|
||||
|
||||
## 📋 Detailed Configuration
|
||||
|
||||
${generateDetailedFleetTable(airlines)}
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Quick Start
|
||||
|
||||
### Update the Catalog
|
||||
|
||||
\`\`\`bash
|
||||
# Set your API key
|
||||
export AFKLM_API_KEY=your_api_key_here
|
||||
|
||||
# Update Air France
|
||||
node fleet-update.js --airline AF
|
||||
|
||||
# Update KLM
|
||||
node fleet-update.js --airline KL
|
||||
|
||||
# Preview changes without saving
|
||||
node fleet-update.js --airline KL --dry-run
|
||||
|
||||
# Regenerate this README with latest stats
|
||||
node generate-readme.js
|
||||
\`\`\`
|
||||
|
||||
### Using the Data
|
||||
|
||||
\`\`\`javascript
|
||||
// Load Air France fleet
|
||||
const response = await fetch('https://raw.githubusercontent.com/.../airlines/AF.json');
|
||||
const fleet = await response.json();
|
||||
|
||||
// Find all Starlink aircraft
|
||||
const starlink = fleet.aircraft.filter(a => a.connectivity.wifi === 'high-speed');
|
||||
console.log(\`\${starlink.length} aircraft with Starlink\`);
|
||||
|
||||
// Get aircraft by type
|
||||
const a350s = fleet.aircraft.filter(a => a.aircraft_type.full_name?.includes('A350'));
|
||||
\`\`\`
|
||||
|
||||
---
|
||||
|
||||
## 📁 Data Structure
|
||||
|
||||
\`\`\`
|
||||
af-klm/
|
||||
├── airlines/
|
||||
│ ├── AF.json # Air France fleet
|
||||
│ └── KL.json # KLM fleet
|
||||
├── schema/
|
||||
│ └── aircraft.schema.json
|
||||
├── fleet-update.js # Update script
|
||||
└── generate-readme.js # This stats generator
|
||||
\`\`\`
|
||||
|
||||
### Aircraft Schema
|
||||
|
||||
\`\`\`json
|
||||
{
|
||||
"registration": "F-HTYA",
|
||||
"aircraft_type": {
|
||||
"iata_code": "359",
|
||||
"manufacturer": "Airbus",
|
||||
"model": "A350",
|
||||
"full_name": "AIRBUS A350-900"
|
||||
},
|
||||
"cabin": {
|
||||
"physical_configuration": "J034W024Y266",
|
||||
"total_seats": 324,
|
||||
"classes": { "business": 34, "premium_economy": 24, "economy": 266 }
|
||||
},
|
||||
"connectivity": {
|
||||
"wifi": "high-speed",
|
||||
"wifi_provider": "Starlink",
|
||||
"satellite": true
|
||||
},
|
||||
"tracking": {
|
||||
"first_seen": "2025-01-15",
|
||||
"last_seen": "2026-02-04",
|
||||
"total_flights": 1250
|
||||
},
|
||||
"history": [
|
||||
{
|
||||
"timestamp": "2026-01-20",
|
||||
"property": "connectivity.wifi",
|
||||
"old_value": "low-speed",
|
||||
"new_value": "high-speed",
|
||||
"source": "airline_api"
|
||||
}
|
||||
]
|
||||
}
|
||||
\`\`\`
|
||||
|
||||
---
|
||||
|
||||
## 🤝 Contributing
|
||||
|
||||
### Daily Updates
|
||||
|
||||
Community members are encouraged to run the update script daily:
|
||||
|
||||
1. Fork this repo
|
||||
2. Set your \`AFKLM_API_KEY\`
|
||||
3. Run \`node fleet-update.js --airline AF\` and \`--airline KL\`
|
||||
4. Run \`node generate-readme.js\` to update stats
|
||||
5. Submit a PR
|
||||
|
||||
### API Key
|
||||
|
||||
Get a free API key at [developer.airfranceklm.com](https://developer.airfranceklm.com)
|
||||
|
||||
---
|
||||
|
||||
## 📋 Schema Version
|
||||
|
||||
Current: **1.0.0**
|
||||
|
||||
---
|
||||
|
||||
## 📄 License
|
||||
|
||||
Under MIT License
|
||||
|
||||
---
|
||||
|
||||
Made with ✈️ by the aviation community
|
||||
`;
|
||||
}
|
||||
|
||||
// Main
|
||||
function main() {
|
||||
console.log('📊 Generating README with fleet statistics...\n');
|
||||
|
||||
const airlines = loadAirlines();
|
||||
|
||||
// Show summary
|
||||
for (const [code, data] of Object.entries(airlines)) {
|
||||
const info = AIRLINE_INFO[code] || { name: code };
|
||||
const wifi = getWifiStats(data.aircraft);
|
||||
console.log(`${info.name}: ${wifi.total} aircraft, ${wifi.highSpeed} Starlink (${wifi.highSpeedPercent}%)`);
|
||||
}
|
||||
|
||||
// Generate and save README
|
||||
const readme = generateReadme(airlines);
|
||||
const readmePath = path.join(__dirname, 'README.md');
|
||||
fs.writeFileSync(readmePath, readme);
|
||||
|
||||
console.log(`\n✅ README.md updated!`);
|
||||
}
|
||||
|
||||
main();
|
||||
|
||||
Executable
+38
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"name": "fleet-catalog",
|
||||
"version": "1.0.0",
|
||||
"description": "Open-source catalog of airline fleets with historical tracking",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"update:af": "node fleet-update.js --airline AF",
|
||||
"update:kl": "node fleet-update.js --airline KL",
|
||||
"update:all": "node fleet-update.js --airline AF && node fleet-update.js --airline KL && node generate-readme.js",
|
||||
"update:af:dry": "node fleet-update.js --airline AF --dry-run",
|
||||
"update:kl:dry": "node fleet-update.js --airline KL --dry-run",
|
||||
"bootstrap:af": "node fleet-update.js --airline AF --bootstrap",
|
||||
"bootstrap:kl": "node fleet-update.js --airline KL --bootstrap",
|
||||
"readme": "node generate-readme.js",
|
||||
"validate": "node scripts/validate.js"
|
||||
},
|
||||
"keywords": [
|
||||
"aviation",
|
||||
"airlines",
|
||||
"fleet",
|
||||
"aircraft",
|
||||
"tracking"
|
||||
],
|
||||
"license": "MIT",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/fleet-catalog/fleet-catalog"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
},
|
||||
"files": [
|
||||
"airlines/",
|
||||
"schema/",
|
||||
"reference/"
|
||||
]
|
||||
}
|
||||
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
{
|
||||
"$schema": "../schema/reference.schema.json",
|
||||
"description": "Cabin class codes used in seat configuration strings",
|
||||
"codes": [
|
||||
{
|
||||
"code": "P",
|
||||
"class": "first",
|
||||
"name": "First Class / La Première",
|
||||
"notes": "Premium first class, used by Air France for La Première"
|
||||
},
|
||||
{
|
||||
"code": "F",
|
||||
"class": "first",
|
||||
"name": "First Class",
|
||||
"notes": "Traditional first class"
|
||||
},
|
||||
{
|
||||
"code": "J",
|
||||
"class": "business",
|
||||
"name": "Business Class",
|
||||
"notes": "Standard business class code"
|
||||
},
|
||||
{
|
||||
"code": "C",
|
||||
"class": "business",
|
||||
"name": "Business Class",
|
||||
"notes": "Alternative business class code, sometimes used for intra-European business"
|
||||
},
|
||||
{
|
||||
"code": "W",
|
||||
"class": "premium_economy",
|
||||
"name": "Premium Economy",
|
||||
"notes": "Premium economy class"
|
||||
},
|
||||
{
|
||||
"code": "Y",
|
||||
"class": "economy",
|
||||
"name": "Economy Class",
|
||||
"notes": "Standard economy class"
|
||||
}
|
||||
],
|
||||
"parsing_notes": [
|
||||
"Configuration strings follow format: [CLASS_CODE][SEAT_COUNT]",
|
||||
"Seat count is typically 2-3 digits (e.g., J034, Y266, or J34, Y266)",
|
||||
"Multiple classes are concatenated: P004J058W028Y206",
|
||||
"Parse left-to-right, extracting each class code followed by its count"
|
||||
],
|
||||
"examples": [
|
||||
{
|
||||
"configuration": "Y148",
|
||||
"parsed": { "economy": 148 },
|
||||
"total": 148,
|
||||
"description": "Single-class economy (e.g., A220)"
|
||||
},
|
||||
{
|
||||
"configuration": "J034W024Y266",
|
||||
"parsed": { "business": 34, "premium_economy": 24, "economy": 266 },
|
||||
"total": 324,
|
||||
"description": "Three-class long-haul (e.g., A350-900)"
|
||||
},
|
||||
{
|
||||
"configuration": "P004J058W028Y206",
|
||||
"parsed": { "first": 4, "business": 58, "premium_economy": 28, "economy": 206 },
|
||||
"total": 296,
|
||||
"description": "Four-class with La Première (e.g., 777-300ER)"
|
||||
},
|
||||
{
|
||||
"configuration": "C108Y066",
|
||||
"parsed": { "business": 108, "economy": 66 },
|
||||
"total": 174,
|
||||
"description": "Two-class short-haul with business (e.g., A320)"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Executable
+87
@@ -0,0 +1,87 @@
|
||||
{
|
||||
"$schema": "../schema/reference.schema.json",
|
||||
"description": "Known WiFi providers and their characteristics",
|
||||
"providers": [
|
||||
{
|
||||
"id": "starlink",
|
||||
"name": "Starlink",
|
||||
"company": "SpaceX",
|
||||
"technology": "LEO satellite",
|
||||
"speed_tier": "high-speed",
|
||||
"typical_speed_mbps": "50-200",
|
||||
"coverage": "global",
|
||||
"notes": "Low-earth orbit constellation, low latency"
|
||||
},
|
||||
{
|
||||
"id": "viasat",
|
||||
"name": "Viasat",
|
||||
"company": "Viasat Inc.",
|
||||
"technology": "Ka-band GEO satellite",
|
||||
"speed_tier": "high-speed",
|
||||
"typical_speed_mbps": "12-100",
|
||||
"coverage": "regional",
|
||||
"notes": "ViaSat-1, ViaSat-2, ViaSat-3 satellites"
|
||||
},
|
||||
{
|
||||
"id": "gogo_2ku",
|
||||
"name": "Gogo 2Ku",
|
||||
"company": "Gogo",
|
||||
"technology": "Dual Ku-band satellite",
|
||||
"speed_tier": "high-speed",
|
||||
"typical_speed_mbps": "15-70",
|
||||
"coverage": "global",
|
||||
"notes": "Dual antenna system for better coverage"
|
||||
},
|
||||
{
|
||||
"id": "gogo_atg",
|
||||
"name": "Gogo ATG",
|
||||
"company": "Gogo",
|
||||
"technology": "Air-to-ground",
|
||||
"speed_tier": "low-speed",
|
||||
"typical_speed_mbps": "3-10",
|
||||
"coverage": "continental_us",
|
||||
"notes": "Ground-based towers, US domestic only"
|
||||
},
|
||||
{
|
||||
"id": "panasonic_ku",
|
||||
"name": "Panasonic Ku-band",
|
||||
"company": "Panasonic Avionics",
|
||||
"technology": "Ku-band satellite",
|
||||
"speed_tier": "low-speed",
|
||||
"typical_speed_mbps": "5-20",
|
||||
"coverage": "global",
|
||||
"notes": "eXConnect service"
|
||||
},
|
||||
{
|
||||
"id": "inmarsat_gx",
|
||||
"name": "Inmarsat GX Aviation",
|
||||
"company": "Inmarsat",
|
||||
"technology": "Ka-band GEO satellite",
|
||||
"speed_tier": "high-speed",
|
||||
"typical_speed_mbps": "15-50",
|
||||
"coverage": "global",
|
||||
"notes": "Global Xpress network"
|
||||
},
|
||||
{
|
||||
"id": "anuvu",
|
||||
"name": "Anuvu",
|
||||
"company": "Anuvu (formerly Global Eagle)",
|
||||
"technology": "Ku-band satellite",
|
||||
"speed_tier": "low-speed",
|
||||
"typical_speed_mbps": "5-15",
|
||||
"coverage": "regional",
|
||||
"notes": "Formerly Global Eagle Entertainment"
|
||||
},
|
||||
{
|
||||
"id": "thales_flexvue",
|
||||
"name": "Thales FlexVue",
|
||||
"company": "Thales",
|
||||
"technology": "Ku/Ka-band satellite",
|
||||
"speed_tier": "high-speed",
|
||||
"typical_speed_mbps": "20-50",
|
||||
"coverage": "global",
|
||||
"notes": "Part of Thales InFlyt Experience"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Executable
+333
@@ -0,0 +1,333 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://github.com/fleet-catalog/schema/aircraft.schema.json",
|
||||
"title": "Airline Fleet Catalog",
|
||||
"description": "Standardized schema for tracking airline fleet properties across multiple carriers",
|
||||
"type": "object",
|
||||
"required": ["schema_version", "airline", "generated_at", "aircraft"],
|
||||
"properties": {
|
||||
"schema_version": {
|
||||
"type": "string",
|
||||
"description": "Semantic version of the schema",
|
||||
"pattern": "^\\d+\\.\\d+\\.\\d+$"
|
||||
},
|
||||
"airline": {
|
||||
"type": "object",
|
||||
"required": ["iata_code", "name"],
|
||||
"properties": {
|
||||
"iata_code": {
|
||||
"type": "string",
|
||||
"description": "2-letter IATA airline code",
|
||||
"pattern": "^[A-Z0-9]{2}$"
|
||||
},
|
||||
"icao_code": {
|
||||
"type": ["string", "null"],
|
||||
"description": "3-letter ICAO airline code",
|
||||
"pattern": "^[A-Z]{3}$"
|
||||
},
|
||||
"name": {
|
||||
"type": "string",
|
||||
"description": "Full airline name"
|
||||
},
|
||||
"country": {
|
||||
"type": ["string", "null"],
|
||||
"description": "ISO 3166-1 alpha-2 country code"
|
||||
}
|
||||
}
|
||||
},
|
||||
"generated_at": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
"description": "ISO 8601 timestamp when this file was generated"
|
||||
},
|
||||
"aircraft_count": {
|
||||
"type": "integer",
|
||||
"description": "Total number of aircraft in this file"
|
||||
},
|
||||
"aircraft": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/aircraft"
|
||||
}
|
||||
}
|
||||
},
|
||||
"$defs": {
|
||||
"aircraft": {
|
||||
"type": "object",
|
||||
"required": ["registration"],
|
||||
"properties": {
|
||||
"registration": {
|
||||
"type": "string",
|
||||
"description": "Aircraft registration (tail number) without dashes"
|
||||
},
|
||||
"icao24": {
|
||||
"type": ["string", "null"],
|
||||
"description": "24-bit ICAO Mode-S transponder address in hexadecimal"
|
||||
},
|
||||
"aircraft_type": {
|
||||
"$ref": "#/$defs/aircraft_type"
|
||||
},
|
||||
"operator": {
|
||||
"$ref": "#/$defs/operator"
|
||||
},
|
||||
"cabin": {
|
||||
"$ref": "#/$defs/cabin"
|
||||
},
|
||||
"connectivity": {
|
||||
"$ref": "#/$defs/connectivity"
|
||||
},
|
||||
"ife": {
|
||||
"$ref": "#/$defs/ife"
|
||||
},
|
||||
"status": {
|
||||
"type": "string",
|
||||
"enum": ["active", "stored", "maintenance", "retired"],
|
||||
"description": "Current operational status"
|
||||
},
|
||||
"tracking": {
|
||||
"$ref": "#/$defs/tracking"
|
||||
},
|
||||
"metadata": {
|
||||
"$ref": "#/$defs/metadata"
|
||||
},
|
||||
"history": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/history_entry"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"aircraft_type": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"iata_code": {
|
||||
"type": ["string", "null"],
|
||||
"description": "IATA aircraft type code (e.g., 77W, 359)"
|
||||
},
|
||||
"icao_code": {
|
||||
"type": ["string", "null"],
|
||||
"description": "ICAO aircraft type designator (e.g., B77W, A359)"
|
||||
},
|
||||
"manufacturer": {
|
||||
"type": ["string", "null"],
|
||||
"description": "Aircraft manufacturer (Airbus, Boeing, Embraer, etc.)"
|
||||
},
|
||||
"model": {
|
||||
"type": ["string", "null"],
|
||||
"description": "Aircraft model (A350, 777, etc.)"
|
||||
},
|
||||
"variant": {
|
||||
"type": ["string", "null"],
|
||||
"description": "Aircraft variant (900, 300ER, etc.)"
|
||||
},
|
||||
"full_name": {
|
||||
"type": ["string", "null"],
|
||||
"description": "Full aircraft type name"
|
||||
}
|
||||
}
|
||||
},
|
||||
"operator": {
|
||||
"type": "object",
|
||||
"description": "Operational details specific to this aircraft",
|
||||
"properties": {
|
||||
"sub_fleet_code": {
|
||||
"type": ["string", "null"],
|
||||
"description": "Internal sub-fleet code (e.g., AB, CA, AR)"
|
||||
},
|
||||
"cabin_crew_employer": {
|
||||
"type": ["string", "null"],
|
||||
"description": "Airline code of cabin crew employer"
|
||||
},
|
||||
"cockpit_crew_employer": {
|
||||
"type": ["string", "null"],
|
||||
"description": "Airline code of cockpit crew employer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"cabin": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"physical_configuration": {
|
||||
"type": ["string", "null"],
|
||||
"description": "Physical seat configuration code (e.g., J034W024Y266)"
|
||||
},
|
||||
"operational_configuration": {
|
||||
"type": ["string", "null"],
|
||||
"description": "Operational/saleable seat configuration"
|
||||
},
|
||||
"saleable_configuration": {
|
||||
"type": ["string", "null"],
|
||||
"description": "Saleable seat configuration"
|
||||
},
|
||||
"total_seats": {
|
||||
"type": ["integer", "null"],
|
||||
"description": "Total number of passenger seats"
|
||||
},
|
||||
"classes": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"first": {
|
||||
"type": "integer",
|
||||
"description": "Number of first class seats"
|
||||
},
|
||||
"business": {
|
||||
"type": "integer",
|
||||
"description": "Number of business class seats"
|
||||
},
|
||||
"premium_economy": {
|
||||
"type": "integer",
|
||||
"description": "Number of premium economy seats"
|
||||
},
|
||||
"economy": {
|
||||
"type": "integer",
|
||||
"description": "Number of economy seats"
|
||||
}
|
||||
}
|
||||
},
|
||||
"freight_configuration": {
|
||||
"type": ["string", "null"],
|
||||
"description": "Cargo hold configuration (e.g., PP008LL012)"
|
||||
}
|
||||
}
|
||||
},
|
||||
"connectivity": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"wifi": {
|
||||
"type": "string",
|
||||
"enum": ["none", "low-speed", "high-speed"],
|
||||
"description": "WiFi availability and speed tier"
|
||||
},
|
||||
"wifi_provider": {
|
||||
"type": ["string", "null"],
|
||||
"description": "WiFi service provider (Starlink, Viasat, Gogo 2Ku, etc.)"
|
||||
},
|
||||
"satellite": {
|
||||
"type": ["boolean", "null"],
|
||||
"description": "Whether satellite connectivity is available"
|
||||
},
|
||||
"live_tv": {
|
||||
"type": ["boolean", "null"],
|
||||
"description": "Whether live TV is available"
|
||||
},
|
||||
"power_outlets": {
|
||||
"type": ["boolean", "null"],
|
||||
"description": "Whether AC power outlets are available"
|
||||
},
|
||||
"usb_ports": {
|
||||
"type": ["boolean", "null"],
|
||||
"description": "Whether USB charging ports are available"
|
||||
}
|
||||
}
|
||||
},
|
||||
"ife": {
|
||||
"type": "object",
|
||||
"description": "In-flight entertainment system",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": ["string", "null"],
|
||||
"enum": ["none", "overhead", "seatback", "streaming", "hybrid", null],
|
||||
"description": "Type of IFE system"
|
||||
},
|
||||
"personal_screens": {
|
||||
"type": ["boolean", "null"],
|
||||
"description": "Whether personal seatback screens are available"
|
||||
}
|
||||
}
|
||||
},
|
||||
"tracking": {
|
||||
"type": "object",
|
||||
"description": "Flight tracking statistics",
|
||||
"properties": {
|
||||
"first_seen": {
|
||||
"type": ["string", "null"],
|
||||
"format": "date",
|
||||
"description": "Date when aircraft was first tracked"
|
||||
},
|
||||
"last_seen": {
|
||||
"type": ["string", "null"],
|
||||
"format": "date",
|
||||
"description": "Date when aircraft was last tracked"
|
||||
},
|
||||
"total_flights": {
|
||||
"type": ["integer", "null"],
|
||||
"description": "Total number of flights tracked"
|
||||
}
|
||||
}
|
||||
},
|
||||
"metadata": {
|
||||
"type": "object",
|
||||
"description": "Additional metadata about the aircraft",
|
||||
"properties": {
|
||||
"delivery_date": {
|
||||
"type": ["string", "null"],
|
||||
"format": "date",
|
||||
"description": "Date aircraft was delivered to airline"
|
||||
},
|
||||
"msn": {
|
||||
"type": ["string", "null"],
|
||||
"description": "Manufacturer Serial Number"
|
||||
},
|
||||
"line_number": {
|
||||
"type": ["string", "null"],
|
||||
"description": "Production line number"
|
||||
},
|
||||
"production_site": {
|
||||
"type": ["string", "null"],
|
||||
"description": "Factory/production site (e.g., Toulouse, Hamburg, Mirabel)"
|
||||
},
|
||||
"engine_type": {
|
||||
"type": ["string", "null"],
|
||||
"description": "Engine model (e.g., Trent XWB-84, GE90-115B)"
|
||||
},
|
||||
"aircraft_name": {
|
||||
"type": ["string", "null"],
|
||||
"description": "Aircraft given name (e.g., 'Fort-de-France')"
|
||||
},
|
||||
"livery": {
|
||||
"type": ["string", "null"],
|
||||
"description": "Special livery or paint scheme"
|
||||
},
|
||||
"comments": {
|
||||
"type": ["string", "null"],
|
||||
"description": "Additional notes or comments"
|
||||
},
|
||||
"created_at": {
|
||||
"type": ["string", "null"],
|
||||
"format": "date-time"
|
||||
},
|
||||
"updated_at": {
|
||||
"type": ["string", "null"],
|
||||
"format": "date-time"
|
||||
}
|
||||
}
|
||||
},
|
||||
"history_entry": {
|
||||
"type": "object",
|
||||
"required": ["timestamp", "property"],
|
||||
"properties": {
|
||||
"timestamp": {
|
||||
"type": "string",
|
||||
"description": "Date or datetime when change was detected"
|
||||
},
|
||||
"property": {
|
||||
"type": "string",
|
||||
"description": "Dot-notation path to the changed property"
|
||||
},
|
||||
"old_value": {
|
||||
"description": "Previous value"
|
||||
},
|
||||
"new_value": {
|
||||
"description": "New value"
|
||||
},
|
||||
"source": {
|
||||
"type": ["string", "null"],
|
||||
"enum": ["flight_api", "airline_api", "manual", "planespotters", "community", null],
|
||||
"description": "Source of the change detection"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
#unique_regulatory_id
|
||||
# 1. read historoical and output
|
||||
# 2. read sequentially
|
||||
|
||||
# Instead of reading all csvs I can read just the latest release csv to get everything.
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
base = Path("data/faa_releasable_historical")
|
||||
for day_dir in sorted(base.glob("2024-02-*")):
|
||||
master = day_dir / "Master.txt"
|
||||
if master.exists():
|
||||
out_csv = master_txt_to_releasable_csv(master, out_dir="data/faa_releasable_historical_csv")
|
||||
print(day_dir.name, "->", out_csv)
|
||||
@@ -0,0 +1,89 @@
|
||||
from pathlib import Path
|
||||
import pandas as pd
|
||||
import re
|
||||
from derive_from_faa_master_txt import concat_faa_historical_df
|
||||
|
||||
def concatenate_aircraft_csvs(
|
||||
input_dir: Path = Path("data/concat"),
|
||||
output_dir: Path = Path("data/planequery_aircraft"),
|
||||
filename_pattern: str = r"planequery_aircraft_(\d{4}-\d{2}-\d{2})_(\d{4}-\d{2}-\d{2})\.csv"
|
||||
):
|
||||
"""
|
||||
Read all CSVs matching the pattern from input_dir in order,
|
||||
concatenate them using concat_faa_historical_df, and output a single CSV.
|
||||
|
||||
Args:
|
||||
input_dir: Directory containing the CSV files to concatenate
|
||||
output_dir: Directory where the output CSV will be saved
|
||||
filename_pattern: Regex pattern to match CSV filenames
|
||||
"""
|
||||
input_dir = Path(input_dir)
|
||||
output_dir = Path(output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Find all matching CSV files
|
||||
pattern = re.compile(filename_pattern)
|
||||
csv_files = []
|
||||
|
||||
for csv_path in sorted(input_dir.glob("*.csv")):
|
||||
match = pattern.search(csv_path.name)
|
||||
if match:
|
||||
start_date = match.group(1)
|
||||
end_date = match.group(2)
|
||||
csv_files.append((start_date, end_date, csv_path))
|
||||
|
||||
# Sort by start date, then end date
|
||||
csv_files.sort(key=lambda x: (x[0], x[1]))
|
||||
|
||||
if not csv_files:
|
||||
raise FileNotFoundError(f"No CSV files matching pattern found in {input_dir}")
|
||||
|
||||
print(f"Found {len(csv_files)} CSV files to concatenate")
|
||||
|
||||
# Read first CSV as base
|
||||
first_start_date, first_end_date, first_path = csv_files[0]
|
||||
print(f"Reading base file: {first_path.name}")
|
||||
df_base = pd.read_csv(
|
||||
first_path,
|
||||
dtype={
|
||||
'transponder_code': str,
|
||||
'unique_regulatory_id': str,
|
||||
'registrant_county': str
|
||||
}
|
||||
)
|
||||
|
||||
# Concatenate remaining CSVs
|
||||
for start_date, end_date, csv_path in csv_files[1:]:
|
||||
print(f"Concatenating: {csv_path.name}")
|
||||
df_new = pd.read_csv(
|
||||
csv_path,
|
||||
dtype={
|
||||
'transponder_code': str,
|
||||
'unique_regulatory_id': str,
|
||||
'registrant_county': str
|
||||
}
|
||||
)
|
||||
df_base = concat_faa_historical_df(df_base, df_new)
|
||||
|
||||
# Verify monotonic increasing download_date
|
||||
assert df_base['download_date'].is_monotonic_increasing, "download_date is not monotonic increasing"
|
||||
|
||||
# Output filename uses first start date and last end date
|
||||
last_start_date, last_end_date, _ = csv_files[-1]
|
||||
output_filename = f"planequery_aircraft_{first_start_date}_{last_end_date}.csv"
|
||||
output_path = output_dir / output_filename
|
||||
|
||||
print(f"Writing output to: {output_path}")
|
||||
df_base.to_csv(output_path, index=False)
|
||||
print(f"Successfully concatenated {len(csv_files)} files into {output_filename}")
|
||||
print(f"Total rows: {len(df_base)}")
|
||||
|
||||
return output_path
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Example usage - modify these paths as needed
|
||||
concatenate_aircraft_csvs(
|
||||
input_dir=Path("data/concat"),
|
||||
output_dir=Path("data/planequery_aircraft")
|
||||
)
|
||||
@@ -0,0 +1,33 @@
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
|
||||
out_dir = Path("data/faa_releasable")
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
zip_name = f"ReleasableAircraft_{date_str}.zip"
|
||||
|
||||
zip_path = out_dir / zip_name
|
||||
if not zip_path.exists():
|
||||
# URL and paths
|
||||
url = "https://registry.faa.gov/database/ReleasableAircraft.zip"
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
req = Request(
|
||||
url,
|
||||
headers={"User-Agent": "Mozilla/5.0"},
|
||||
method="GET",
|
||||
)
|
||||
|
||||
with urlopen(req, timeout=120) as r:
|
||||
body = r.read()
|
||||
zip_path.write_bytes(body)
|
||||
|
||||
OUT_ROOT = Path("data/planequery_aircraft")
|
||||
OUT_ROOT.mkdir(parents=True, exist_ok=True)
|
||||
from derive_from_faa_master_txt import convert_faa_master_txt_to_df, concat_faa_historical_df
|
||||
from get_latest_planequery_aircraft_release import get_latest_aircraft_csv_df
|
||||
df_new = convert_faa_master_txt_to_df(zip_path, date_str)
|
||||
df_base, start_date_str = get_latest_aircraft_csv_df()
|
||||
df_base = concat_faa_historical_df(df_base, df_new)
|
||||
assert df_base['download_date'].is_monotonic_increasing, "download_date is not monotonic increasing"
|
||||
df_base.to_csv(OUT_ROOT / f"planequery_aircraft_{start_date_str}_{date_str}.csv", index=False)
|
||||
@@ -0,0 +1,127 @@
|
||||
from pathlib import Path
|
||||
import zipfile
|
||||
import pandas as pd
|
||||
from faa_aircraft_registry import read
|
||||
|
||||
def convert_faa_master_txt_to_df(zip_path: Path, date: str):
|
||||
with zipfile.ZipFile(zip_path) as z:
|
||||
registrations = read(z)
|
||||
|
||||
df = pd.DataFrame(registrations['master'].values())
|
||||
|
||||
df.insert(0, "download_date", date)
|
||||
|
||||
registrant = pd.json_normalize(df["registrant"]).add_prefix("registrant_")
|
||||
df = df.drop(columns="registrant").join(registrant)
|
||||
|
||||
# Move transponder_code_hex to second column (after registration_number)
|
||||
cols = df.columns.tolist()
|
||||
cols.remove("transponder_code_hex")
|
||||
cols.insert(1, "transponder_code_hex")
|
||||
df = df[cols]
|
||||
|
||||
df = df.rename(columns={"aircraft_type": "aircraft_type_2"})
|
||||
aircraft = pd.json_normalize(df["aircraft"].where(df["aircraft"].notna(), {})).add_prefix("aircraft_")
|
||||
df = df.drop(columns="aircraft").join(aircraft)
|
||||
df = df.rename(columns={"engine_type": "engine_type_2"})
|
||||
engine = pd.json_normalize(df["engine"].where(df["engine"].notna(), {})).add_prefix("engine_")
|
||||
df = df.drop(columns="engine").join(engine)
|
||||
certification = pd.json_normalize(df["certification"].where(df["certification"].notna(), {})).add_prefix("certificate_")
|
||||
df = df.drop(columns="certification").join(certification)
|
||||
|
||||
# Create planequery_airframe_id
|
||||
df["planequery_airframe_id"] = (
|
||||
normalize(df["aircraft_manufacturer"])
|
||||
+ "|"
|
||||
+ normalize(df["aircraft_model"])
|
||||
+ "|"
|
||||
+ normalize(df["serial_number"])
|
||||
)
|
||||
|
||||
# Move planequery_airframe_id to come after registration_number
|
||||
cols = df.columns.tolist()
|
||||
cols.remove("planequery_airframe_id")
|
||||
reg_idx = cols.index("registration_number")
|
||||
cols.insert(reg_idx + 1, "planequery_airframe_id")
|
||||
df = df[cols]
|
||||
|
||||
# Convert all NaN to empty strings
|
||||
df = df.fillna("")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
|
||||
def normalize(s: pd.Series) -> pd.Series:
|
||||
return (
|
||||
s.fillna("")
|
||||
.astype(str)
|
||||
.str.upper()
|
||||
.str.strip()
|
||||
# collapse whitespace
|
||||
.str.replace(r"\s+", " ", regex=True)
|
||||
# remove characters that cause false mismatches
|
||||
.str.replace(r"[^\w\-]", "", regex=True)
|
||||
)
|
||||
|
||||
|
||||
def concat_faa_historical_df(df_base, df_new):
|
||||
df_new = df_new[df_base.columns]
|
||||
df_base = pd.concat([df_base, df_new], ignore_index=True)
|
||||
|
||||
CONTENT_COLS = [
|
||||
c for c in df_base.columns
|
||||
if c not in {"download_date"}
|
||||
]
|
||||
|
||||
# Normalize values to handle numeric type, formatting, and list ordering differences
|
||||
def normalize_series(series):
|
||||
def normalize_value(val):
|
||||
# Handle lists (sort them for consistent comparison)
|
||||
if isinstance(val, list):
|
||||
return "|".join(sorted(str(v) for v in val))
|
||||
|
||||
# Convert to string
|
||||
val_str = str(val).strip()
|
||||
|
||||
# Handle empty strings
|
||||
if val_str == "" or val_str == "nan":
|
||||
return ""
|
||||
|
||||
# Check if it looks like a list representation (starts with [ )
|
||||
if val_str.startswith('[') and val_str.endswith(']'):
|
||||
try:
|
||||
# Try to parse as a list-like string
|
||||
import ast
|
||||
parsed = ast.literal_eval(val_str)
|
||||
if isinstance(parsed, list):
|
||||
return "|".join(sorted(str(v) for v in parsed))
|
||||
except (ValueError, SyntaxError):
|
||||
pass # Not a valid list, continue to other checks
|
||||
|
||||
# Try to normalize as number
|
||||
try:
|
||||
# Remove leading zeros and convert float/int representations
|
||||
num_val = float(val_str)
|
||||
# If it's a whole number, return as int string (no .0)
|
||||
if num_val == int(num_val):
|
||||
return str(int(num_val))
|
||||
# Otherwise return as float
|
||||
return str(num_val)
|
||||
except (ValueError, OverflowError):
|
||||
# Not a number, return as-is
|
||||
return val_str
|
||||
|
||||
return series.apply(normalize_value)
|
||||
|
||||
df_base["row_fingerprint"] = (
|
||||
df_base[CONTENT_COLS]
|
||||
.apply(normalize_series, axis=0)
|
||||
.apply(lambda row: "|".join(row), axis=1)
|
||||
)
|
||||
|
||||
df_base = df_base.drop_duplicates(
|
||||
subset=["row_fingerprint"],
|
||||
keep="first"
|
||||
).drop(columns=["row_fingerprint"])
|
||||
return df_base
|
||||
+86
-33
@@ -1,63 +1,116 @@
|
||||
'''Generated with ChatGPT 5.2 prompt
|
||||
scrape-faa-releasable-aircraft
|
||||
Every day it creates a new commit that takes ReleasableAircraft zip from FAA takes Master.txt to make these files (it does this so that all files stay under 100mb). For every commit day I want to recombine all the files into one Master.txt again. It has data/commits since 2023.
|
||||
scrape-faa-releasable-aircraft % ls
|
||||
ACFTREF.txt DOCINDEX.txt MASTER-1.txt MASTER-3.txt MASTER-5.txt MASTER-7.txt MASTER-9.txt RESERVED.txt
|
||||
DEALER.txt ENGINE.txt MASTER-2.txt MASTER-4.txt MASTER-6.txt MASTER-8.txt README.md ardata.pdf
|
||||
'''
|
||||
"""
|
||||
For each commit-day in Feb 2024 (last commit per day):
|
||||
- Write ALL FAA text files from that commit into: data/faa_releasable_historical/YYYY-MM-DD/
|
||||
ACFTREF.txt, DEALER.txt, DOCINDEX.txt, ENGINE.txt, RESERVED.txt
|
||||
- Recombine MASTER-*.txt into Master.txt
|
||||
- Produce Master.csv via convert_faa_master_txt_to_csv
|
||||
|
||||
Assumes the non-master files are present in every commit.
|
||||
"""
|
||||
import subprocess, re
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
from collections import OrderedDict
|
||||
from derive_from_faa_master_txt import convert_faa_master_txt_to_df, concat_faa_historical_df
|
||||
import zipfile
|
||||
import pandas as pd
|
||||
import argparse
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
def run(*args: str) -> str:
|
||||
return subprocess.check_output(args, text=True).strip()
|
||||
# Parse command line arguments
|
||||
parser = argparse.ArgumentParser(description="Process historical FAA data from git commits")
|
||||
parser.add_argument("since", help="Start date (YYYY-MM-DD)")
|
||||
parser.add_argument("until", help="End date (YYYY-MM-DD)")
|
||||
args = parser.parse_args()
|
||||
|
||||
# Get commits that touched any MASTER-*.txt, oldest -> newest
|
||||
log = run("git", "log", "--reverse", "--format=%H %cs", "--", ".")
|
||||
# If you want to restrict to only commits that touched the master parts, use:
|
||||
# log = run("git", "log", "--reverse", "--format=%H %cs", "--", "MASTER-1.txt")
|
||||
# Clone repository if it doesn't exist
|
||||
REPO = Path("data/scrape-faa-releasable-aircraft")
|
||||
OUT_ROOT = Path("data/faa_releasable_historical")
|
||||
OUT_ROOT.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def run_git_text(*args: str) -> str:
|
||||
return subprocess.check_output(["git", "-C", str(REPO), *args], text=True).strip()
|
||||
|
||||
def run_git_bytes(*args: str) -> bytes:
|
||||
return subprocess.check_output(["git", "-C", str(REPO), *args])
|
||||
|
||||
# Parse dates and adjust --since to the day before
|
||||
since_date = datetime.strptime(args.since, "%Y-%m-%d")
|
||||
adjusted_since = (since_date - timedelta(days=1)).strftime("%Y-%m-%d")
|
||||
|
||||
# All commits in specified date range (oldest -> newest)
|
||||
log = run_git_text(
|
||||
"log",
|
||||
"--reverse",
|
||||
"--format=%H %cs",
|
||||
f"--since={adjusted_since}",
|
||||
f"--until={args.until}",
|
||||
)
|
||||
lines = [ln for ln in log.splitlines() if ln.strip()]
|
||||
if not lines:
|
||||
raise SystemExit("No commits found.")
|
||||
raise SystemExit(f"No commits found between {args.since} and {args.until}.")
|
||||
|
||||
# Map date -> last commit SHA on that date (Ordered by history)
|
||||
# date -> last SHA that day
|
||||
date_to_sha = OrderedDict()
|
||||
for ln in lines:
|
||||
sha, date = ln.split()
|
||||
# keep last SHA per day
|
||||
date_to_sha[date] = sha
|
||||
|
||||
out_root = Path("out_master_by_day")
|
||||
out_root.mkdir(exist_ok=True)
|
||||
|
||||
OTHER_FILES = ["ACFTREF.txt", "DEALER.txt", "DOCINDEX.txt", "ENGINE.txt", "RESERVED.txt"]
|
||||
master_re = re.compile(r"^MASTER-(\d+)\.txt$")
|
||||
|
||||
df_base = pd.DataFrame()
|
||||
start_date = None
|
||||
end_date = None
|
||||
for date, sha in date_to_sha.items():
|
||||
# list files at this commit, filter MASTER-*.txt in repo root
|
||||
names = run("git", "ls-tree", "--name-only", sha).splitlines()
|
||||
if start_date is None:
|
||||
start_date = date
|
||||
end_date = date
|
||||
day_dir = OUT_ROOT / date
|
||||
day_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Write auxiliary files (assumed present)
|
||||
for fname in OTHER_FILES:
|
||||
(day_dir / fname).write_bytes(run_git_bytes("show", f"{sha}:{fname}"))
|
||||
|
||||
# Recombine MASTER parts
|
||||
names = run_git_text("ls-tree", "--name-only", sha).splitlines()
|
||||
parts = []
|
||||
for n in names:
|
||||
m = master_re.match(n)
|
||||
if m:
|
||||
parts.append((int(m.group(1)), n))
|
||||
parts.sort()
|
||||
|
||||
if not parts:
|
||||
# no master parts in that commit/day; skip
|
||||
continue
|
||||
raise RuntimeError(f"{date} {sha[:7]}: no MASTER-*.txt parts found")
|
||||
|
||||
day_dir = out_root / date
|
||||
day_dir.mkdir(parents=True, exist_ok=True)
|
||||
out_path = day_dir / "Master.txt"
|
||||
|
||||
with out_path.open("wb") as w:
|
||||
master_path = day_dir / "MASTER.txt"
|
||||
with master_path.open("wb") as w:
|
||||
for _, fname in parts:
|
||||
data = subprocess.check_output(["git", "show", f"{sha}:{fname}"])
|
||||
data = run_git_bytes("show", f"{sha}:{fname}")
|
||||
w.write(data)
|
||||
if data and not data.endswith(b"\n"):
|
||||
w.write(b"\n")
|
||||
|
||||
print(f"{date} {sha[:7]} -> {out_path} ({len(parts)} parts)")
|
||||
# 3) Zip the day's files
|
||||
zip_path = day_dir / f"ReleasableAircraft.zip"
|
||||
with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as z:
|
||||
for p in day_dir.iterdir():
|
||||
z.write(p, arcname=p.name)
|
||||
|
||||
print(f"\nDone. Output root: {out_root.resolve()}")
|
||||
print(f"{date} {sha[:7]} -> {day_dir} (master parts: {len(parts)})")
|
||||
# 4) Convert ZIP -> CSV
|
||||
df_new = convert_faa_master_txt_to_df(zip_path, date)
|
||||
if df_base.empty:
|
||||
df_base = df_new
|
||||
print(len(df_base), "total entries so far")
|
||||
# Delete all files in the day directory
|
||||
shutil.rmtree(day_dir)
|
||||
continue
|
||||
|
||||
df_base = concat_faa_historical_df(df_base, df_new)
|
||||
shutil.rmtree(day_dir)
|
||||
print(len(df_base), "total entries so far")
|
||||
|
||||
assert df_base['download_date'].is_monotonic_increasing, "download_date is not monotonic increasing"
|
||||
df_base.to_csv(OUT_ROOT / f"planequery_aircraft_{start_date}_{end_date}.csv", index=False)
|
||||
# TODO: get average number of new rows per day.
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Optional
|
||||
import re
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import json
|
||||
|
||||
|
||||
REPO = "PlaneQuery/planequery-aircraft"
|
||||
LATEST_RELEASE_URL = f"https://api.github.com/repos/{REPO}/releases/latest"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReleaseAsset:
|
||||
name: str
|
||||
download_url: str
|
||||
size: int # bytes
|
||||
|
||||
|
||||
def _http_get_json(url: str, headers: dict[str, str]) -> dict:
|
||||
req = urllib.request.Request(url, headers=headers, method="GET")
|
||||
with urllib.request.urlopen(req, timeout=120) as resp:
|
||||
data = resp.read()
|
||||
return json.loads(data.decode("utf-8"))
|
||||
|
||||
|
||||
def get_latest_release_assets(repo: str = REPO, github_token: Optional[str] = None) -> list[ReleaseAsset]:
|
||||
url = f"https://api.github.com/repos/{repo}/releases/latest"
|
||||
headers = {
|
||||
"Accept": "application/vnd.github+json",
|
||||
"User-Agent": "planequery-aircraft-downloader/1.0",
|
||||
}
|
||||
if github_token:
|
||||
headers["Authorization"] = f"Bearer {github_token}"
|
||||
|
||||
payload = _http_get_json(url, headers=headers)
|
||||
assets = []
|
||||
for a in payload.get("assets", []):
|
||||
assets.append(
|
||||
ReleaseAsset(
|
||||
name=a["name"],
|
||||
download_url=a["browser_download_url"],
|
||||
size=int(a.get("size", 0)),
|
||||
)
|
||||
)
|
||||
return assets
|
||||
|
||||
|
||||
def pick_asset(
|
||||
assets: Iterable[ReleaseAsset],
|
||||
*,
|
||||
exact_name: Optional[str] = None,
|
||||
name_regex: Optional[str] = None,
|
||||
) -> ReleaseAsset:
|
||||
assets = list(assets)
|
||||
|
||||
if exact_name:
|
||||
for a in assets:
|
||||
if a.name == exact_name:
|
||||
return a
|
||||
raise FileNotFoundError(f"No asset exactly named {exact_name!r}. Available: {[a.name for a in assets]}")
|
||||
|
||||
if name_regex:
|
||||
rx = re.compile(name_regex)
|
||||
matches = [a for a in assets if rx.search(a.name)]
|
||||
if not matches:
|
||||
raise FileNotFoundError(f"No asset matched regex {name_regex!r}. Available: {[a.name for a in assets]}")
|
||||
if len(matches) > 1:
|
||||
raise FileExistsError(f"Regex {name_regex!r} matched multiple assets: {[m.name for m in matches]}")
|
||||
return matches[0]
|
||||
|
||||
raise ValueError("Provide either exact_name=... or name_regex=...")
|
||||
|
||||
|
||||
def download_asset(asset: ReleaseAsset, out_path: Path, github_token: Optional[str] = None) -> Path:
|
||||
out_path = Path(out_path)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
headers = {
|
||||
"User-Agent": "planequery-aircraft-downloader/1.0",
|
||||
"Accept": "application/octet-stream",
|
||||
}
|
||||
if github_token:
|
||||
headers["Authorization"] = f"Bearer {github_token}"
|
||||
|
||||
req = urllib.request.Request(asset.download_url, headers=headers, method="GET")
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=300) as resp, out_path.open("wb") as f:
|
||||
# Stream download
|
||||
while True:
|
||||
chunk = resp.read(1024 * 1024) # 1 MiB
|
||||
if not chunk:
|
||||
break
|
||||
f.write(chunk)
|
||||
except urllib.error.HTTPError as e:
|
||||
body = e.read().decode("utf-8", errors="replace") if hasattr(e, "read") else ""
|
||||
raise RuntimeError(f"HTTPError {e.code} downloading {asset.name}: {body[:500]}") from e
|
||||
|
||||
return out_path
|
||||
|
||||
|
||||
def download_latest_aircraft_csv(
|
||||
output_dir: Path = Path("downloads"),
|
||||
github_token: Optional[str] = None,
|
||||
repo: str = REPO,
|
||||
) -> Path:
|
||||
"""
|
||||
Download the latest planequery_aircraft_*.csv file from the latest GitHub release.
|
||||
|
||||
Args:
|
||||
output_dir: Directory to save the downloaded file (default: "downloads")
|
||||
github_token: Optional GitHub token for authentication
|
||||
repo: GitHub repository in format "owner/repo" (default: REPO)
|
||||
|
||||
Returns:
|
||||
Path to the downloaded file
|
||||
"""
|
||||
assets = get_latest_release_assets(repo, github_token=github_token)
|
||||
asset = pick_asset(assets, name_regex=r"^planequery_aircraft_.*\.csv$")
|
||||
saved_to = download_asset(asset, output_dir / asset.name, github_token=github_token)
|
||||
print(f"Downloaded: {asset.name} ({asset.size} bytes) -> {saved_to}")
|
||||
return saved_to
|
||||
|
||||
def get_latest_aircraft_csv_df():
|
||||
csv_path = download_latest_aircraft_csv()
|
||||
import pandas as pd
|
||||
df = pd.read_csv(csv_path, dtype={'transponder_code': str,
|
||||
'unique_regulatory_id': str,
|
||||
'registrant_county': str})
|
||||
df = df.fillna("")
|
||||
# Extract date from filename pattern: planequery_aircraft_{date}_{date}.csv
|
||||
match = re.search(r"planequery_aircraft_(\d{4}-\d{2}-\d{2})_", str(csv_path))
|
||||
if not match:
|
||||
raise ValueError(f"Could not extract date from filename: {csv_path.name}")
|
||||
|
||||
date_str = match.group(1)
|
||||
return df, date_str
|
||||
|
||||
if __name__ == "__main__":
|
||||
download_latest_aircraft_csv()
|
||||
@@ -1,48 +0,0 @@
|
||||
from faa_aircraft_registry import read
|
||||
import pandas as pd
|
||||
import zipfile
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
|
||||
out_dir = Path("data/faa_releasable")
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
zip_name = f"ReleasableAircraft_{date_str}.zip"
|
||||
csv_name = f"ReleasableAircraft_{date_str}.csv"
|
||||
|
||||
zip_path = out_dir / zip_name
|
||||
csv_path = out_dir / csv_name
|
||||
|
||||
# URL and paths
|
||||
url = "https://registry.faa.gov/database/ReleasableAircraft.zip"
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
req = Request(
|
||||
url,
|
||||
headers={"User-Agent": "Mozilla/5.0"},
|
||||
method="GET",
|
||||
)
|
||||
|
||||
with urlopen(req, timeout=120) as r:
|
||||
body = r.read()
|
||||
zip_path.write_bytes(body)
|
||||
|
||||
with zipfile.ZipFile(zip_path) as z:
|
||||
registrations = read(z)
|
||||
|
||||
df = pd.DataFrame(registrations['master'].values())
|
||||
col = "transponder_code_hex"
|
||||
df = df[[col] + [c for c in df.columns if c != col]]
|
||||
df = df.rename(columns={"transponder_code_hex": "icao"})
|
||||
registrant = pd.json_normalize(df["registrant"]).add_prefix("registrant_")
|
||||
df = df.drop(columns="registrant").join(registrant)
|
||||
df = df.rename(columns={"aircraft_type": "aircraft_type_2"})
|
||||
aircraft = pd.json_normalize(df["aircraft"]).add_prefix("aircraft_")
|
||||
df = df.drop(columns="aircraft").join(aircraft)
|
||||
df = df.rename(columns={"engine_type": "engine_type_2"})
|
||||
engine = pd.json_normalize(df["engine"].where(df["engine"].notna(), {})).add_prefix("engine_")
|
||||
df = df.drop(columns="engine").join(engine)
|
||||
df = df.sort_values(by=["icao"])
|
||||
df.to_csv(csv_path, index=False)
|
||||
|
||||
Reference in New Issue
Block a user