CalvinBackup/OpenAirframes
1
0
Fork 0
mirror of https://github.com/PlaneQuery/OpenAirframes.git synced 2026-07-25 11:10:49 +02:00
Code Issues Packages Projects Releases Wiki Activity

src/adsb/run_local.py works

Browse Source
This commit is contained in:
ggman12
2026-02-16 17:45:31 -05:00
parent 9cb4c5045b
commit 13432068e6
1 changed files with 35 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/adsb/run_local.py
+35 -52
View File
@@ -8,10 +8,10 @@ Usage:
# Single day (yesterday by default) # Single day (yesterday by default)
python -m src.adsb.run_local python -m src.adsb.run_local
# Single day (specific date) # Single day (specific date, processes 2024-01-15 only)
python -m src.adsb.run_local 2024-01-15 python -m src.adsb.run_local 2024-01-15 2024-01-16
# Date range (inclusive) # Date range (end date is exclusive)
python -m src.adsb.run_local 2024-01-01 2024-01-07 python -m src.adsb.run_local 2024-01-01 2024-01-07
""" """
import argparse import argparse
@@ -38,12 +38,12 @@ def main():
parser.add_argument( parser.add_argument(
"start_date", "start_date",
nargs="?", nargs="?",
help="Start date (YYYY-MM-DD). Default: yesterday" help="Start date (YYYY-MM-DD, inclusive). Default: yesterday"
) )
parser.add_argument( parser.add_argument(
"end_date", "end_date",
nargs="?", nargs="?",
help="End date (YYYY-MM-DD, inclusive). If omitted, processes single day" help="End date (YYYY-MM-DD, exclusive). If omitted, processes single day (start_date + 1)"
) )
parser.add_argument( parser.add_argument(
"--chunks", "--chunks",
@@ -70,39 +70,35 @@ def main():
else: else:
start_date = datetime.utcnow() - timedelta(days=1) start_date = datetime.utcnow() - timedelta(days=1)
end_date = None
if args.end_date: if args.end_date:
end_date = datetime.strptime(args.end_date, "%Y-%m-%d") end_date = datetime.strptime(args.end_date, "%Y-%m-%d")
else:
# Default: process single day (end = start + 1 day, exclusive)
end_date = start_date + timedelta(days=1)
start_str = start_date.strftime("%Y-%m-%d") start_str = start_date.strftime("%Y-%m-%d")
end_str = end_date.strftime("%Y-%m-%d") if end_date else None end_str = end_date.strftime("%Y-%m-%d")
# Generate date chunks if processing a range # Generate date chunks
date_chunks = [] date_chunks = []
if end_str: current = start_date
current = start_date while current < end_date:
while current < end_date: chunk_end = min(current + timedelta(days=args.chunk_days), end_date)
chunk_end = min(current + timedelta(days=args.chunk_days), end_date) date_chunks.append({
date_chunks.append({ 'start': current.strftime("%Y-%m-%d"),
'start': current.strftime("%Y-%m-%d"), 'end': chunk_end.strftime("%Y-%m-%d")
'end': chunk_end.strftime("%Y-%m-%d") })
}) current = chunk_end
current = chunk_end
else:
# Single day
date_chunks = [{'start': start_str, 'end': start_str}]
print("=" * 60) print("=" * 60)
print("ADS-B Processing Pipeline") print("ADS-B Processing Pipeline")
print("=" * 60) print("=" * 60)
if end_str: print(f"Date range: {start_str} to {end_str} (exclusive)")
print(f"Date range: {start_str} to {end_str}") print(f"Date chunks: {len(date_chunks)} ({args.chunk_days} days each)")
print(f"Date chunks: {len(date_chunks)} ({args.chunk_days} days each)")
else:
print(f"Date: {start_str}")
print(f"ICAO chunks: {args.chunks}") print(f"ICAO chunks: {args.chunks}")
print("=" * 60) print("=" * 60)
# Process each date chunk
# Process each date chunk # Process each date chunk
for idx, date_chunk in enumerate(date_chunks, 1): for idx, date_chunk in enumerate(date_chunks, 1):
chunk_start = date_chunk['start'] chunk_start = date_chunk['start']
@@ -117,12 +113,8 @@ def main():
print("Step 1: Download and Extract") print("Step 1: Download and Extract")
print("=" * 60) print("=" * 60)
if chunk_start == chunk_end: cmd = ["python", "-m", "src.adsb.download_and_list_icaos",
cmd = ["python", "-m", "src.adsb.download_and_list_icaos", "--start-date", chunk_start, "--end-date", chunk_end]
"--date", chunk_start]
else:
cmd = ["python", "-m", "src.adsb.download_and_list_icaos",
"--start-date", chunk_start, "--end-date", chunk_end]
run_cmd(cmd, "Download and extract") run_cmd(cmd, "Download and extract")
# Step 2: Process chunks # Step 2: Process chunks
@@ -132,17 +124,11 @@ def main():
for chunk_id in range(args.chunks): for chunk_id in range(args.chunks):
print(f"\n--- ICAO Chunk {chunk_id + 1}/{args.chunks} ---") print(f"\n--- ICAO Chunk {chunk_id + 1}/{args.chunks} ---")
if chunk_start == chunk_end: cmd = ["python", "-m", "src.adsb.process_icao_chunk",
cmd = ["python", "-m", "src.adsb.process_icao_chunk", "--chunk-id", str(chunk_id),
"--chunk-id", str(chunk_id), "--total-chunks", str(args.chunks),
"--total-chunks", str(args.chunks), "--start-date", chunk_start,
"--date", chunk_start] "--end-date", chunk_end]
else:
cmd = ["python", "-m", "src.adsb.process_icao_chunk",
"--chunk-id", str(chunk_id),
"--total-chunks", str(args.chunks),
"--start-date", chunk_start,
"--end-date", chunk_end]
run_cmd(cmd, f"Process ICAO chunk {chunk_id}") run_cmd(cmd, f"Process ICAO chunk {chunk_id}")
# Step 3: Combine all chunks to CSV # Step 3: Combine all chunks to CSV
@@ -152,12 +138,10 @@ def main():
chunks_dir = "./data/output/adsb_chunks" chunks_dir = "./data/output/adsb_chunks"
cmd = ["python", "-m", "src.adsb.combine_chunks_to_csv", cmd = ["python", "-m", "src.adsb.combine_chunks_to_csv",
"--chunks-dir", chunks_dir] "--chunks-dir", chunks_dir,
"--start-date", start_str,
if end_str: "--end-date", end_str,
cmd.extend(["--start-date", start_str, "--end-date", end_str]) "--stream"]
else:
cmd.extend(["--date", start_str])
if args.skip_base: if args.skip_base:
cmd.append("--skip-base") cmd.append("--skip-base")
@@ -170,10 +154,9 @@ def main():
# Show output # Show output
output_dir = "./data/openairframes" output_dir = "./data/openairframes"
if end_str: # Calculate actual end date for filename (end_date - 1 day since it's exclusive)
output_file = f"openairframes_adsb_{start_str}_{end_str}.csv.gz" actual_end = (end_date - timedelta(days=1)).strftime("%Y-%m-%d")
else: output_file = f"openairframes_adsb_{start_str}_{actual_end}.csv.gz"
output_file = f"openairframes_adsb_{start_str}_{start_str}.csv.gz"
output_path = os.path.join(output_dir, output_file) output_path = os.path.join(output_dir, output_file)
if os.path.exists(output_path): if os.path.exists(output_path):