diff --git a/src/adsb/download_and_list_icaos.py b/src/adsb/download_and_list_icaos.py index 6eda8a2..7132cc5 100644 --- a/src/adsb/download_and_list_icaos.py +++ b/src/adsb/download_and_list_icaos.py @@ -122,7 +122,8 @@ def process_single_day(target_day: datetime) -> tuple[str | None, list[str]]: from pathlib import Path import tarfile -def split_folders_into_gzip_archives(extract_dir: Path, tar_output_dir: Path, icaos: list[str], parts = 16) -> list[str]: +NUMBER_PARTS = 16 +def split_folders_into_gzip_archives(extract_dir: Path, tar_output_dir: Path, icaos: list[str], parts = NUMBER_PARTS) -> list[str]: traces_dir = extract_dir / "traces" buckets = sorted(traces_dir.iterdir()) tars = [] diff --git a/src/adsb/process_icao_chunk.py b/src/adsb/process_icao_chunk.py index 60bd225..19927d9 100644 --- a/src/adsb/process_icao_chunk.py +++ b/src/adsb/process_icao_chunk.py @@ -101,9 +101,6 @@ def process_chunk( total_rows += len(batch_rows) batch_rows = [] gc.collect() - - print(f"Part {part_id}: {total_rows} rows, {time.perf_counter() - start_time:.1f}s | {get_resource_usage()}") - gc.collect() if batch_rows: