Make global NUMBER_PARTS. remove print.

This commit is contained in:
ggman12
2026-02-17 16:42:59 -05:00
parent 9c54d9f1e4
commit 6306aade16
2 changed files with 2 additions and 4 deletions
+2 -1
View File
@@ -122,7 +122,8 @@ def process_single_day(target_day: datetime) -> tuple[str | None, list[str]]:
from pathlib import Path
import tarfile
def split_folders_into_gzip_archives(extract_dir: Path, tar_output_dir: Path, icaos: list[str], parts = 16) -> list[str]:
NUMBER_PARTS = 16
def split_folders_into_gzip_archives(extract_dir: Path, tar_output_dir: Path, icaos: list[str], parts = NUMBER_PARTS) -> list[str]:
traces_dir = extract_dir / "traces"
buckets = sorted(traces_dir.iterdir())
tars = []
-3
View File
@@ -101,9 +101,6 @@ def process_chunk(
total_rows += len(batch_rows)
batch_rows = []
gc.collect()
print(f"Part {part_id}: {total_rows} rows, {time.perf_counter() - start_time:.1f}s | {get_resource_usage()}")
gc.collect()
if batch_rows: