mirror of
https://github.com/PlaneQuery/OpenAirframes.git
synced 2026-04-23 11:36:35 +02:00
Make global NUMBER_PARTS. remove print.
This commit is contained in:
@@ -122,7 +122,8 @@ def process_single_day(target_day: datetime) -> tuple[str | None, list[str]]:
|
||||
|
||||
from pathlib import Path
|
||||
import tarfile
|
||||
def split_folders_into_gzip_archives(extract_dir: Path, tar_output_dir: Path, icaos: list[str], parts = 16) -> list[str]:
|
||||
NUMBER_PARTS = 16
|
||||
def split_folders_into_gzip_archives(extract_dir: Path, tar_output_dir: Path, icaos: list[str], parts = NUMBER_PARTS) -> list[str]:
|
||||
traces_dir = extract_dir / "traces"
|
||||
buckets = sorted(traces_dir.iterdir())
|
||||
tars = []
|
||||
|
||||
@@ -101,9 +101,6 @@ def process_chunk(
|
||||
total_rows += len(batch_rows)
|
||||
batch_rows = []
|
||||
gc.collect()
|
||||
|
||||
print(f"Part {part_id}: {total_rows} rows, {time.perf_counter() - start_time:.1f}s | {get_resource_usage()}")
|
||||
|
||||
gc.collect()
|
||||
|
||||
if batch_rows:
|
||||
|
||||
Reference in New Issue
Block a user