use date folder

This commit is contained in:
ggman12
2026-02-17 16:16:46 -05:00
parent 121dccf26c
commit ca5cb23a4d
+2 -2
View File
@@ -144,12 +144,12 @@ def main():
df_compressed = compress_parquet_part(args.part_id, args.date)
# Write parquet
df_compressed_output = OUTPUT_DIR / "compressed" / f"part_{args.part_id}_{args.date}.parquet"
df_compressed_output = OUTPUT_DIR / "compressed" / args.date/ f"part_{args.part_id}_{args.date}.parquet"
os.makedirs(df_compressed_output.parent, exist_ok=True)
df_compressed.write_parquet(df_compressed_output, compression='snappy')
# Write CSV
csv_output = OUTPUT_DIR / "compressed" / f"part_{args.part_id}_{args.date}.csv"
csv_output = OUTPUT_DIR / "compressed" / args.date / f"part_{args.part_id}_{args.date}.csv"
df_compressed.write_csv(csv_output)
print(df_compressed)