mirror of
https://github.com/PlaneQuery/OpenAirframes.git
synced 2026-07-18 07:57:18 +02:00
use date folder
This commit is contained in:
@@ -144,12 +144,12 @@ def main():
|
|||||||
df_compressed = compress_parquet_part(args.part_id, args.date)
|
df_compressed = compress_parquet_part(args.part_id, args.date)
|
||||||
|
|
||||||
# Write parquet
|
# Write parquet
|
||||||
df_compressed_output = OUTPUT_DIR / "compressed" / f"part_{args.part_id}_{args.date}.parquet"
|
df_compressed_output = OUTPUT_DIR / "compressed" / args.date/ f"part_{args.part_id}_{args.date}.parquet"
|
||||||
os.makedirs(df_compressed_output.parent, exist_ok=True)
|
os.makedirs(df_compressed_output.parent, exist_ok=True)
|
||||||
df_compressed.write_parquet(df_compressed_output, compression='snappy')
|
df_compressed.write_parquet(df_compressed_output, compression='snappy')
|
||||||
|
|
||||||
# Write CSV
|
# Write CSV
|
||||||
csv_output = OUTPUT_DIR / "compressed" / f"part_{args.part_id}_{args.date}.csv"
|
csv_output = OUTPUT_DIR / "compressed" / args.date / f"part_{args.part_id}_{args.date}.csv"
|
||||||
df_compressed.write_csv(csv_output)
|
df_compressed.write_csv(csv_output)
|
||||||
|
|
||||||
print(df_compressed)
|
print(df_compressed)
|
||||||
|
|||||||
Reference in New Issue
Block a user