FEATURE: Add contributions framework. Fix and improve daily adsb release using Github actions for map reduce.

This commit is contained in:
ggman12
2026-02-11 14:04:27 -05:00
parent 27da93801e
commit 722bcdf791
29 changed files with 2347 additions and 343 deletions
+2 -14
View File
@@ -79,8 +79,8 @@ class AdsbProcessingStack(Stack):
# --- MAP: worker task definition ---
map_task_def = ecs.FargateTaskDefinition(
self, "MapTaskDef",
cpu=16384, # 16 vCPU
memory_limit_mib=65536, # 64 GB — high memory for pandas operations on large datasets
cpu=4096, # 4 vCPU
memory_limit_mib=30720, # 30 GB
task_role=task_role,
runtime_platform=ecs.RuntimePlatform(
cpu_architecture=ecs.CpuArchitecture.ARM64,
@@ -149,18 +149,6 @@ class AdsbProcessingStack(Stack):
name="RUN_ID",
value=sfn.JsonPath.string_at("$.run_id"),
),
sfn_tasks.TaskEnvironmentVariable(
name="CLICKHOUSE_HOST",
value=sfn.JsonPath.string_at("$.clickhouse_host"),
),
sfn_tasks.TaskEnvironmentVariable(
name="CLICKHOUSE_USERNAME",
value=sfn.JsonPath.string_at("$.clickhouse_username"),
),
sfn_tasks.TaskEnvironmentVariable(
name="CLICKHOUSE_PASSWORD",
value=sfn.JsonPath.string_at("$.clickhouse_password"),
),
],
)
],