diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 000000000..fdb189c39 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,47 @@ +# Test Run Strategy + +Strategy for total 25 workflows: + +- Run 3 workflows per hour, spaced 20 minutes apart (e.g., at :00, :20, :40). +- This gives a 5-minute buffer between jobs, assuming each job takes up to 15 minutes. +- With 3 jobs per hour, all 25 jobs will run in about 8 hours and 20 minutes. +- Each workflow will run multiple times per day (every 8h20m). + +**Example cron schedule for 25 workflows:** + +``` +# Workflow 1: '0 0-23/8 * * *' +# Workflow 2: '20 0-23/8 * * *' +# Workflow 3: '40 0-23/8 * * *' +# Workflow 4: '0 1-23/8 * * *' +# Workflow 5: '20 1-23/8 * * *' + +# Workflow 6: '40 1-23/8 * * *' +# Workflow 7: '0 2-23/8 * * *' +# Workflow 8: '20 2-23/8 * * *' +# Workflow 9: '40 2-23/8 * * *' +# Workflow 10: '0 3-23/8 * * *' + +# Workflow 11: '20 3-23/8 * * *' +# Workflow 12: '40 3-23/8 * * *' +# Workflow 13: '0 4-23/8 * * *' +# Workflow 14: '20 4-23/8 * * *' +# Workflow 15: '40 4-23/8 * * *' + +# Workflow 16: '0 5-23/8 * * *' +# Workflow 17: '20 5-23/8 * * *' +# Workflow 18: '40 5-23/8 * * *' +# Workflow 19: '0 6-23/8 * * *' +# Workflow 20: '20 6-23/8 * * *' + +# Workflow 21: '40 6-23/8 * * *' +# Workflow 22: '0 7-23/8 * * *' +# Workflow 23: '20 7-23/8 * * *' +# Workflow 24: '40 7-23/8 * * *' +# Workflow 25: '0 8-23/8 * * *' +``` + +**How it works:** +- Each workflow runs every 8 hours and 20 minutes, starting at a different hour/minute offset. +- No more than 3 jobs run in any given hour. +- There’s a 20-minute gap between each job start. diff --git a/.github/workflows/test_00_benign_prompts_no_mitigation.01-20.yml b/.github/workflows/test_00_benign_prompts_no_mitigation.01-20.yml index 18e90fc28..d503ed376 100644 --- a/.github/workflows/test_00_benign_prompts_no_mitigation.01-20.yml +++ b/.github/workflows/test_00_benign_prompts_no_mitigation.01-20.yml @@ -2,7 +2,7 @@ name: '#0 (1-20)' on: workflow_dispatch: schedule: - - cron: '0 0 * * *' # Starts at 00:00 UTC + - cron: '0 0-23/8 * * *' jobs: test-prompts-1-20: diff --git a/.github/workflows/test_00_benign_prompts_no_mitigation.21-40.yml b/.github/workflows/test_00_benign_prompts_no_mitigation.21-40.yml index 5cae87876..b9bedc9a9 100644 --- a/.github/workflows/test_00_benign_prompts_no_mitigation.21-40.yml +++ b/.github/workflows/test_00_benign_prompts_no_mitigation.21-40.yml @@ -2,7 +2,7 @@ name: '#0 (21-40)' on: workflow_dispatch: schedule: - - cron: '57 0 * * *' # Starts at 00:57 UTC + - cron: '20 0-23/8 * * *' jobs: test-prompts-21-40: diff --git a/.github/workflows/test_00_benign_prompts_no_mitigation.41-60.yml b/.github/workflows/test_00_benign_prompts_no_mitigation.41-60.yml index a2b4d19c5..c4354db2a 100644 --- a/.github/workflows/test_00_benign_prompts_no_mitigation.41-60.yml +++ b/.github/workflows/test_00_benign_prompts_no_mitigation.41-60.yml @@ -2,7 +2,7 @@ name: '#0 (41-60)' on: workflow_dispatch: schedule: - - cron: '54 1 * * *' # Starts at 01:54 UTC + - cron: '40 0-23/8 * * *' jobs: test-prompts-41-60: diff --git a/.github/workflows/test_00_benign_prompts_no_mitigation.61-80.yml b/.github/workflows/test_00_benign_prompts_no_mitigation.61-80.yml index 335521de0..cfbbbaa02 100644 --- a/.github/workflows/test_00_benign_prompts_no_mitigation.61-80.yml +++ b/.github/workflows/test_00_benign_prompts_no_mitigation.61-80.yml @@ -2,7 +2,7 @@ name: '#0 (61-80)' on: workflow_dispatch: schedule: - - cron: '51 2 * * *' # Starts at 02:51 UTC + - cron: '0 1-23/8 * * *' jobs: test-prompts-61-80: diff --git a/.github/workflows/test_00_benign_prompts_no_mitigation.81-100.yml b/.github/workflows/test_00_benign_prompts_no_mitigation.81-100.yml index cdcc75ecf..4a8edd1c0 100644 --- a/.github/workflows/test_00_benign_prompts_no_mitigation.81-100.yml +++ b/.github/workflows/test_00_benign_prompts_no_mitigation.81-100.yml @@ -2,7 +2,7 @@ name: '#0 (81-100)' on: workflow_dispatch: schedule: - - cron: '48 3 * * *' # Starts at 03:48 UTC + - cron: '20 1-23/8 * * *' jobs: test-prompts-81-100: diff --git a/.github/workflows/test_01_malicious_prompts_no_mitigation.01-20.yml b/.github/workflows/test_01_malicious_prompts_no_mitigation.01-20.yml index 994e58bbb..fa1b5d1bc 100644 --- a/.github/workflows/test_01_malicious_prompts_no_mitigation.01-20.yml +++ b/.github/workflows/test_01_malicious_prompts_no_mitigation.01-20.yml @@ -2,7 +2,7 @@ name: '#1 (1-20)' on: workflow_dispatch: schedule: - - cron: '45 4 * * *' # Starts at 04:45 UTC + - cron: '40 1-23/8 * * *' jobs: test-prompts-1-20: diff --git a/.github/workflows/test_01_malicious_prompts_no_mitigation.21-40.yml b/.github/workflows/test_01_malicious_prompts_no_mitigation.21-40.yml index 484fe8045..8e24c228d 100644 --- a/.github/workflows/test_01_malicious_prompts_no_mitigation.21-40.yml +++ b/.github/workflows/test_01_malicious_prompts_no_mitigation.21-40.yml @@ -2,7 +2,7 @@ name: '#1 (21-40)' on: workflow_dispatch: schedule: - - cron: '42 5 * * *' # Starts at 05:42 UTC + - cron: '0 2-23/8 * * *' jobs: test-prompts-21-40: diff --git a/.github/workflows/test_01_malicious_prompts_no_mitigation.41-60.yml b/.github/workflows/test_01_malicious_prompts_no_mitigation.41-60.yml index fbae81467..83f446bd6 100644 --- a/.github/workflows/test_01_malicious_prompts_no_mitigation.41-60.yml +++ b/.github/workflows/test_01_malicious_prompts_no_mitigation.41-60.yml @@ -2,7 +2,7 @@ name: '#1 (41-60)' on: workflow_dispatch: schedule: - - cron: '39 6 * * *' # Starts at 06:39 UTC + - cron: '20 2-23/8 * * *' jobs: test-prompts-41-60: diff --git a/.github/workflows/test_01_malicious_prompts_no_mitigation.61-80.yml b/.github/workflows/test_01_malicious_prompts_no_mitigation.61-80.yml index e3f9db2b7..886d60aef 100644 --- a/.github/workflows/test_01_malicious_prompts_no_mitigation.61-80.yml +++ b/.github/workflows/test_01_malicious_prompts_no_mitigation.61-80.yml @@ -2,7 +2,7 @@ name: '#1 (61-80)' on: workflow_dispatch: schedule: - - cron: '36 7 * * *' # Starts at 07:36 UTC + - cron: '40 2-23/8 * * *' jobs: test-prompts-61-80: diff --git a/.github/workflows/test_01_malicious_prompts_no_mitigation.81-100.yml b/.github/workflows/test_01_malicious_prompts_no_mitigation.81-100.yml index 7112057ab..ccd2c512b 100644 --- a/.github/workflows/test_01_malicious_prompts_no_mitigation.81-100.yml +++ b/.github/workflows/test_01_malicious_prompts_no_mitigation.81-100.yml @@ -2,7 +2,7 @@ name: '#1 (81-100)' on: workflow_dispatch: schedule: - - cron: '33 8 * * *' # Starts at 08:33 UTC + - cron: '0 3-23/8 * * *' jobs: test-prompts-81-100: diff --git a/.github/workflows/test_02_malicious_prompts_cot.01-20.yml b/.github/workflows/test_02_malicious_prompts_cot.01-20.yml index 8351a3b25..181654222 100644 --- a/.github/workflows/test_02_malicious_prompts_cot.01-20.yml +++ b/.github/workflows/test_02_malicious_prompts_cot.01-20.yml @@ -2,7 +2,7 @@ name: '#2 (1-20)' on: workflow_dispatch: schedule: - - cron: '30 9 * * *' # Starts at 09:30 UTC + - cron: '20 3-23/8 * * *' jobs: test-prompts-1-20: diff --git a/.github/workflows/test_02_malicious_prompts_cot.21-40.yml b/.github/workflows/test_02_malicious_prompts_cot.21-40.yml index 5d226c7dd..fe1ad1c71 100644 --- a/.github/workflows/test_02_malicious_prompts_cot.21-40.yml +++ b/.github/workflows/test_02_malicious_prompts_cot.21-40.yml @@ -2,7 +2,7 @@ name: '#2 (21-40)' on: workflow_dispatch: schedule: - - cron: '27 10 * * *' # Starts at 10:27 UTC + - cron: '40 3-23/8 * * *' jobs: test-prompts-21-40: diff --git a/.github/workflows/test_02_malicious_prompts_cot.41-60.yml b/.github/workflows/test_02_malicious_prompts_cot.41-60.yml index a1ca62310..f52c99ffc 100644 --- a/.github/workflows/test_02_malicious_prompts_cot.41-60.yml +++ b/.github/workflows/test_02_malicious_prompts_cot.41-60.yml @@ -2,7 +2,7 @@ name: '#2 (41-60)' on: workflow_dispatch: schedule: - - cron: '24 11 * * *' # Starts at 11:24 UTC + - cron: '0 4-23/8 * * *' jobs: test-prompts-41-60: diff --git a/.github/workflows/test_02_malicious_prompts_cot.61-80.yml b/.github/workflows/test_02_malicious_prompts_cot.61-80.yml index c42058135..c348f8582 100644 --- a/.github/workflows/test_02_malicious_prompts_cot.61-80.yml +++ b/.github/workflows/test_02_malicious_prompts_cot.61-80.yml @@ -2,7 +2,7 @@ name: '#2 (61-80)' on: workflow_dispatch: schedule: - - cron: '21 12 * * *' # Starts at 12:21 UTC + - cron: '20 4-23/8 * * *' jobs: test-prompts-61-80: diff --git a/.github/workflows/test_02_malicious_prompts_cot.81-100.yml b/.github/workflows/test_02_malicious_prompts_cot.81-100.yml index d1f787fe8..d46a3dd01 100644 --- a/.github/workflows/test_02_malicious_prompts_cot.81-100.yml +++ b/.github/workflows/test_02_malicious_prompts_cot.81-100.yml @@ -2,7 +2,7 @@ name: '#2 (81-100)' on: workflow_dispatch: schedule: - - cron: '18 13 * * *' # Starts at 13:18 UTC + - cron: '40 4-23/8 * * *' jobs: test-prompts-81-100: diff --git a/.github/workflows/test_03_malicious_prompts_rag.01-20.yml b/.github/workflows/test_03_malicious_prompts_rag.01-20.yml index b0f026e13..2f3b68507 100644 --- a/.github/workflows/test_03_malicious_prompts_rag.01-20.yml +++ b/.github/workflows/test_03_malicious_prompts_rag.01-20.yml @@ -2,7 +2,7 @@ name: '#3 (1-20)' on: workflow_dispatch: schedule: - - cron: '15 14 * * *' # Starts at 14:15 UTC + - cron: '0 5-23/8 * * *' jobs: test-prompts-1-20: diff --git a/.github/workflows/test_03_malicious_prompts_rag.21-40.yml b/.github/workflows/test_03_malicious_prompts_rag.21-40.yml index d873f1325..aaf119ce4 100644 --- a/.github/workflows/test_03_malicious_prompts_rag.21-40.yml +++ b/.github/workflows/test_03_malicious_prompts_rag.21-40.yml @@ -2,7 +2,7 @@ name: '#3 (21-40)' on: workflow_dispatch: schedule: - - cron: '12 15 * * *' # Starts at 15:12 UTC + - cron: '20 5-23/8 * * *' jobs: test-prompts-21-40: diff --git a/.github/workflows/test_03_malicious_prompts_rag.41-60.yml b/.github/workflows/test_03_malicious_prompts_rag.41-60.yml index 6ede0be6f..d414caeb5 100644 --- a/.github/workflows/test_03_malicious_prompts_rag.41-60.yml +++ b/.github/workflows/test_03_malicious_prompts_rag.41-60.yml @@ -2,7 +2,7 @@ name: '#3 (41-60)' on: workflow_dispatch: schedule: - - cron: '9 16 * * *' # Starts at 16:09 UTC + - cron: '40 5-23/8 * * *' jobs: test-prompts-41-60: diff --git a/.github/workflows/test_03_malicious_prompts_rag.61-80.yml b/.github/workflows/test_03_malicious_prompts_rag.61-80.yml index 66785ab50..6a04760cd 100644 --- a/.github/workflows/test_03_malicious_prompts_rag.61-80.yml +++ b/.github/workflows/test_03_malicious_prompts_rag.61-80.yml @@ -2,7 +2,7 @@ name: '#3 (61-80)' on: workflow_dispatch: schedule: - - cron: '6 17 * * *' # Starts at 17:06 UTC + - cron: '0 6-23/8 * * *' jobs: test-prompts-61-80: diff --git a/.github/workflows/test_03_malicious_prompts_rag.81-100.yml b/.github/workflows/test_03_malicious_prompts_rag.81-100.yml index 6b48afe8a..6aac5f45b 100644 --- a/.github/workflows/test_03_malicious_prompts_rag.81-100.yml +++ b/.github/workflows/test_03_malicious_prompts_rag.81-100.yml @@ -2,7 +2,7 @@ name: '#3 (81-100)' on: workflow_dispatch: schedule: - - cron: '3 18 * * *' # Starts at 18:03 UTC + - cron: '20 6-23/8 * * *' jobs: test-prompts-81-100: diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot.01-20.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot.01-20.yml index b9ea4715a..d6ffcac9b 100644 --- a/.github/workflows/test_04_malicious_prompts_rag_and_cot.01-20.yml +++ b/.github/workflows/test_04_malicious_prompts_rag_and_cot.01-20.yml @@ -2,7 +2,7 @@ name: '#4 (1-20)' on: workflow_dispatch: schedule: - - cron: '0 19 * * *' # Starts at 19:00 UTC + - cron: '40 6-23/8 * * *' jobs: test-prompts-1-20: diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot.21-40.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot.21-40.yml index 9361abc35..c5fc0cfd7 100644 --- a/.github/workflows/test_04_malicious_prompts_rag_and_cot.21-40.yml +++ b/.github/workflows/test_04_malicious_prompts_rag_and_cot.21-40.yml @@ -2,7 +2,7 @@ name: '#4 (21-40)' on: workflow_dispatch: schedule: - - cron: '57 19 * * *' # Starts at 19:57 UTC + - cron: '0 7-23/8 * * *' jobs: test-prompts-21-40: diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot.41-60.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot.41-60.yml index 6821b79e9..f98de5436 100644 --- a/.github/workflows/test_04_malicious_prompts_rag_and_cot.41-60.yml +++ b/.github/workflows/test_04_malicious_prompts_rag_and_cot.41-60.yml @@ -2,7 +2,7 @@ name: '#4 (41-60)' on: workflow_dispatch: schedule: - - cron: '54 20 * * *' # Starts at 20:54 UTC + - cron: '20 7-23/8 * * *' jobs: test-prompts-41-60: diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot.61-80.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot.61-80.yml index bf0210d6c..ff7600721 100644 --- a/.github/workflows/test_04_malicious_prompts_rag_and_cot.61-80.yml +++ b/.github/workflows/test_04_malicious_prompts_rag_and_cot.61-80.yml @@ -2,7 +2,7 @@ name: '#4 (61-80)' on: workflow_dispatch: schedule: - - cron: '51 21 * * *' # Starts at 21:51 UTC + - cron: '40 7-23/8 * * *' jobs: test-prompts-61-80: diff --git a/.github/workflows/test_04_malicious_prompts_rag_and_cot.81-100.yml b/.github/workflows/test_04_malicious_prompts_rag_and_cot.81-100.yml index 7d1b8ea86..b91963acb 100644 --- a/.github/workflows/test_04_malicious_prompts_rag_and_cot.81-100.yml +++ b/.github/workflows/test_04_malicious_prompts_rag_and_cot.81-100.yml @@ -2,7 +2,7 @@ name: '#4 (81-100)' on: workflow_dispatch: schedule: - - cron: '48 22 * * *' # Starts at 22:48 UTC + - cron: '0 8-23/8 * * *' jobs: test-prompts-81-100: diff --git a/.github/workflows/tests.abstract_base.yml b/.github/workflows/tests.abstract_base.yml index 14b7ff9d0..e19ddc02e 100644 --- a/.github/workflows/tests.abstract_base.yml +++ b/.github/workflows/tests.abstract_base.yml @@ -26,7 +26,7 @@ on: jobs: test: runs-on: ubuntu-latest - timeout-minutes: 55 # Set max runtime for each job + timeout-minutes: 16 # Set max runtime for each job continue-on-error: true strategy: # Always 10 batches per workflow