test: enforce Wave A quality lock

This commit is contained in:
Joseph Magly
2026-08-14 19:47:46 -04:00
parent 57523ab483
commit 9746c21b63
18 changed files with 1712 additions and 44 deletions
+92 -19
View File
@@ -203,9 +203,11 @@ jobs:
scripts/check_mutation_score.py
scripts/check_quality_policy.py
scripts/check_conditional_policy.py
scripts/check_test_risk_map.py
scripts/conditional_gate_summary.py
scripts/run_conditional_gate.py
scripts/run_repeat_gate.py
scripts/write_test_evidence.py
scripts/check_supply_chain_policy.py
scripts/gemma4_12b_recursive_loop.py
@@ -217,9 +219,11 @@ jobs:
scripts/check_mutation_score.py
scripts/check_quality_policy.py
scripts/check_conditional_policy.py
scripts/check_test_risk_map.py
scripts/conditional_gate_summary.py
scripts/run_conditional_gate.py
scripts/run_repeat_gate.py
scripts/write_test_evidence.py
scripts/check_supply_chain_policy.py
scripts/gemma4_12b_recursive_loop.py || true
@@ -288,26 +292,57 @@ jobs:
--cov-report="xml:test-results/coverage-py${{ matrix.python-version }}.xml" \
--cov-report="json:test-results/coverage-py${{ matrix.python-version }}.json"
- name: Generate exact-base coverage for module regression comparison
if: matrix.python-version == '3.12'
env:
BASE_TEST_ENV: /tmp/obliteratus-base-test-env
BASE_WORKTREE: /tmp/obliteratus-base-worktree
COVERAGE_BASE: ${{ github.event.pull_request.base.sha || github.event.before }}
run: |
git rev-parse --verify "${COVERAGE_BASE}^{commit}"
git worktree add --detach "$BASE_WORKTREE" "$COVERAGE_BASE"
(
cd "$BASE_WORKTREE"
UV_PROJECT_ENVIRONMENT="$BASE_TEST_ENV" \
uv sync --locked --no-default-groups --extra dev --no-editable
COVERAGE_FILE="$RUNNER_TEMP/.coverage-base" \
"$BASE_TEST_ENV/bin/python" -m pytest \
-m "not slow and not gpu and not mps and not mlx and not network and not download and not remote and not operator_ui" \
--cov-branch \
--cov-fail-under=0 \
--cov-report="json:$GITHUB_WORKSPACE/test-results/base-coverage-py3.12.json"
) | tee test-results/base-tests-py3.12.log
- name: Enforce line and branch coverage floors
env:
COVERAGE_BASE: ${{ github.event.pull_request.base.sha || github.event.before }}
run: >-
"$TEST_ENV/bin/python" scripts/check_coverage_thresholds.py
"test-results/coverage-py${{ matrix.python-version }}.json"
--min-line 60
--min-branch 42
--min-file obliteratus/device.py=70
--min-file obliteratus/models/loader.py=70
--min-file obliteratus/architecture_profiles.py=70
--min-file obliteratus/cli.py=70
--min-file obliteratus/mlx_backend.py=70
--min-file obliteratus/evaluation/metrics.py=70
--min-file obliteratus/evaluation/advanced_metrics.py=70
--min-file obliteratus/reporting/report.py=70
--min-file obliteratus/community.py=70
--min-file obliteratus/telemetry.py=70
--min-changed 90
--base-ref "$COVERAGE_BASE"
run: |
module_args=()
if [ "${{ matrix.python-version }}" = "3.12" ]; then
module_args=(
--base-report test-results/base-coverage-py3.12.json
--touched-module-no-regression
--new-module-min-line 80
--new-module-min-branch 75
)
fi
"$TEST_ENV/bin/python" scripts/check_coverage_thresholds.py \
"test-results/coverage-py${{ matrix.python-version }}.json" \
--min-line 60 \
--min-branch 42 \
--min-file obliteratus/device.py=70 \
--min-file obliteratus/models/loader.py=70 \
--min-file obliteratus/architecture_profiles.py=70 \
--min-file obliteratus/cli.py=70 \
--min-file obliteratus/mlx_backend.py=70 \
--min-file obliteratus/evaluation/metrics.py=70 \
--min-file obliteratus/evaluation/advanced_metrics.py=70 \
--min-file obliteratus/reporting/report.py=70 \
--min-file obliteratus/community.py=70 \
--min-file obliteratus/telemetry.py=70 \
--min-changed 90 \
--base-ref "$COVERAGE_BASE" \
"${module_args[@]}"
- name: Enforce mature CPU-scope coverage and immutable quality policy
run: |
@@ -315,6 +350,25 @@ jobs:
--policy ci/test-quality-policy.json \
--coverage "test-results/coverage-py${{ matrix.python-version }}.json"
"$TEST_ENV/bin/python" scripts/check_conditional_policy.py
"$TEST_ENV/bin/python" scripts/check_test_risk_map.py
- name: Write normalized test trend evidence
if: always()
env:
COVERAGE_BASE: ${{ github.event.pull_request.base.sha || github.event.before }}
run: |
base_args=()
if [ -f test-results/base-coverage-py3.12.json ]; then
base_args=(--base-coverage test-results/base-coverage-py3.12.json)
fi
"$TEST_ENV/bin/python" scripts/write_test_evidence.py \
--coverage "test-results/coverage-py${{ matrix.python-version }}.json" \
--junit "test-results/junit-py${{ matrix.python-version }}.xml" \
--head-sha "$GITHUB_SHA" \
--base-sha "$COVERAGE_BASE" \
--python-version "${{ matrix.python-version }}" \
--output "test-results/test-trend-py${{ matrix.python-version }}.json" \
"${base_args[@]}"
- name: Upload test and coverage evidence
if: always()
@@ -323,7 +377,7 @@ jobs:
name: test-evidence-py${{ matrix.python-version }}
path: test-results/
if-no-files-found: error
retention-days: 14
retention-days: 90
quality-depth:
name: Quality depth
@@ -375,6 +429,25 @@ jobs:
"$QUALITY_ENV/bin/python" scripts/check_mutation_score.py \
quality-evidence/mutation-stats.json --minimum 70
- name: Write normalized quality trend evidence
if: always()
env:
COVERAGE_BASE: ${{ github.event.pull_request.base.sha || github.event.before }}
run: |
evidence_args=()
if [ -f quality-evidence/repeat-gate.json ]; then
evidence_args+=(--repeat quality-evidence/repeat-gate.json)
fi
if [ -f quality-evidence/mutation-stats.json ]; then
evidence_args+=(--mutation quality-evidence/mutation-stats.json)
fi
"$QUALITY_ENV/bin/python" scripts/write_test_evidence.py \
--head-sha "$GITHUB_SHA" \
--base-sha "$COVERAGE_BASE" \
--python-version "3.12" \
--output quality-evidence/quality-trend-py3.12.json \
"${evidence_args[@]}"
- name: Upload quality-depth evidence
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
@@ -382,7 +455,7 @@ jobs:
name: quality-depth-py3.12
path: quality-evidence/
if-no-files-found: error
retention-days: 14
retention-days: 90
supply-chain:
name: Supply chain