CalvinBackup/OpenAirframes
1
0
Fork 0
mirror of https://github.com/PlaneQuery/OpenAirframes.git synced 2026-07-30 21:37:25 +02:00
Code Issues Packages Projects Releases Wiki Activity

modify existing json schema instead of creating a new file every time

Browse Source
This commit is contained in:
ggman12
2026-02-12 15:40:01 -05:00
parent 4b756cdaef
commit 0b89138daf
4 changed files with 72 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/update-community-prs.yaml
+20 -36
View File
@@ -5,7 +5,7 @@ on:
branches: [main] branches: [main]
paths: paths:
- 'community/**' - 'community/**'
- 'schemas/community_submission.*.schema.json' - 'schemas/community_submission.v1.schema.json'
permissions: permissions:
contents: write contents: write
@@ -29,18 +29,9 @@ jobs:
- name: Install dependencies - name: Install dependencies
run: pip install jsonschema run: pip install jsonschema
- name: Get current schema version
id: schema
run: |
# Find the latest schema version on main
latest=$(ls schemas/community_submission.v*.schema.json 2>/dev/null | sed 's/.*\.v\([0-9]*\)\.schema\.json/\1/' | sort -n | tail -1)
echo "latest_version=${latest:-1}" >> "$GITHUB_OUTPUT"
echo "Latest schema version: ${latest:-1}"
- name: Find and update open community PRs - name: Find and update open community PRs
env: env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
LATEST_SCHEMA_VERSION: ${{ steps.schema.outputs.latest_version }}
run: | run: |
# Get list of open community PRs # Get list of open community PRs
prs=$(gh pr list --label community --state open --json number,headRefName --jq '.[] | "\(.number) \(.headRefName)"') prs=$(gh pr list --label community --state open --json number,headRefName --jq '.[] | "\(.number) \(.headRefName)"')
@@ -57,35 +48,28 @@ jobs:
git fetch origin "$branch_name" git fetch origin "$branch_name"
git checkout "$branch_name" git checkout "$branch_name"
# Check if this PR has a schema file that needs updating # Merge main into PR branch
pr_schema=$(ls schemas/community_submission.v*.schema.json 2>/dev/null | sed 's/.*\.v\([0-9]*\)\.schema\.json/\1/' | sort -n | tail -1) git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
if [ "$pr_schema" -le "$LATEST_SCHEMA_VERSION" ] 2>/dev/null; then if git merge origin/main -m "Merge main to update schema"; then
echo " PR schema version ($pr_schema) <= main version ($LATEST_SCHEMA_VERSION)" # Regenerate schema for this PR's submission (adds any new tags)
python -m src.contributions.regenerate_pr_schema || true
# Merge main into PR branch # If there are changes, commit and push
git config user.name "github-actions[bot]" if [ -n "$(git status --porcelain schemas/)" ]; then
git config user.email "github-actions[bot]@users.noreply.github.com" git add schemas/
git commit -m "Update schema with new tags"
if git merge origin/main -m "Merge main to update schema baseline"; then git push origin "$branch_name"
# Regenerate schema for this PR's submission echo " Updated PR #$pr_number with schema changes"
python -m src.contributions.regenerate_pr_schema || true
# If there are changes, commit and push
if [ -n "$(git status --porcelain schemas/)" ]; then
new_version=$((LATEST_SCHEMA_VERSION + 1))
git add schemas/
git commit -m "Update schema to v${new_version} (rebased on main)"
git push origin "$branch_name"
echo " Updated PR #$pr_number with new schema"
else
git push origin "$branch_name"
echo " Merged main into PR #$pr_number"
fi
else else
echo " Merge conflict in PR #$pr_number, adding comment" git push origin "$branch_name"
gh pr comment "$pr_number" --body $'⚠️ **Merge Conflict**\n\nAnother community submission was merged and this PR has conflicts.\n\nA maintainer may need to:\n1. Close this PR\n2. Remove the `approved` label from the original issue\n3. Re-add the `approved` label to regenerate the PR' echo " Merged main into PR #$pr_number"
git merge --abort fi
else
echo " Merge conflict in PR #$pr_number, adding comment"
gh pr comment "$pr_number" --body $'⚠️ **Merge Conflict**\n\nAnother community submission was merged and this PR has conflicts.\n\nA maintainer may need to:\n1. Close this PR\n2. Remove the `approved` label from the original issue\n3. Re-add the `approved` label to regenerate the PR'
git merge --abort
fi fi
fi fi
src/contributions/approve_submission.py
+10 -13
View File
@@ -21,13 +21,13 @@ import urllib.request
import urllib.error import urllib.error
from datetime import datetime, timezone from datetime import datetime, timezone
from .schema import extract_json_from_issue_body, extract_contributor_name_from_issue_body, parse_and_validate, get_latest_schema_version, load_schema from .schema import extract_json_from_issue_body, extract_contributor_name_from_issue_body, parse_and_validate, load_schema, SCHEMAS_DIR
from .contributor import ( from .contributor import (
generate_contributor_uuid, generate_contributor_uuid,
generate_submission_filename, generate_submission_filename,
compute_content_hash, compute_content_hash,
) )
from .update_schema import generate_new_schema, check_for_new_tags, get_existing_tag_definitions from .update_schema import generate_updated_schema, check_for_new_tags, get_existing_tag_definitions
from .read_community_data import build_tag_type_registry from .read_community_data import build_tag_type_registry
@@ -190,17 +190,15 @@ def process_submission(
commit_message = f"Add community submission from @{author_username} (closes #{issue_number})" commit_message = f"Add community submission from @{author_username} (closes #{issue_number})"
create_or_update_file(file_path, content_json, commit_message, branch_name) create_or_update_file(file_path, content_json, commit_message, branch_name)
# Update schema with any new tags (creates new version if needed) # Update schema with any new tags (modifies v1 in place)
schema_updated = False schema_updated = False
new_version = None
new_tags = [] new_tags = []
try: try:
# Build tag registry from new submissions # Build tag registry from new submissions
tag_registry = build_tag_type_registry(submissions) tag_registry = build_tag_type_registry(submissions)
# Get current schema and merge existing tags # Get current schema and merge existing tags
current_version = get_latest_schema_version() current_schema = load_schema()
current_schema = load_schema(current_version)
existing_tags = get_existing_tag_definitions(current_schema) existing_tags = get_existing_tag_definitions(current_schema)
# Merge existing tags into registry # Merge existing tags into registry
@@ -213,15 +211,14 @@ def process_submission(
new_tags = check_for_new_tags(tag_registry, current_schema) new_tags = check_for_new_tags(tag_registry, current_schema)
if new_tags: if new_tags:
# Generate new schema version # Generate updated schema
new_version = current_version + 1 updated_schema = generate_updated_schema(current_schema, tag_registry)
new_schema = generate_new_schema(current_schema, tag_registry, new_version) schema_json = json.dumps(updated_schema, indent=2) + "\n"
schema_json = json.dumps(new_schema, indent=2) + "\n"
create_or_update_file( create_or_update_file(
f"schemas/community_submission.v{new_version}.schema.json", "schemas/community_submission.v1.schema.json",
schema_json, schema_json,
f"Create schema v{new_version} with new tags: {', '.join(new_tags)}", f"Update schema with new tags: {', '.join(new_tags)}",
branch_name branch_name
) )
schema_updated = True schema_updated = True
@@ -231,7 +228,7 @@ def process_submission(
# Create PR # Create PR
schema_note = "" schema_note = ""
if schema_updated: if schema_updated:
schema_note = f"\n**Schema Updated:** Created v{new_version} with new tags: `{', '.join(new_tags)}`\n" schema_note = f"\n**Schema Updated:** Added new tags: `{', '.join(new_tags)}`\n"
pr_body = f"""## Community Submission pr_body = f"""## Community Submission
src/contributions/regenerate_pr_schema.py
+14 -16
View File
@@ -1,8 +1,8 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
""" """
Regenerate schema for a PR branch after main has been merged in. Regenerate schema for a PR branch after main has been merged in.
This script looks at the submission files in this branch and generates This script looks at the submission files in this branch and updates
an updated schema version if new tags were introduced. the schema if new tags were introduced.
Usage: python -m src.contributions.regenerate_pr_schema Usage: python -m src.contributions.regenerate_pr_schema
""" """
@@ -18,15 +18,14 @@ from src.contributions.read_community_data import read_all_submissions, build_ta
from src.contributions.update_schema import ( from src.contributions.update_schema import (
get_existing_tag_definitions, get_existing_tag_definitions,
check_for_new_tags, check_for_new_tags,
generate_new_schema, generate_updated_schema,
) )
from src.contributions.schema import get_latest_schema_version, load_schema, SCHEMAS_DIR from src.contributions.schema import load_schema, SCHEMAS_DIR
def main(): def main():
"""Main entry point.""" """Main entry point."""
# Get current schema version and load it # Load current schema
current_version = get_latest_schema_version()
current_schema = load_schema() current_schema = load_schema()
# Get existing tag definitions from schema # Get existing tag definitions from schema
@@ -46,20 +45,19 @@ def main():
new_tags = check_for_new_tags(tag_registry, current_schema) new_tags = check_for_new_tags(tag_registry, current_schema)
if new_tags: if new_tags:
# Generate new schema version
new_version = current_version + 1
print(f"Found new tags: {new_tags}") print(f"Found new tags: {new_tags}")
print(f"Generating schema v{new_version}") print("Updating schema...")
# Generate new schema with updated tag definitions # Generate updated schema
new_schema = generate_new_schema(current_schema, tag_registry, new_version) updated_schema = generate_updated_schema(current_schema, tag_registry)
# Write new schema version # Write updated schema (in place)
new_schema_path = SCHEMAS_DIR / f"community_submission.v{new_version}.schema.json" schema_path = SCHEMAS_DIR / "community_submission.v1.schema.json"
with open(new_schema_path, 'w') as f: with open(schema_path, 'w') as f:
json.dump(new_schema, f, indent=2) json.dump(updated_schema, f, indent=2)
f.write("\n")
print(f"Created {new_schema_path}") print(f"Updated {schema_path}")
else: else:
print("No new tags found, schema is up to date") print("No new tags found, schema is up to date")
src/contributions/update_schema.py
+28 -34
View File
@@ -40,23 +40,19 @@ def type_name_to_json_schema(type_name: str) -> dict:
return type_map.get(type_name, {"$ref": "#/$defs/tagValue"}) return type_map.get(type_name, {"$ref": "#/$defs/tagValue"})
def generate_new_schema(base_schema: dict, tag_registry: dict[str, str], new_version: int) -> dict: def generate_updated_schema(base_schema: dict, tag_registry: dict[str, str]) -> dict:
""" """
Generate a new schema version with explicit tag definitions. Generate an updated schema with explicit tag definitions.
Args: Args:
base_schema: The current schema to base the new one on base_schema: The current schema to update
tag_registry: Dict mapping tag name to type name tag_registry: Dict mapping tag name to type name
new_version: The new version number
Returns: Returns:
Complete new schema dict Updated schema dict
""" """
schema = json.loads(json.dumps(base_schema)) # Deep copy schema = json.loads(json.dumps(base_schema)) # Deep copy
# Update title with new version
schema["title"] = f"PlaneQuery Aircraft Community Submission (v{new_version})"
# Build tag properties with explicit types # Build tag properties with explicit types
tag_properties = {} tag_properties = {}
for tag_name, type_name in sorted(tag_registry.items()): for tag_name, type_name in sorted(tag_registry.items()):
@@ -89,57 +85,55 @@ def check_for_new_tags(tag_registry: dict[str, str], current_schema: dict) -> li
return [tag for tag in tag_registry if tag not in existing_tags] return [tag for tag in tag_registry if tag not in existing_tags]
def create_new_schema_version( def update_schema_file(
tag_registry: dict[str, str], tag_registry: dict[str, str],
check_only: bool = False check_only: bool = False
) -> tuple[int | None, list[str]]: ) -> tuple[bool, list[str]]:
""" """
Create a new schema version if there are new tags. Update the v1 schema file with new tag definitions.
Args: Args:
tag_registry: Dict mapping tag name to type name tag_registry: Dict mapping tag name to type name
check_only: If True, only check if update is needed without writing check_only: If True, only check if update is needed without writing
Returns: Returns:
Tuple of (new_version or None if no update, list_of_new_tags) Tuple of (was_updated, list_of_new_tags)
""" """
current_version = get_latest_schema_version() current_schema = load_schema()
current_schema = load_schema(current_version)
# Find new tags # Find new tags
new_tags = check_for_new_tags(tag_registry, current_schema) new_tags = check_for_new_tags(tag_registry, current_schema)
if not new_tags: if not new_tags:
return None, [] return False, []
if check_only: if check_only:
return current_version + 1, new_tags return True, new_tags
# Generate and write new schema # Generate and write updated schema (in place)
new_version = current_version + 1 updated_schema = generate_updated_schema(current_schema, tag_registry)
new_schema = generate_new_schema(current_schema, tag_registry, new_version) schema_path = get_schema_path()
new_schema_path = get_schema_path(new_version)
with open(new_schema_path, "w") as f: with open(schema_path, "w") as f:
json.dump(new_schema, f, indent=2) json.dump(updated_schema, f, indent=2)
f.write("\n") f.write("\n")
return new_version, new_tags return True, new_tags
def update_schema_from_submissions(check_only: bool = False) -> tuple[int | None, list[str]]: def update_schema_from_submissions(check_only: bool = False) -> tuple[bool, list[str]]:
""" """
Read all submissions and create a new schema version if needed. Read all submissions and update the schema if needed.
Args: Args:
check_only: If True, only check if update is needed without writing check_only: If True, only check if update is needed without writing
Returns: Returns:
Tuple of (new_version or None if no update, list_of_new_tags) Tuple of (was_updated, list_of_new_tags)
""" """
submissions = read_all_submissions() submissions = read_all_submissions()
tag_registry = build_tag_type_registry(submissions) tag_registry = build_tag_type_registry(submissions)
return create_new_schema_version(tag_registry, check_only) return update_schema_file(tag_registry, check_only)
def main(): def main():
@@ -148,21 +142,21 @@ def main():
args = parser.parse_args() args = parser.parse_args()
new_version, new_tags = update_schema_from_submissions(check_only=args.check) was_updated, new_tags = update_schema_from_submissions(check_only=args.check)
if args.check: if args.check:
if new_version: if was_updated:
print(f"Schema update needed -> v{new_version}. New tags: {', '.join(new_tags)}") print(f"Schema update needed. New tags: {', '.join(new_tags)}")
sys.exit(1) sys.exit(1)
else: else:
print(f"Schema is up to date (v{get_latest_schema_version()})") print("Schema is up to date")
sys.exit(0) sys.exit(0)
else: else:
if new_version: if was_updated:
print(f"Created {get_schema_path(new_version)}") print(f"Updated {get_schema_path()}")
print(f"Added tags: {', '.join(new_tags)}") print(f"Added tags: {', '.join(new_tags)}")
else: else:
print(f"No update needed (v{get_latest_schema_version()})") print("No update needed")
if __name__ == "__main__": if __name__ == "__main__":