"""Shared model of the Ghostgram string tables, used by generate.py and check.py. Source layout: Ghostgram/Strings//.json, each a flat {"key": "text"} object. A key is ".." or "."; the first segment is the file name. Plural forms are separate keys "#" (CLDR categories) and always take {count}. Substitutions are named: "Deleted {date}". "{{" and "}}" are literal braces. """ import json import os import re import unicodedata LANGUAGES = ["en", "ru", "uk", "ar"] BASE_LANGUAGE = "en" # Missing keys in these languages are errors; in the others a warning until the translation wave. COMPLETE_LANGUAGES = ["en", "ru"] PLURAL_CATEGORIES = { "en": ["one", "other"], "ru": ["one", "few", "many", "other"], "uk": ["one", "few", "many", "other"], "ar": ["zero", "one", "two", "few", "many", "other"], } ALL_PLURAL_CATEGORIES = ["zero", "one", "two", "few", "many", "other"] SEGMENT_RE = re.compile(r"^[a-z][A-Za-z0-9]*$") ARGUMENT_RE = re.compile(r"^[a-z][A-Za-z0-9]*$") TOKEN_RE = re.compile(r"\{\{|\}\}|\{([^{}]*)\}|[{}]") CYRILLIC_RE = re.compile(r"[Ѐ-ӿԀ-ԯ]") # Leaf names that are Swift keywords are emitted with backticks; these can not be used at all. FORBIDDEN_LEAVES = {"self", "Self", "init", "deinit", "subscript", "Type", "Protocol", "_"} # Nested enum names that would shadow what the generated code refers to. FORBIDDEN_ENUMS = {"Swift", "GGLocalization", "GGStrings", "Type", "Self", "Protocol", "Any", "AnyObject"} SWIFT_KEYWORDS = { "associatedtype", "class", "deinit", "enum", "extension", "fileprivate", "func", "import", "init", "inout", "internal", "let", "open", "operator", "private", "precedencegroup", "protocol", "public", "rethrows", "static", "struct", "subscript", "typealias", "var", "break", "case", "catch", "continue", "default", "defer", "do", "else", "fallthrough", "for", "guard", "if", "in", "repeat", "return", "throw", "switch", "where", "while", "as", "is", "nil", "super", "self", "true", "false", "try", "throws", "any", "some", "await", "async", } class Diagnostics: def __init__(self): self.errors = [] self.warnings = [] def error(self, message): self.errors.append(message) def warning(self, message): self.warnings.append(message) def default_root(): return os.path.dirname(os.path.abspath(__file__)) def default_generated_dir(root): return os.path.normpath(os.path.join(root, "..", "GGCore", "Sources", "Strings", "Generated")) def split_key(key): """Returns (base, category) where category is None for a plain key.""" if "#" in key: base, category = key.split("#", 1) return base, category return key, None def placeholders(text, diagnostics=None, where=""): """Named placeholders in order of first appearance.""" result = [] for match in TOKEN_RE.finditer(text): token = match.group(0) if token in ("{{", "}}"): continue if token in ("{", "}"): if diagnostics is not None: diagnostics.error("%s: unbalanced brace in %r (use {{ and }} for literal braces)" % (where, text)) continue name = match.group(1) if not ARGUMENT_RE.match(name) or name in SWIFT_KEYWORDS: if diagnostics is not None: diagnostics.error("%s: invalid placeholder {%s}" % (where, name)) continue if name not in result: result.append(name) return result def load_tables(root, diagnostics): """Reads every /.json. Returns ({lang: {key: text}}, {lang: {key: path}}).""" tables = {} origins = {} for language in LANGUAGES: tables[language] = {} origins[language] = {} directory = os.path.join(root, language) if not os.path.isdir(directory): continue for name in sorted(os.listdir(directory)): if not name.endswith(".json"): continue path = os.path.join(directory, name) namespace = name[:-len(".json")] relative = os.path.relpath(path, root) if not SEGMENT_RE.match(namespace): diagnostics.error("%s: file name must be a camelCase namespace" % relative) continue try: with open(path, "r", encoding="utf-8") as handle: content = json.load(handle) except (OSError, ValueError) as error: diagnostics.error("%s: can not parse JSON: %s" % (relative, error)) continue if not isinstance(content, dict): diagnostics.error("%s: top level must be an object" % relative) continue for key, value in content.items(): if not isinstance(value, str): diagnostics.error("%s: %s: value must be a string" % (relative, key)) continue base, category = split_key(key) segments = base.split(".") if segments[0] != namespace: diagnostics.error("%s: key %s must start with \"%s.\"" % (relative, key, namespace)) continue if len(segments) < 2: diagnostics.error("%s: key %s needs at least two segments" % (relative, key)) continue bad = [segment for segment in segments if not SEGMENT_RE.match(segment)] if bad: diagnostics.error("%s: key %s: segments must be camelCase identifiers" % (relative, key)) continue if segments[-1] in FORBIDDEN_LEAVES: diagnostics.error("%s: key %s: last segment can not be %r" % (relative, key, segments[-1])) continue enums = [pascal(segment) for segment in segments[:-1]] if any(name in FORBIDDEN_ENUMS for name in enums): diagnostics.error("%s: key %s: a segment collides with a reserved type name" % (relative, key)) continue if category is not None and category not in ALL_PLURAL_CATEGORIES: diagnostics.error("%s: key %s: unknown plural category %r" % (relative, key, category)) continue if key in tables[language]: diagnostics.error("%s: duplicate key %s (also in %s)" % (relative, key, origins[language][key])) continue tables[language][key] = value origins[language][key] = relative return tables, origins def pascal(segment): return segment[:1].upper() + segment[1:] class Entry: """One accessor: a plain string or a plural group.""" def __init__(self, base, kind, arguments, doc): self.base = base self.kind = kind self.arguments = arguments self.doc = doc @property def segments(self): return self.base.split(".") @property def accessor_path(self): segments = self.segments return ".".join([pascal(segment) for segment in segments[:-1]] + [segments[-1]]) def build_entries(tables, diagnostics): """Accessors are generated from the base language.""" base_table = tables.get(BASE_LANGUAGE, {}) groups = {} for key in base_table: base, category = split_key(key) group = groups.setdefault(base, {"plain": None, "plural": {}}) if category is None: group["plain"] = key else: group["plural"][category] = key entries = [] for base in sorted(groups): group = groups[base] if group["plain"] is not None and group["plural"]: diagnostics.error("%s: key is both a plain string and a plural" % base) continue if group["plain"] is not None: text = base_table[group["plain"]] arguments = placeholders(text, diagnostics, "%s/%s" % (BASE_LANGUAGE, base)) entries.append(Entry(base, "plain", arguments, text)) else: arguments = [] for category in ALL_PLURAL_CATEGORIES: key = group["plural"].get(category) if key is None: continue for name in placeholders(base_table[key], diagnostics, "%s/%s" % (BASE_LANGUAGE, key)): if name not in arguments: arguments.append(name) if "count" in arguments: arguments.remove("count") arguments.insert(0, "count") doc = base_table.get(group["plural"].get("other") or sorted(group["plural"].values())[0]) entries.append(Entry(base, "plural", arguments, doc)) # A leaf and a nested enum may share a spelling only when their case differs, which the # camelCase/PascalCase split already guarantees; two accessors with one path can not exist. return entries def swift_identifier(name): if name in SWIFT_KEYWORDS: return "`%s`" % name return name def swift_doc(text): single = text.replace("\r", " ").replace("\n", "⏎") if len(single) > 160: single = single[:157] + "..." return single def render_strings_swift(entries): tree = {"children": {}, "leaves": []} for entry in entries: node = tree for segment in entry.segments[:-1]: node = node["children"].setdefault(pascal(segment), {"children": {}, "leaves": []}) node["leaves"].append(entry) lines = [ "// Generated by Ghostgram/Strings/generate.py from Ghostgram/Strings//*.json. Do not edit.", "// Every accessor is computed: a translated string is never stored, so a language switch applies at once.", "", "public enum GGStrings {", ] def emit(node, indent): pad = " " * indent first = True for entry in sorted(node["leaves"], key=lambda item: item.segments[-1]): if not first: lines.append("") first = False name = swift_identifier(entry.segments[-1]) lines.append("%s/// %s" % (pad, swift_doc(entry.doc))) if entry.kind == "plain" and not entry.arguments: lines.append("%spublic static var %s: Swift.String {" % (pad, name)) lines.append("%s return GGLocalization.shared.string(\"%s\")" % (pad, entry.base)) lines.append("%s}" % pad) continue parameters = [] pairs = [] for argument in entry.arguments: if argument == "count": parameters.append("count: Swift.Int") if entry.kind == "plain": pairs.append("(\"count\", Swift.String(count))") else: parameters.append("%s: Swift.String" % argument) pairs.append("(\"%s\", %s)" % (argument, argument)) signature = "%spublic static func %s(%s) -> Swift.String {" % (pad, name, ", ".join(parameters)) lines.append(signature) if entry.kind == "plain": lines.append("%s return GGLocalization.shared.format(\"%s\", [%s])" % (pad, entry.base, ", ".join(pairs))) else: lines.append("%s return GGLocalization.shared.plural(\"%s\", count: count, [%s])" % (pad, entry.base, ", ".join(pairs))) lines.append("%s}" % pad) for name in sorted(node["children"]): if not first: lines.append("") first = False lines.append("%spublic enum %s {" % (pad, name)) emit(node["children"][name], indent + 1) lines.append("%s}" % pad) emit(tree, 1) lines.append("}") lines.append("") return "\n".join(lines) def escape_for_swift_source(text): """JSON text with every invisible or line-breaking character written as a \\u escape.""" output = [] for character in text: category = unicodedata.category(character) if category in ("Cc", "Cf", "Zl", "Zp", "Co", "Cs"): code = ord(character) if code > 0xFFFF: code -= 0x10000 output.append("\\u%04x\\u%04x" % (0xD800 + (code >> 10), 0xDC00 + (code & 0x3FF))) else: output.append("\\u%04x" % code) else: output.append(character) return "".join(output) def raw_swift_literal(text): hashes = 1 while ('"' + "#" * hashes) in text or ("\\" + "#" * hashes) in text: hashes += 1 return "#" * hashes + '"' + text + '"' + "#" * hashes def namespace_chunks(table): """{namespace: compact JSON} for one language, one chunk per namespace.""" namespaces = {} for key, value in table.items(): namespaces.setdefault(key.split(".", 1)[0], {})[key] = value chunks = [] for namespace in sorted(namespaces): text = json.dumps(namespaces[namespace], ensure_ascii=False, sort_keys=True, separators=(",", ":")) chunks.append((namespace, escape_for_swift_source(text))) return chunks def render_data_swift(tables): lines = [ "// Generated by Ghostgram/Strings/generate.py from Ghostgram/Strings//*.json. Do not edit.", "// String tables as raw JSON literals, one per namespace, parsed lazily on first use. There are no", "// bundle resources, so the app and every extension see the same strings.", "", "enum GGStringsData {", " static let languages: [String] = [%s]" % ", ".join("\"%s\"" % language for language in LANGUAGES), "", " static func chunks(for language: String) -> [String]? {", " switch language {", ] for language in LANGUAGES: lines.append(" case \"%s\":" % language) lines.append(" return GGStringsData.%s" % language) lines.append(" default:") lines.append(" return nil") lines.append(" }") lines.append(" }") for language in LANGUAGES: lines.append("") chunks = namespace_chunks(tables.get(language, {})) if not chunks: lines.append(" private static let %s: [String] = []" % language) continue lines.append(" private static let %s: [String] = [" % language) for namespace, text in chunks: lines.append(" // %s" % namespace) lines.append(" %s," % raw_swift_literal(text)) lines.append(" ]") lines.append("}") lines.append("") return "\n".join(lines) def render_all(root, diagnostics): """Returns {file name: content} of the generated Swift sources.""" tables, _ = load_tables(root, diagnostics) entries = build_entries(tables, diagnostics) return { "GGStrings.swift": render_strings_swift(entries), "GGStringsData.swift": render_data_swift(tables), }