Merge commit '7621e2f8dec938cf48181c8b10afc9b01f444e68' into beta

third-party/libprisma/Sources/Highlight.cpp

@@ -0,0 +1,42 @@
+#include "Highlight.h"
+
+#include "LanguageTree.h"
+
+GrammarPtr::GrammarPtr(std::shared_ptr<LanguageTree> tree, size_t path)
+    : m_tree(tree)
+    , m_path(path)
+{
+
+}
+
+const Grammar* GrammarPtr::operator->() const
+{
+    return get();
+}
+
+const Grammar* GrammarPtr::get() const
+{
+    return m_tree->resolveGrammar(m_path);
+}
+
+PatternPtr::PatternPtr(std::shared_ptr<LanguageTree> tree, size_t path)
+    : m_tree(tree)
+    , m_path(path)
+{
+
+}
+
+const Pattern* PatternPtr::get() const
+{
+    return m_tree->resolvePattern(m_path);
+}
+
+const Grammar* Pattern::inside() const
+{
+    if (m_inside)
+    {
+        return m_inside->get();
+    }
+
+    return nullptr;
+}

third-party/libprisma/Sources/Highlight.h

@@ -0,0 +1,149 @@
+#pragma once
+#include <string>
+#include <vector>
+#include <boost/regex.hpp>
+#include <map>
+#include <functional>
+
+class GrammarPtr;
+class GrammarToken;
+class Pattern;
+class TokenList;
+
+class LanguageTree;
+
+struct Grammar {
+    std::vector<GrammarToken> tokens;
+};
+
+class GrammarPtr
+{
+public:
+    GrammarPtr(std::shared_ptr<LanguageTree> tree, size_t path);
+
+    const Grammar* operator->() const;
+    const Grammar* get() const;
+
+private:
+    std::shared_ptr<LanguageTree> m_tree;
+    size_t m_path;
+};
+
+class Pattern
+{
+public:
+    Pattern(std::string_view pattern, boost::regex_constants::syntax_option_type flags, bool lookbehind, bool greedy, std::string alias, GrammarPtr inside)
+        : Pattern(pattern, flags, lookbehind, greedy, alias, std::make_shared<GrammarPtr>(inside))
+    {
+
+    }
+
+    Pattern(std::string_view pattern, boost::regex_constants::syntax_option_type flags, bool lookbehind = false, bool greedy = false, std::string alias = "", std::shared_ptr<GrammarPtr> inside = nullptr)
+        : m_regex(boost::regex(std::string{pattern}, flags | boost::regex_constants::optimize))
+        , m_lookbehind(lookbehind)
+        , m_greedy(greedy)
+        , m_alias(alias)
+        , m_inside(inside)
+    {
+    }
+
+    std::string_view match(bool& success, size_t& pos, std::string_view text) const
+    {
+        try {
+            boost::cmatch m;
+            
+            auto flags = boost::regex_constants::match_not_dot_newline;
+            auto match = boost::regex_search(text.data() + pos, text.data() + text.size(), m, m_regex, flags);
+            if (match)
+            {
+                success = true;
+                pos += m.position();
+                
+                if (m_lookbehind && m[1].matched)
+                {
+                    // change the match to remove the text matched by the Prism lookbehind group
+                    auto lookbehindLength = m[1].length();
+                    pos += lookbehindLength;
+                    
+                    return text.substr(pos, m[0].length() - lookbehindLength);
+                }
+                
+                return text.substr(pos, m[0].length());
+            }
+        } catch(...) {
+        }
+
+        return {};
+    }
+
+    bool lookbehind() const
+    {
+        return m_lookbehind;
+    }
+
+    bool greedy() const
+    {
+        return m_greedy;
+    }
+
+    std::string alias() const
+    {
+        return m_alias;
+    }
+
+    const Grammar* inside() const;
+
+private:
+    boost::regex m_regex;
+    bool m_lookbehind;
+    bool m_greedy;
+    std::string m_alias;
+    std::shared_ptr<GrammarPtr> m_inside;
+};
+
+class PatternPtr
+{
+public:
+    PatternPtr(std::shared_ptr<LanguageTree> tree, size_t path);
+
+    const Pattern* operator->() const
+    {
+        return get();
+    }
+
+    const Pattern* get() const;
+
+private:
+    std::shared_ptr<LanguageTree> m_tree;
+    size_t m_path;
+};
+
+class GrammarToken
+{
+public:
+    GrammarToken(const std::string name, std::vector<PatternPtr> patterns)
+        : m_name(name)
+        , m_patterns(std::move(patterns))
+    {
+
+    }
+
+    const std::string &name() const
+    {
+        return m_name;
+    }
+
+    std::vector<PatternPtr>::const_iterator cbegin() const noexcept
+    {
+        return m_patterns.cbegin();
+    }
+
+    std::vector<PatternPtr>::const_iterator cend() const noexcept
+    {
+        return m_patterns.cend();
+    }
+
+private:
+    std::string m_name;
+    const std::vector<PatternPtr> m_patterns;
+};

third-party/libprisma/Sources/LanguageTree.cpp

@@ -0,0 +1,184 @@
+#include "LanguageTree.h"
+
+#include "TokenList.h"
+#include <boost/regex.hpp>
+
+void LanguageTree::load(const std::string& content)
+{
+    Buffer buffer{ content };
+    parsePatterns(buffer);
+    parseGrammars(buffer);
+    parseLanguages(buffer);
+}
+
+inline uint8_t freadUint8(Buffer &buffer)
+{
+    uint8_t value = 0;
+    if (buffer.offset + sizeof(uint8_t) <= buffer.content.size()) {
+        memcpy(&value, &buffer.content[buffer.offset], sizeof(uint8_t));
+        buffer.offset += sizeof(uint8_t);
+    }
+    return value;
+}
+
+inline uint16_t freadUint16(Buffer &buffer)
+{
+    uint16_t value = 0;
+    if (buffer.offset + sizeof(uint16_t) <= buffer.content.size()) {
+		memcpy(&value, &buffer.content[buffer.offset], sizeof(uint16_t));
+        buffer.offset += sizeof(uint16_t);
+	}
+    return value;
+}
+
+inline std::string freadString(Buffer &buffer)
+{
+    size_t length = freadUint8(buffer);
+    if (length >= 254)
+    {
+        size_t a = freadUint8(buffer);
+        size_t b = freadUint8(buffer);
+        size_t c = freadUint8(buffer);
+        length = a | (b << 8) | (c << 16);
+    }
+
+    std::string str(length, '\0');
+    if (buffer.offset + length <= buffer.content.size())
+	{
+		memcpy(&str[0], &buffer.content[buffer.offset], length);
+        buffer.offset += length;
+	}
+    return str;
+}
+
+void LanguageTree::parseLanguages(Buffer &buffer)
+{
+    uint16_t count = freadUint16(buffer);
+
+    for (int i = 0; i < count; ++i)
+    {
+        std::string name = freadString(buffer);
+        std::string displayName = freadString(buffer);
+        size_t index = freadUint16(buffer);
+        m_languages.emplace(name, std::pair<std::string, size_t>(displayName, index));
+    }
+}
+
+void LanguageTree::parseGrammars(Buffer &buffer)
+{
+    uint16_t count = freadUint16(buffer);
+
+    for (int i = 0; i < count; ++i)
+    {
+        auto grammar = std::make_shared<Grammar>();
+        const auto keys = freadUint8(buffer);
+
+        for (int j = 0; j < keys; ++j)
+        {
+            std::vector<PatternPtr> indices;
+
+            const auto key = freadString(buffer);
+            uint8_t ids = freadUint8(buffer);
+
+            for (int k = 0; k < ids; ++k)
+            {
+                indices.push_back(PatternPtr(shared_from_this(), freadUint16(buffer)));
+            }
+
+            grammar->tokens.push_back(GrammarToken(key, indices));
+        }
+
+        m_grammars.push_back(grammar);
+    }
+}
+
+void LanguageTree::parsePatterns(Buffer &buffer)
+{
+    uint16_t count = freadUint16(buffer);
+
+    for (int i = 0; i < count; ++i)
+    {
+        std::string item = freadString(buffer);
+        std::string_view value(item);
+        std::string alias;
+
+        size_t beg = value.find_first_of('/');
+        size_t end = value.find_last_of('/');
+
+        if (beg != std::string::npos && end != std::string::npos)
+        {
+            std::string_view pattern = value.substr(beg + 1, end - (beg + 1));
+            std::string_view options = value.substr(end + 1);
+
+            size_t aliasBeg = options.find_first_of(',');
+            size_t aliasEnd = options.find_last_of(',');
+
+            std::string alias{ options.substr(aliasBeg + 1, aliasEnd - (aliasBeg + 1)) };
+            size_t inside = 0;
+
+            if (aliasEnd + 1 < options.size())
+            {
+                for (int i = aliasEnd + 1; i < options.size(); i++)
+                {
+                    char c = options[i];
+                    if (c >= '0' && c <= '9')
+                    {
+                        inside = inside * 10 + (c - '0');
+                    }
+                    else
+                    {
+                        assert(false);
+                    }
+                }
+            }
+            else
+            {
+                inside = std::string::npos;
+            }
+
+            bool lookbehind = false;
+            bool greedy = false;
+
+            boost::regex_constants::syntax_option_type flags
+                = boost::regex_constants::ECMAScript | boost::regex_constants::no_mod_m;
+
+            for (char c : options.substr(0, aliasBeg))
+            {
+                switch (c)
+                {
+                case 'l':
+                    lookbehind = true;
+                    break;
+                case 'y':
+                    greedy = true;
+                    break;
+                case 'i':
+                    flags |= boost::regex_constants::icase;
+                    break;
+                case 'm':
+                    flags &= ~boost::regex_constants::no_mod_m;
+                    break;
+                }
+            }
+
+            if (inside != std::string::npos)
+            {
+                m_patterns.push_back(std::make_shared<Pattern>(pattern, flags, lookbehind, greedy, std::string{ alias }, std::make_shared<GrammarPtr>(shared_from_this(), inside)));
+            }
+            else
+            {
+                m_patterns.push_back(std::make_shared<Pattern>(pattern, flags, lookbehind, greedy, std::string{ alias }));
+            }
+        }
+    }
+}
+
+const Pattern* LanguageTree::resolvePattern(size_t path)
+{
+    return m_patterns[path].get();
+}
+
+const Grammar* LanguageTree::resolveGrammar(size_t path)
+{
+    return m_grammars[path].get();
+}

third-party/libprisma/Sources/LanguageTree.h

@@ -0,0 +1,60 @@
+#pragma once
+
+#include <sstream>
+#include <fstream>
+#include <optional>
+
+#include "Highlight.h"
+
+struct Buffer {
+    const std::string &content;
+    int64_t offset = 0;
+};
+
+class LanguageTree : public std::enable_shared_from_this<LanguageTree>
+{
+public:
+    LanguageTree() = default;
+
+    void load(const std::string& content);
+
+    const Pattern* resolvePattern(size_t path);
+    const Grammar* resolveGrammar(size_t path);
+
+    std::map<std::string, std::string> keys() const
+    {
+        std::map<std::string, std::string> keys;
+
+        for (const auto& kv : m_languages)
+        {
+            if (kv.second.first.empty())
+            {
+                continue;
+            }
+
+            keys.emplace(kv.first, kv.second.first);
+        }
+
+        return keys;
+    }
+
+    const Grammar* find(const std::string& key) const
+    {
+        const auto& value = m_languages.find(key);
+        if (value != m_languages.end())
+        {
+            return m_grammars[value->second.second].get();
+        }
+
+        return nullptr;
+    }
+
+private:
+    void parseLanguages(Buffer &buffer);
+    void parseGrammars(Buffer &buffer);
+    void parsePatterns(Buffer &buffer);
+
+    std::map<std::string, std::pair<std::string, size_t>> m_languages;
+    std::vector<std::shared_ptr<Grammar>> m_grammars;
+    std::vector<std::shared_ptr<Pattern>> m_patterns;
+};

third-party/libprisma/Sources/SyntaxHighlighter.cpp

@@ -0,0 +1,199 @@
+#include "SyntaxHighlighter.h"
+#include "LanguageTree.h"
+#include "TokenList.h"
+
+SyntaxHighlighter::SyntaxHighlighter(const std::string& path)
+{
+    m_tree = std::make_shared<LanguageTree>();
+    m_tree->load(path);
+}
+
+TokenList SyntaxHighlighter::tokenize(const std::string& text, const std::string& language)
+{
+    const Grammar* grammar = m_tree->find(language);
+    if (grammar)
+    {
+        return tokenize(text, grammar);
+    }
+
+    return TokenList(text);
+}
+
+std::map<std::string, std::string> SyntaxHighlighter::languages() const
+{
+    return m_tree->keys();
+}
+
+TokenList SyntaxHighlighter::tokenize(std::string_view text, const Grammar* grammar)
+{
+    TokenList tokenList(text);
+    matchGrammar(text, tokenList, grammar, tokenList.head, 0, nullptr);
+
+    return tokenList;
+}
+
+void SyntaxHighlighter::matchGrammar(std::string_view text, TokenList& tokenList, const Grammar* grammar, TokenListPtr startNode, size_t startPos, RematchOptions* rematch)
+{
+    for (const auto& token : grammar->tokens)
+    {
+        int x = 0;
+        for (auto j = token.cbegin(); j != token.cend(); ++j)
+        {
+            if (rematch && rematch->j == x && rematch->token == token.name())
+            {
+                return;
+            }
+
+            const auto& pattern = *j;
+            const auto& inside = pattern->inside();
+            const bool greedy = pattern->greedy();
+
+            size_t pos = startPos;
+
+            // iterate the token list and keep track of the current token/string position
+            for (TokenListPtr currentNode = startNode->next;
+                currentNode != tokenList.head;
+                pos += currentNode->length(), currentNode = currentNode->next)
+            {
+                if (rematch && pos >= rematch->reach)
+                {
+                    break;
+                }
+
+                if (tokenList.length > text.length())
+                {
+                    // Something went terribly wrong, ABORT, ABORT!
+                    return;
+                }
+
+                if (currentNode->isSyntax())
+                {
+                    continue;
+                }
+
+                const auto& currentText = dynamic_cast<Text&>(*currentNode);
+                std::string_view str = currentText.value();
+
+                auto removeCount = 1; // this is the to parameter of removeBetween
+                std::string_view match;
+                bool matchSuccess = false;
+                size_t matchIndex = pos;
+
+                if (greedy)
+                {
+                    match = pattern->match(matchSuccess, matchIndex, text);
+                    if (!matchSuccess || matchIndex >= text.length())
+                    {
+                        break;
+                    }
+
+                    auto from = matchIndex;
+                    auto to = matchIndex + match.length();
+                    auto p = pos;
+
+                    // find the node that contains the match
+                    p += currentNode->length();
+                    while (from >= p)
+                    {
+                        currentNode = currentNode->next;
+                        p += currentNode->length();
+                    }
+                    // adjust pos (and p)
+                    p -= currentNode->length();
+                    pos = p;
+
+                    // the current node is a Token, then the match starts inside another Token, which is invalid
+                    if (currentNode->isSyntax())
+                    {
+                        continue;
+                    }
+
+                    // find the last node which is affected by this match
+                    for (TokenListPtr k = currentNode;
+                        k != tokenList.head && (p < to || !k->isSyntax());
+                        k = k->next)
+                    {
+                        removeCount++;
+                        p += k->length();
+                    }
+                    removeCount--;
+
+                    // replace with the new match
+                    str = text.substr(pos, p - pos);
+                    matchIndex -= pos;
+                }
+                else
+                {
+                    matchIndex = 0;
+                    match = pattern->match(matchSuccess, matchIndex, str);
+                    if (!matchSuccess)
+                    {
+                        continue;
+                    }
+                }
+
+                auto from = matchIndex;
+                auto before = str.substr(0, from);
+                auto after = str.substr(from + match.length());
+
+                auto reach = pos + str.length();
+                if (rematch && reach > rematch->reach)
+                {
+                    rematch->reach = reach;
+                }
+
+                TokenListPtr removeFrom = currentNode->prev;
+
+                if (before.size())
+                {
+                    removeFrom = tokenList.addAfter(removeFrom, before);
+                    pos += before.length();
+                }
+
+                tokenList.removeRange(removeFrom, removeCount);
+
+                TokenList tokenEntries = [&]() {
+                    if (inside)
+                    {
+                        return tokenize(match, inside);
+                    }
+                    else
+                    {
+                        return TokenList(match);
+                    }
+                }();
+
+                currentNode = tokenList.addAfter(removeFrom, token.name(),
+                    std::move(tokenEntries),
+                    pattern->alias(),
+                    match.size());
+
+                if (after.size())
+                {
+                    tokenList.addAfter(currentNode, after);
+                }
+
+                if (removeCount > 1)
+                {
+                    // at least one Token object was removed, so we have to do some rematching
+                    // this can only happen if the current pattern is greedy
+                    RematchOptions nestedRematch = {
+                        .token = token.name(),
+                        .reach = reach,
+                        .j = x
+                    };
+
+                    matchGrammar(text, tokenList, grammar, currentNode->prev, pos, &nestedRematch);
+
+                    // the reach might have been extended because of the rematching
+                    if (rematch && nestedRematch.reach > rematch->reach)
+                    {
+                        rematch->reach = nestedRematch.reach;
+                    }
+                }
+            }
+
+            ++x;
+        }
+    }
+}

third-party/libprisma/Sources/SyntaxHighlighter.h

@@ -0,0 +1,33 @@
+#pragma once
+#include <sstream>
+
+#include "TokenList.h"
+#include <vector>
+#include <map>
+#include <optional>
+
+class LanguageTree;
+struct Grammar;
+
+struct RematchOptions
+{
+    std::string token;
+    size_t reach;
+    int j;
+};
+
+class SyntaxHighlighter
+{
+public:
+    SyntaxHighlighter(const std::string& languages);
+
+    TokenList tokenize(const std::string& text, const std::string& language);
+
+    std::map<std::string, std::string> languages() const;
+
+private:
+    TokenList tokenize(std::string_view text, const Grammar* grammar);
+    void matchGrammar(std::string_view text, TokenList& tokenList, const Grammar* grammar, TokenListPtr startNode, size_t startPos, RematchOptions* rematch);
+
+    std::shared_ptr<LanguageTree> m_tree;
+};

third-party/libprisma/Sources/Syntaxer.mm

@@ -0,0 +1,245 @@
+//
+//  SyntaxHighligher.m
+//  CodeSyntax
+//
+//  Created by Mike Renoir on 26.10.2023.
+//
+
+#import <libprisma/Syntaxer.h>
+#import "SyntaxHighlighter.h"
+#import "TokenList.h"
+
+UIColor * color(UInt32 rgb, UInt32 alpha) {
+    return [UIColor colorWithRed:((rgb >> 16) & 0xff) / 255.0 green: ((rgb >> 8) & 0xff) / 255.0 blue: (rgb & 0xff) / 255.0 alpha: alpha];
+}
+
+
+
+NSDictionary<NSString *, UIColor *> *light = @{
+    @"comment": color(0x708090, 1.0), 
+    @"block-comment": color(0x708090, 1.0),
+    @"prolog": color(0x708090, 1.0),
+    @"doctype": color(0x708090, 1.0),
+    @"cdata": color(0x708090, 1.0),
+    @"punctuation": color(0x999999, 1.0),
+    @"property": color(0x990055, 1.0),
+    @"tag": color(0x990055, 1.0),
+    @"boolean": color(0x990055, 1.0),
+    @"number": color(0x990055, 1.0),
+    @"constant": color(0x990055, 1.0),
+    @"symbol": color(0x990055, 1.0),
+    @"deleted": color(0x990055, 1.0),
+    @"selector": color(0x669900, 1.0),
+    @"attr-name": color(0x669900, 1.0),
+    @"string": color(0x669900, 1.0),
+    @"char": color(0x669900, 1.0),
+    @"builtin": color(0x669900, 1.0),
+    @"inserted": color(0x669900, 1.0),
+    @"operator": color(0x9AAE6C, 1.0),
+    @"entity": color(0x9AAE6C, 1.0),
+    @"url": color(0x9AAE6C, 1.0),
+    @"atrule": color(0x0077AA, 1.0),
+    @"attr-value": color(0x0077AA, 1.0),
+    @"keyword": color(0x0077AA, 1.0),
+    @"function-definition": color(0x0077AA, 1.0),
+    @"class-name": color(0xDD4A68, 1.0),
+};
+
+NSDictionary<NSString *, UIColor *> *dark = @{
+    @"comment": color(0x999999, 1.0),
+    @"block-comment": color(0x999999, 1.0),
+    @"prolog": color(0x999999, 1.0),
+    @"doctype": color(0x999999, 1.0),
+    @"cdata": color(0x999999, 1.0),
+    @"punctuation": color(0xcccccc, 1.0),
+    @"property": color(0xf8c555, 1.0),
+    @"tag": color(0xe2777a, 1.0),
+    @"boolean": color(0xf08d49, 1.0),
+    @"number": color(0xf08d49, 1.0),
+    @"constant": color(0xf8c555, 1.0),
+    @"symbol": color(0xf8c555, 1.0),
+    @"deleted": color(0xe2777a, 1.0),
+    @"selector": color(0xcc99cd, 1.0),
+    @"attr-name": color(0xe2777a, 1.0),
+    @"string": color(0x7ec699, 1.0),
+    @"char": color(0x7ec699, 1.0),
+    @"builtin": color(0xcc99cd, 1.0),
+    @"inserted": color(0x669900, 1.0),
+    @"operator": color(0x67cdcc, 1.0),
+    @"entity": color(0x67cdcc, 1.0),
+    @"url": color(0x67cdcc, 1.0),
+    @"atrule": color(0xcc99cd, 1.0),
+    @"attr-value": color(0x7ec699, 1.0),
+    @"keyword": color(0xcc99cd, 1.0),
+    @"function-definition": color(0xf08d49, 1.0),
+    @"class-name": color(0xf8c555, 1.0),
+};
+
+
+
+std::string dataToString(NSData *nsData) {
+    const void *dataBytes = [nsData bytes];
+    NSUInteger dataLength = [nsData length];
+    
+    return std::string(static_cast<const char*>(dataBytes), dataLength);
+}
+
+NSString* stringViewToNSString(std::string_view sv) {
+    std::string cppString(sv.data(), sv.length());
+    return [NSString stringWithUTF8String:cppString.c_str()];
+}
+
+NSString* stringToNSString(const std::string& cppString) {
+    return [NSString stringWithUTF8String:cppString.c_str()];
+}
+
+@implementation SyntaxterTheme
+
+-(id)initWithDark:(BOOL)dark textColor:(UIColor *)textColor textFont:(UIFont *)textFont italicFont:(UIFont *)italicFont mediumFont:(UIFont *)mediumFont {
+    if (self = [super init]) {
+        _dark = dark;
+        _textColor = textColor;
+        _textFont = textFont;
+        _italicFont = italicFont;
+        _mediumFont = mediumFont;
+    }
+    return self;
+}
+
+@end
+
+
+@interface Brush : NSObject
+@property (nonatomic, strong) UIFont *font;
+@property (nonatomic, strong) UIColor *color;
+@end
+
+@implementation Brush
+
+-(id)initWith:(UIFont *)font color:(UIColor *)color {
+    if (self = [super init]) {
+        _font = font;
+        _color = color;
+    }
+    return self;
+}
+
+@end
+
+
+void applyString(NSString * string, NSMutableAttributedString * attributed, Brush *brush) {
+    if (string != nil) {
+        NSMutableAttributedString *substring = [[NSMutableAttributedString alloc] initWithString: string];
+        NSRange range = NSMakeRange(0, string.length);
+        [substring addAttribute:NSForegroundColorAttributeName value: brush.color range:range];
+        [substring addAttribute:NSFontAttributeName value:brush.font range:range];
+        [attributed appendAttributedString:substring];
+    }
+}
+
+Brush *makeBrush(std::string alias, std::string type, SyntaxterTheme * theme, Brush *previous) {
+    NSString * aliasKey = stringToNSString(alias);
+    NSString * typeKey = stringToNSString(type);
+    
+    
+    
+    NSDictionary<NSString *, UIColor *> *colors;
+    if (theme.dark) {
+        colors = dark;
+    } else {
+        colors = light;
+    }
+    
+    UIColor *color = colors[aliasKey];
+    UIFont *font = theme.textFont;
+    
+    if (color == nil) {
+        color = colors[typeKey];
+    }
+    if (color == nil) {
+        color = previous.color;
+    }
+    if (color == nil) {
+        color = theme.textColor;
+    }
+    if ([typeKey isEqualToString:@"bold"]) {
+        font = theme.mediumFont;
+    }
+    if ([typeKey isEqualToString:@"italic"]) {
+        font = theme.italicFont;
+    }
+    
+    if (previous != nil) {
+        font = previous.font;
+    }
+    
+    return [[Brush alloc] initWith:font color:color];
+}
+
+@interface Syntaxer ()
+@property (atomic) std::shared_ptr<SyntaxHighlighter> m_highlighter;
+@end
+
+@implementation Syntaxer
+-(id)init {
+    if (self = [super init]) {
+        NSString *path = [[[[NSBundle bundleForClass:[self class]] bundlePath] stringByAppendingPathComponent:@"LibprismaBundle.bundle"] stringByAppendingPathComponent:@"grammars.dat"];
+        NSData *grammar = [NSData dataWithContentsOfFile:path];
+        if (grammar == nil) {
+            return nil;
+        }
+        
+        std::string text = dataToString(grammar);
+        _m_highlighter = std::make_shared<SyntaxHighlighter>(text);
+    }
+    return self;
+}
+
+-(NSAttributedString *)syntax:(NSString *)code language: (NSString *)language theme: (SyntaxterTheme *) theme {
+    NSMutableAttributedString *string = [[NSMutableAttributedString alloc] init];
+    std::string c_code = code.UTF8String;
+    std::string c_language = language.UTF8String;
+
+    TokenList tokens = _m_highlighter->tokenize(c_code, c_language);
+    for (auto it = tokens.begin(); it != tokens.end(); ++it)
+    {
+        auto& node = *it;
+        Brush *brush;
+        if (node.isSyntax()) {
+            const auto& child = dynamic_cast<const Syntax&>(node);
+            brush = makeBrush(child.alias(), child.type(), theme, nil);
+        } else {
+            brush = makeBrush("", "", theme, nil);
+        }
+        [self paint:node string:string brush:brush theme: theme];
+    }
+    return string;
+}
+
+
+- (void) paint:(const TokenListNode &)node string: (NSMutableAttributedString *) string brush:(Brush *)upperBrash theme: (SyntaxterTheme *) theme {
+    if (node.isSyntax())
+    {
+        const auto& child = dynamic_cast<const Syntax&>(node);
+        
+        for (auto j = child.begin(); j != child.end(); ++j)
+        {
+            auto& innerNode = *j;
+            if (innerNode.isSyntax())
+            {
+                const auto& innerChild = dynamic_cast<const Syntax&>(innerNode);
+                Brush *brush = makeBrush(innerChild.alias(), innerChild.type(), theme, upperBrash);
+                [self paint:innerNode string:string brush:brush theme: theme];
+            } else {
+                const auto& innerChild = dynamic_cast<const Text&>(innerNode);
+                applyString(stringViewToNSString(innerChild.value()), string, upperBrash);
+            }
+        }
+    } else {
+        const auto& child = dynamic_cast<const Text&>(node);
+        applyString(stringViewToNSString(child.value()), string, upperBrash);
+    }
+}
+
+
+@end

third-party/libprisma/Sources/TokenList.cpp

@@ -0,0 +1,71 @@
+#include "TokenList.h"
+
+TokenList::TokenList(std::string_view value)
+    : head(new TokenListNode())
+    , length(1)
+{
+    const TokenListPtr newNode = new Text(head, head, value);
+    head->next = newNode;
+}
+
+TokenList::~TokenList()
+{
+    TokenListPtr next = head->next;
+    while (head != next)
+    {
+        TokenListPtr current = next;
+        next = next->next;
+        delete current;
+    }
+
+    delete head;
+}
+
+TokenListPtr TokenList::addAfter(TokenListPtr node, const std::string& type, TokenList&& children, const std::string& alias, size_t textLength)
+{
+    const TokenListPtr next = node->next;
+    const TokenListPtr newNode = new Syntax(node, next, type, std::move(children), alias, textLength);
+
+    node->next = newNode;
+    next->prev = newNode;
+    length++;
+
+    return newNode;
+}
+
+TokenListPtr TokenList::addAfter(TokenListPtr node, std::string_view value)
+{
+    const TokenListPtr next = node->next;
+    const TokenListPtr newNode = new Text(node, next, value);
+
+    node->next = newNode;
+    next->prev = newNode;
+    length++;
+
+    return newNode;
+}
+
+void TokenList::removeRange(TokenListPtr node, size_t count)
+{
+    TokenListPtr item = node->next;
+
+    for (size_t i = 0; i < count && item != nullptr; i++)
+    {
+        node->next = item->next;
+        node->next->prev = node;
+        delete item;
+
+        item = node->next;
+        length--;
+    }
+}
+
+Syntax::Syntax(TokenListPtr prev, TokenListPtr next, const std::string& type, TokenList&& children, const std::string& alias, size_t length)
+    : TokenListNode(prev, next)
+    , m_type(type)
+    , m_children(std::move(children))
+    , m_alias(alias)
+    , m_length(length)
+{
+
+}

third-party/libprisma/Sources/TokenList.h

@@ -0,0 +1,170 @@
+#pragma once
+
+#include <memory>
+#include <string>
+
+struct TokenListNode
+{
+    TokenListNode(TokenListNode* prev, TokenListNode* next)
+        : prev(prev)
+        , next(next)
+    {
+    }
+
+    TokenListNode()
+        : prev(nullptr)
+        , next(nullptr)
+    {
+    }
+
+    virtual ~TokenListNode() = default;
+
+    TokenListNode(const TokenListNode&) = delete;
+    TokenListNode& operator=(const TokenListNode&) = delete;
+
+    virtual size_t length() const
+    {
+        return 0;
+    }
+
+    virtual bool isSyntax() const
+    {
+        return true;
+    }
+
+    TokenListNode* prev;
+    TokenListNode* next;
+};
+
+typedef TokenListNode* TokenListPtr;
+
+class TokenList
+{
+public:
+    TokenList(std::string_view text);
+    ~TokenList();
+
+    TokenList(const TokenList&) = delete;
+    TokenList& operator=(const TokenList&) = delete;
+    TokenList(TokenList&& old) noexcept
+    {
+        head = old.head;
+        length = old.length;
+
+        old.head = new TokenListNode();
+        old.head->next = old.head;
+        old.length = 0;
+    }
+
+    struct ConstIterator
+    {
+        using iterator_category = std::forward_iterator_tag;
+        using difference_type = std::ptrdiff_t;
+        using value_type = const TokenListNode;
+        using pointer = const TokenListNode*;
+        using reference = const TokenListNode&;
+
+        ConstIterator(pointer ptr) : m_ptr(ptr) {}
+
+        reference operator*() const { return *m_ptr; }
+        pointer operator->() { return m_ptr; }
+        ConstIterator& operator++() { m_ptr = m_ptr->next; return *this; }
+        friend bool operator== (const ConstIterator& a, const ConstIterator& b) { return a.m_ptr == b.m_ptr; };
+        friend bool operator!= (const ConstIterator& a, const ConstIterator& b) { return a.m_ptr != b.m_ptr; };
+
+    private:
+        pointer m_ptr;
+    };
+
+    ConstIterator begin() const { return ConstIterator(head->next); }
+    ConstIterator end() const { return ConstIterator(head); }
+
+    TokenListPtr addAfter(TokenListPtr node, const std::string& type, TokenList&& children, const std::string& alias, size_t length);
+    TokenListPtr addAfter(TokenListPtr node, std::string_view value);
+    void removeRange(TokenListPtr node, size_t count);
+
+    TokenListPtr head;
+    size_t length;
+};
+
+class Text : public TokenListNode
+{
+public:
+    Text(TokenListPtr prev, TokenListPtr next, std::string_view value)
+        : TokenListNode(prev, next)
+        , m_value(value)
+    {
+
+    }
+
+    Text(const Text&) = delete;
+    Text& operator=(const Text&) = delete;
+
+    const std::string_view& value() const
+    {
+        return m_value;
+    }
+
+    size_t length() const
+    {
+        return m_value.size();
+    }
+
+    bool isSyntax() const
+    {
+        return false;
+    }
+
+private:
+    const std::string_view m_value;
+};
+
+class Syntax : public TokenListNode
+{
+public:
+    Syntax(TokenListPtr prev, TokenListPtr next, const std::string& type, TokenList&& children, const std::string& alias, size_t length);
+
+    Syntax(const Syntax&) = delete;
+    Syntax& operator=(const Syntax&) = delete;
+
+    size_t length() const
+    {
+        return m_length;
+    }
+
+    bool isSyntax() const
+    {
+        return true;
+    }
+
+    const std::string &type() const
+    {
+        return m_type;
+    }
+
+    TokenList::ConstIterator begin() const
+    {
+        return m_children.begin();
+    }
+
+    TokenList::ConstIterator end() const
+    {
+        return m_children.end();
+    }
+
+    const std::string &alias() const
+    {
+        return m_alias;
+    }
+
+    const TokenList &children() const
+	{
+		return m_children;
+	}
+
+public:
+    std::string m_type;
+    TokenList m_children;
+    std::string m_alias;
+    size_t m_length;
+};