refactor: migrate to modular tool-based architecture

- Implement tool registry system with individual tool modules
- Reorganize transformers into categorized source modules
- Remove emojiLibrary.js, consolidate into EmojiUtils and emojiData
- Fix mobile close button and tooltip functionality
- Add build system for transforms and emoji data
- Migrate from Python backend to pure JavaScript
- Add comprehensive documentation and testing
- Improve code organization and maintainability
- Ignore generated files (transforms-bundle.js, emojiData.js)
This commit is contained in:
Dustin Farley
2025-12-02 20:26:32 -08:00
parent 105084437a
commit dc10a90851
146 changed files with 12712 additions and 8171 deletions
+48
View File
@@ -0,0 +1,48 @@
<div v-if="activeTab === 'tokenizer'" class="tab-content">
<div class="transform-layout">
<div class="input-section">
<div class="section-header">
<h3><i class="fas fa-layer-group"></i> Tokenizer Visualization <small>{{ tokenizerEngine }}</small></h3>
<p>Paste text to see how different tokenizers segment it.</p>
</div>
<div class="options-grid" style="margin-bottom:8px;">
<label>
Engine
<select v-model="tokenizerEngine" @change="runTokenizer">
<option value="byte">UTF-8 bytes</option>
<option value="word">Naive words</option>
<option value="cl100k">OpenAI: cl100k_base (GPT‑3.5/4)</option>
<option value="o200k">OpenAI: o200k_base (GPT‑4o)</option>
<option value="p50k">OpenAI: p50k_base</option>
<option value="r50k">OpenAI: r50k_base</option>
</select>
</label>
</div>
<textarea
id="tokenizer-input"
v-model="tokenizerInput"
placeholder="Paste text to visualize tokens"
@input="runTokenizer"
></textarea>
</div>
<div class="output-section">
<div class="output-heading">
<h4>
<i class="fas fa-chart-bar"></i> Tokens
<small>{{ tokenizerTokens.length }} total · {{ tokenizerWordCount }} words · {{ tokenizerCharCount }} chars</small>
</h4>
</div>
<div class="token-tiles" v-if="tokenizerTokens.length" style="display:flex; flex-wrap:wrap; gap:6px;">
<div v-for="(tok, i) in tokenizerTokens" :key="i" class="token-chip" style="background:var(--button-bg); border:1px solid var(--input-border); padding:6px 8px; border-radius:6px; font-family:'Fira Code', monospace;">
<span style="opacity:.75; margin-right:6px;">{{ i }}</span>
<span>{{ tok.text }}</span>
<span v-if="tok.id !== undefined" style="opacity:.6; margin-left:6px;">#{{ tok.id }}</span>
</div>
</div>
<div class="output-instructions" v-else>
<small>Tokens will appear here.</small>
</div>
</div>
</div>
</div>