mirror of
https://github.com/elder-plinius/P4RS3LT0NGV3.git
synced 2026-02-12 16:52:46 +00:00
dc10a90851
- Implement tool registry system with individual tool modules - Reorganize transformers into categorized source modules - Remove emojiLibrary.js, consolidate into EmojiUtils and emojiData - Fix mobile close button and tooltip functionality - Add build system for transforms and emoji data - Migrate from Python backend to pure JavaScript - Add comprehensive documentation and testing - Improve code organization and maintainability - Ignore generated files (transforms-bundle.js, emojiData.js)
48 lines
3.0 KiB
HTML
48 lines
3.0 KiB
HTML
<div v-if="activeTab === 'tokenizer'" class="tab-content">
|
||
<div class="transform-layout">
|
||
<div class="input-section">
|
||
<div class="section-header">
|
||
<h3><i class="fas fa-layer-group"></i> Tokenizer Visualization <small>{{ tokenizerEngine }}</small></h3>
|
||
<p>Paste text to see how different tokenizers segment it.</p>
|
||
</div>
|
||
<div class="options-grid" style="margin-bottom:8px;">
|
||
<label>
|
||
Engine
|
||
<select v-model="tokenizerEngine" @change="runTokenizer">
|
||
<option value="byte">UTF-8 bytes</option>
|
||
<option value="word">Naive words</option>
|
||
<option value="cl100k">OpenAI: cl100k_base (GPT‑3.5/4)</option>
|
||
<option value="o200k">OpenAI: o200k_base (GPT‑4o)</option>
|
||
<option value="p50k">OpenAI: p50k_base</option>
|
||
<option value="r50k">OpenAI: r50k_base</option>
|
||
</select>
|
||
</label>
|
||
</div>
|
||
<textarea
|
||
id="tokenizer-input"
|
||
v-model="tokenizerInput"
|
||
placeholder="Paste text to visualize tokens"
|
||
@input="runTokenizer"
|
||
></textarea>
|
||
</div>
|
||
|
||
<div class="output-section">
|
||
<div class="output-heading">
|
||
<h4>
|
||
<i class="fas fa-chart-bar"></i> Tokens
|
||
<small>{{ tokenizerTokens.length }} total · {{ tokenizerWordCount }} words · {{ tokenizerCharCount }} chars</small>
|
||
</h4>
|
||
</div>
|
||
<div class="token-tiles" v-if="tokenizerTokens.length" style="display:flex; flex-wrap:wrap; gap:6px;">
|
||
<div v-for="(tok, i) in tokenizerTokens" :key="i" class="token-chip" style="background:var(--button-bg); border:1px solid var(--input-border); padding:6px 8px; border-radius:6px; font-family:'Fira Code', monospace;">
|
||
<span style="opacity:.75; margin-right:6px;">{{ i }}</span>
|
||
<span>{{ tok.text }}</span>
|
||
<span v-if="tok.id !== undefined" style="opacity:.6; margin-left:6px;">#{{ tok.id }}</span>
|
||
</div>
|
||
</div>
|
||
<div class="output-instructions" v-else>
|
||
<small>Tokens will appear here.</small>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div> |