P4RS3LT0NGV3/templates/tokenizer.html

<div v-if="activeTab === 'tokenizer'" class="tab-content">
                <div class="transform-layout">
                    <div class="input-section">
                        <div class="section-header">
                            <h3><i class="fas fa-layer-group"></i> Tokenizer Visualization <small>{{ tokenizerEngine }}</small></h3>
                            <p>Paste text to see how different tokenizers segment it.</p>
                        </div>
                        <div class="options-grid u-mb-8">
                            <label>
                                Engine
                                <select v-model="tokenizerEngine" @change="runTokenizer">
                                    <option value="byte">UTF-8 bytes</option>
                                    <option value="word">Naive words</option>
                                    <option value="cl100k">OpenAI: cl100k_base (GPT‑3.5/4)</option>
                                    <option value="o200k">OpenAI: o200k_base (GPT‑4o)</option>
                                    <option value="p50k">OpenAI: p50k_edit (Code editing models)</option>
                                    <option value="r50k">OpenAI: r50k_base</option>
                                </select>
                            </label>
                        </div>
                        <textarea 
                            id="tokenizer-input" 
                            v-model="tokenizerInput" 
                            placeholder="Paste text to visualize tokens"
                            @input="runTokenizer"
                        ></textarea>
                    </div>

                    <div class="output-section">
                        <div class="output-heading">
                            <h4>
                                <i class="fas fa-chart-bar"></i> Tokens 
                                <small>{{ tokenizerTokens.length }} total · {{ tokenizerWordCount }} words · {{ tokenizerCharCount }} chars</small>
                            </h4>
                        </div>
                        <div class="token-tiles tokenizer-tiles" v-if="tokenizerTokens.length">
                            <div v-for="(tok, i) in tokenizerTokens" :key="i" class="token-chip token-chip--tokenizer">
                                <span class="token-chip__idx">{{ i }}</span>
                                <span>{{ tok.text }}</span>
                                <span v-if="tok.id !== undefined" class="token-chip__id">#{{ tok.id }}</span>
                            </div>
                        </div>
                        <div class="output-instructions" v-else>
                            <small>Tokens will appear here.</small>
                        </div>
                    </div>
                </div>
            </div>