Tokenade UI: fix overlapping by boxing controls, adding padding/margins, and enabling wrap; standardize spacing to match Transform page

css/style.css

@@ -1943,6 +1943,121 @@ html {
     padding: 6px 10px;
 }
 
+/* Hacker style controls */
+.hacker-controls {
+    display: grid;
+    grid-template-columns: repeat(auto-fill, minmax(220px, 1fr));
+    gap: 12px;
+    align-items: end;
+}
+.slider-block {
+    background: var(--main-bg-color);
+    border: 1px solid var(--input-border);
+    border-radius: 6px;
+    padding: 8px;
+    display: flex;
+    flex-direction: column;
+}
+.hacker-slider {
+    width: 100%;
+    appearance: none;
+    height: 4px;
+    background: linear-gradient(90deg, rgba(100,181,246,.8), rgba(66,165,245,.4));
+    outline: none;
+    border-radius: 3px;
+}
+.slider-block .hacker-slider { margin-top: 6px; margin-bottom: 4px; }
+.hacker-slider::-webkit-slider-thumb {
+    appearance: none;
+    width: 14px;
+    height: 14px;
+    background: var(--accent-color);
+    border-radius: 50%;
+    box-shadow: 0 0 8px rgba(100,181,246,.6);
+}
+.slider-value {
+    display: inline-block;
+    margin-left: 6px;
+    font-family: 'Fira Code', monospace;
+    color: var(--accent-color);
+}
+.segmented {
+    display: flex;
+    gap: 6px;
+    align-items: center;
+    flex-wrap: wrap;
+    padding: 8px;
+    background: var(--main-bg-color);
+    border: 1px solid var(--input-border);
+    border-radius: 6px;
+}
+.segmented-label { opacity: .7; margin-right: 6px; }
+.segmented button {
+    background: var(--button-bg);
+    border: 1px solid var(--input-border);
+    color: var(--text-color);
+    padding: 6px 10px;
+    border-radius: 6px;
+    cursor: pointer;
+}
+.segmented button.active {
+    background: var(--accent-color);
+    color: var(--main-bg-color);
+    border-color: var(--accent-color);
+}
+.switch { display:flex; gap:6px; align-items:center; padding: 6px 8px; background: var(--main-bg-color); border:1px solid var(--input-border); border-radius:6px; }
+.switch span { opacity:.85; }
+
+/* Tokenade section visual parity with transform cards */
+.token-bomb-section {
+    background: var(--main-bg-color);
+    border-radius: 8px;
+    padding: 16px;
+    border: 1px solid var(--input-border);
+    margin-bottom: 16px;
+}
+.token-bomb-section .section-header { margin-bottom: 10px; }
+.token-bomb-section .output-instructions { margin-top: 8px; }
+
+/* Presets row polish */
+.tokenade-presets {
+    display: flex;
+    align-items: center;
+    gap: 8px;
+    flex-wrap: wrap;
+    margin: 8px 0 12px 0;
+}
+.tokenade-presets .transform-button { min-width: 160px; }
+
+/* Quick picks panel */
+.carrier-quick-grid { 
+    background: var(--secondary-bg);
+    border: 1px solid var(--input-border);
+    border-radius: 8px;
+    padding: 10px;
+    margin-bottom: 8px;
+}
+.carrier-quick-grid .emoji-grid { 
+    border: none !important; 
+    background: transparent !important; 
+    padding: 6px !important; 
+}
+.tokenade-carrier-options { margin-top: 8px; }
+.carrier-quick-grid .emoji-button { 
+    width: 32px; height: 32px; font-size: 18px;
+}
+
+/* Align tokenade options with transform look */
+.tokenade-carrier-options label,
+.hacker-controls .slider-block,
+.hacker-controls .segmented,
+.hacker-controls .switch { 
+    margin-bottom: 6px;
+}
+
+/* Reduce extra whitespace at bottom of tab cards */
+.tab-content .transform-layout > *:last-child { margin-bottom: 0; }
+
 .transform-layout {
     position: relative;
 }

index.html

@@ -78,6 +78,13 @@
                 >
                     <i class="fas fa-bomb"></i> Tokenade
                 </button>
+                <button 
+                    :class="{ active: activeTab === 'tokenizer' }" 
+                    @click="switchToTab('tokenizer')"
+                    title="Tokenizer visualization"
+                >
+                    <i class="fas fa-layer-group"></i> Tokenizer
+                </button>
             </div>
 
             <!-- Steganography Tab -->
@@ -183,6 +190,53 @@
                 </div>
             </div>
 
+            <!-- Tokenizer Visualization Tab -->
+            <div v-if="activeTab === 'tokenizer'" class="tab-content">
+                <div class="transform-layout">
+                    <div class="input-section">
+                        <div class="section-header">
+                            <h3><i class="fas fa-layer-group"></i> Tokenizer Visualization <small>{{ tokenizerEngine }}</small></h3>
+                            <p>Paste text to see how different tokenizers segment it.</p>
+                        </div>
+                        <div class="options-grid" style="margin-bottom:8px;">
+                            <label>
+                                Engine
+                                <select v-model="tokenizerEngine" @change="runTokenizer">
+                                    <option value="byte">UTF-8 bytes</option>
+                                    <option value="word">Naive words</option>
+                                    <option value="gpt3">Experimental: gpt-3-encoder (if available)</option>
+                                </select>
+                            </label>
+                        </div>
+                        <textarea 
+                            id="tokenizer-input" 
+                            v-model="tokenizerInput" 
+                            placeholder="Paste text to visualize tokens"
+                            @input="runTokenizer"
+                        ></textarea>
+                    </div>
+
+                    <div class="output-section">
+                        <div class="output-heading">
+                            <h4>
+                                <i class="fas fa-chart-bar"></i> Tokens 
+                                <small>{{ tokenizerTokens.length }} total · {{ tokenizerWordCount }} words · {{ tokenizerCharCount }} chars</small>
+                            </h4>
+                        </div>
+                        <div class="token-tiles" v-if="tokenizerTokens.length" style="display:flex; flex-wrap:wrap; gap:6px;">
+                            <div v-for="(tok, i) in tokenizerTokens" :key="i" class="token-chip" style="background:var(--button-bg); border:1px solid var(--input-border); padding:6px 8px; border-radius:6px; font-family:'Fira Code', monospace;">
+                                <span style="opacity:.75; margin-right:6px;">{{ i }}</span>
+                                <span>{{ tok.text }}</span>
+                                <span v-if="tok.id !== undefined" style="opacity:.6; margin-left:6px;">#{{ tok.id }}</span>
+                            </div>
+                        </div>
+                        <div class="output-instructions" v-else>
+                            <small>Tokens will appear here.</small>
+                        </div>
+                    </div>
+                </div>
+            </div>
+
             <!-- Tokenade Tab -->
             <div v-if="activeTab === 'tokenade'" class="tab-content">
                 <div class="transform-layout">
@@ -206,43 +260,44 @@
                             <button class="transform-button" title="High density" @click="applyTokenadePreset('heavy')">🗿 Heavyweight</button>
                             <button class="transform-button" title="Extreme density (use with care)" @click="applyTokenadePreset('super')">⚓ Super Heavyweight</button>
                         </div>
-                        <div class="token-bomb-controls options-grid">
-                            <label title="Nesting levels; higher = more layers of grouping">
+                        <div class="token-bomb-controls options-grid hacker-controls">
+                            <label class="slider-block" title="Nesting levels; higher = more layers of grouping">
                                 Depth (nesting)
-                                <input type="number" v-model.number="tbDepth" min="1" max="8" />
+                                <input class="hacker-slider" type="range" v-model.number="tbDepth" min="1" max="8" />
+                                <span class="slider-value">{{ tbDepth }}</span>
                             </label>
-                            <label title="How many items per level; higher = wider structure">
+                            <label class="slider-block" title="How many items per level; higher = wider structure">
                                 Breadth per level
-                                <input type="number" v-model.number="tbBreadth" min="1" max="12" />
+                                <input class="hacker-slider" type="range" v-model.number="tbBreadth" min="1" max="12" />
+                                <span class="slider-value">{{ tbBreadth }}</span>
                             </label>
-                            <label title="How many times to repeat the whole block">
+                            <label class="slider-block" title="How many times to repeat the whole block">
                                 Repeats (blocks)
-                                <input type="number" v-model.number="tbRepeats" min="1" max="100" />
+                                <input class="hacker-slider" type="range" v-model.number="tbRepeats" min="1" max="100" />
+                                <span class="slider-value">{{ tbRepeats }}</span>
                             </label>
-                            <label title="Invisible separator inserted between units">
-                                Separator
-                                <select v-model="tbSeparator">
-                                    <option value="zwj">Zero-Width Joiner (\u200D)</option>
-                                    <option value="zwnj">Zero-Width Non-Joiner (\u200C)</option>
-                                    <option value="zwsp">Zero-Width Space (\u200B)</option>
-                                    <option value="none">None</option>
-                                </select>
-                            </label>
-                            <label title="Adds VS16/VS15 after glyphs to increase token churn">
-                                Include Variation Selectors
+                            <div class="segmented" title="Invisible separator inserted between units">
+                                <div class="segmented-label">Separator</div>
+                                <button :class="{active: tbSeparator==='zwj'}" @click="tbSeparator='zwj'">ZWJ</button>
+                                <button :class="{active: tbSeparator==='zwnj'}" @click="tbSeparator='zwnj'">ZWNJ</button>
+                                <button :class="{active: tbSeparator==='zwsp'}" @click="tbSeparator='zwsp'">ZWSP</button>
+                                <button :class="{active: tbSeparator==='none'}" @click="tbSeparator='none'">None</button>
+                            </div>
+                            <label class="switch" title="Adds VS16/VS15 after glyphs to increase token churn">
                                 <input type="checkbox" v-model="tbIncludeVS" />
+                                <span>Variation Selectors</span>
                             </label>
-                            <label title="Sprinkles random zero-width codepoints between segments">
-                                Include Invisible Noise
+                            <label class="switch" title="Sprinkles random zero-width codepoints between segments">
                                 <input type="checkbox" v-model="tbIncludeNoise" />
+                                <span>Invisible Noise</span>
                             </label>
-                            <label title="Use random emojis instead of cycling a fixed set">
-                                Randomize Emojis
+                            <label class="switch" title="Use random emojis instead of cycling a fixed set">
                                 <input type="checkbox" v-model="tbRandomizeEmojis" />
+                                <span>Randomize</span>
                             </label>
-                            <label title="Automatically copy result after generation">
-                                Auto-copy
+                            <label class="switch" title="Automatically copy result after generation">
                                 <input type="checkbox" v-model="tbAutoCopy" />
+                                <span>Auto-copy</span>
                             </label>
                         </div>
                         <div class="output-instructions">
@@ -748,164 +803,7 @@
         </div>
     </div>
 
-    <!--
-    <!-- Tokenade Tab -->
-    <!-- <div v-if="activeTab === 'tokenade'" class="tab-content"> -->
-        <div class="transform-layout">
-            <div class="token-bomb-section">
-                <div class="section-header">
-                    <h3>
-                        <span title="High-density token payload builder">💥 Tokenade Generator</span>
-                        <small>Craft dense token sequences with emojis and zero-width characters</small>
-                    </h3>
-                </div>
-                <div class="output-instructions">
-                    <small>
-                        <strong>Disclaimer:</strong> These payloads can be disruptive to models, UIs, and tools. Use responsibly and avoid deploying in production or against systems without consent.
-                    </small>
-                </div>
-                <div class="tokenade-presets">
-                    <label style="margin-right:8px">Presets</label>
-                    <button class="transform-button" title="Very light density" @click="applyTokenadePreset('feather')">🪶 Featherweight</button>
-                    <button class="transform-button" title="Light density" @click="applyTokenadePreset('light')">🍃 Lightweight</button>
-                    <button class="transform-button" title="Balanced density" @click="applyTokenadePreset('middle')">🪨 Middleweight</button>
-                    <button class="transform-button" title="High density" @click="applyTokenadePreset('heavy')">🗿 Heavyweight</button>
-                    <button class="transform-button" title="Extreme density (use with care)" @click="applyTokenadePreset('super')">⚓ Super Heavyweight</button>
-                </div>
-                <div class="token-bomb-controls options-grid">
-                    <label title="Nesting levels; higher = more layers of grouping">
-                        Depth (nesting)
-                        <input type="number" v-model.number="tbDepth" min="1" max="8" />
-                    </label>
-                    <label title="How many items per level; higher = wider structure">
-                        Breadth per level
-                        <input type="number" v-model.number="tbBreadth" min="1" max="12" />
-                    </label>
-                    <label title="How many times to repeat the whole block">
-                        Repeats (blocks)
-                        <input type="number" v-model.number="tbRepeats" min="1" max="100" />
-                    </label>
-                    <label title="Invisible separator inserted between units">
-                        Separator
-                        <select v-model="tbSeparator">
-                            <option value="zwj">Zero-Width Joiner (\u200D)</option>
-                            <option value="zwnj">Zero-Width Non-Joiner (\u200C)</option>
-                            <option value="zwsp">Zero-Width Space (\u200B)</option>
-                            <option value="none">None</option>
-                        </select>
-                    </label>
-                    <label title="Adds VS16/VS15 after glyphs to increase token churn">
-                        Include Variation Selectors
-                        <input type="checkbox" v-model="tbIncludeVS" />
-                    </label>
-                    <label title="Sprinkles random zero-width codepoints between segments">
-                        Include Invisible Noise
-                        <input type="checkbox" v-model="tbIncludeNoise" />
-                    </label>
-                    <label title="Use random emojis instead of cycling a fixed set">
-                        Randomize Emojis
-                        <input type="checkbox" v-model="tbRandomizeEmojis" />
-                    </label>
-                    <label title="Automatically copy result after generation">
-                        Auto-copy
-                        <input type="checkbox" v-model="tbAutoCopy" />
-                    </label>
-                </div>
-                <div class="output-instructions">
-                    <small>
-                        Estimated length: {{ estimateTokenadeLength().toLocaleString() }} chars
-                    </small>
-                </div>
-                <div class="tokenade-carrier-options options-grid">
-                    <label title="Condense payload into a single emoji carrier">
-                        Single emoji carrier mode
-                        <input type="checkbox" v-model="tbSingleCarrier" />
-                    </label>
-                    <label v-if="tbSingleCarrier" title="Choose the carrier emoji (type to search, paste, or drop)">
-                        Carrier
-                        <input 
-                            type="text" 
-                            v-model="tbCarrier" 
-                            placeholder="Type to search/paste an emoji"
-                            list="emoji-carrier-list"
-                            @focus="setCarrierFromSelected"
-                            @input="onCarrierInput"
-                        />
-                        <datalist id="emoji-carrier-list">
-                            <option v-for="em in carrierEmojiList" :key="em" :value="em">{{ em }}</option>
-                        </datalist>
-                    </label>
-                </div>
-                <div v-if="tbSingleCarrier" class="tokenade-payload-drop">
-                    <div class="drop-zone" 
-                         @dragover.prevent 
-                         @drop="handleTokenadeDrop"
-                         @paste="handleTokenadePaste"
-                         title="Drag/Drop or paste emojis here to embed into the carrier">
-                        Drop or paste emojis to embed →
-                    </div>
-                    <div class="payload-list" v-if="tbPayloadEmojis.length">
-                        <div class="payload-item" v-for="(em, idx) in tbPayloadEmojis" :key="idx">
-                            <span class="payload-emoji">{{ em }}</span>
-                            <button class="decode-button" @click="removeTokenadePayloadAt(idx)" title="Remove">✖</button>
-                        </div>
-                        <button class="decode-button" @click="clearTokenadePayload" title="Clear all">Clear</button>
-                    </div>
-                </div>
-                <div class="token-bomb-actions">
-                    <button class="transform-button" @click="generateTokenBomb" title="Build the tokenade with current settings">
-                        <i class="fas fa-hammer"></i> Generate Tokenade
-                    </button>
-                    <button class="copy-button" v-if="tokenBombOutput" @click="copyToClipboard(tokenBombOutput)">
-                        <i class="fas fa-copy"></i> Copy
-                    </button>
-                    <div class="output-instructions" v-if="tokenBombOutput">
-                        <small>
-                            Length: {{ tokenBombOutput.length.toLocaleString() }} chars · Tip: Increasing depth/breadth grows size <em>multiplicatively</em>.
-                        </small>
-                    </div>
-                </div>
-                <div class="output-container" v-if="tokenBombOutput">
-                    <textarea readonly v-model="tokenBombOutput" aria-label="Token bomb output"></textarea>
-                </div>
-            </div>
-
-            <!-- Text Payload Generator (simple) -->
-            <div class="transform-section">
-                <h4><i class="fas fa-font"></i> Text Payload Generator</h4>
-                <div class="options-grid">
-                    <label>
-                        Base text
-                        <input type="text" v-model="tpBase" placeholder="Enter base snippet (e.g., hello)" />
-                    </label>
-                    <label>
-                        Repeat count
-                        <input type="number" v-model.number="tpRepeat" min="1" max="1000" />
-                    </label>
-                    <label>
-                        Add combining marks
-                        <input type="checkbox" v-model="tpCombining" />
-                    </label>
-                    <label>
-                        Add zero-width spacing
-                        <input type="checkbox" v-model="tpZW" />
-                    </label>
-                </div>
-                <div class="token-bomb-actions">
-                    <button class="transform-button" @click="generateTextPayload">
-                        <i class="fas fa-hammer"></i> Generate Text Payload
-                    </button>
-                    <button class="copy-button" v-if="textPayload" @click="copyToClipboard(textPayload)">
-                        <i class="fas fa-copy"></i> Copy
-                    </button>
-                </div>
-                <div class="output-container" v-if="textPayload">
-                    <textarea readonly v-model="textPayload" aria-label="Text payload output"></textarea>
-                </div>
-            </div>
-        </div>
-    </div> -->
-    -->
+        
     <!-- Global Advanced Unicode / Steg Options Panel -->
     <div id="unicode-options-panel" class="unicode-options-panel">
         <div class="unicode-panel-header">

js/app.js

@@ -74,6 +74,12 @@ window.app = new Vue({
         tpCombining: true,
         tpZW: false,
         textPayload: '',
+        // Tokenizer tab
+        tokenizerInput: '',
+        tokenizerEngine: 'byte',
+        tokenizerTokens: [],
+        tokenizerCharCount: 0,
+        tokenizerWordCount: 0,
         
         // History of copied content
         copyHistory: [],
@@ -155,6 +161,9 @@ window.app = new Vue({
                     this.initializeCategoryNavigation();
                 });
             }
+            if (tabName === 'tokenizer') {
+                this.$nextTick(() => this.runTokenizer());
+            }
         },
         
         // Get transforms grouped by category
@@ -1899,6 +1908,48 @@ window.app = new Vue({
             this.tbPayloadEmojis.push(...onlyEmojis);
         }
         ,
+        // Tokenizer visualization
+        runTokenizer() {
+            const text = this.tokenizerInput || '';
+            const engine = this.tokenizerEngine;
+            const tokens = [];
+            if (!text) { this.tokenizerTokens = []; this.tokenizerCharCount = 0; this.tokenizerWordCount = 0; return; }
+            if (engine === 'byte') {
+                // Split into UTF-8 bytes, display hex and glyphs
+                const encoder = new TextEncoder();
+                const bytes = encoder.encode(text);
+                for (let i=0;i<bytes.length;i++) {
+                    tokens.push({ id: bytes[i], text: `0x${bytes[i].toString(16).padStart(2,'0')}` });
+                }
+            } else if (engine === 'word') {
+                // Naive word split incl. punctuation
+                const parts = text.split(/(\s+|[\.,!?:;()\[\]{}])/);
+                for (const p of parts) { if (p) tokens.push({ text: p }); }
+            } else if (engine === 'gpt3' && window.gpt3enc && window.gpt3enc.encode) {
+                try {
+                    const ids = window.gpt3enc.encode(text);
+                    for (const id of ids) {
+                        const piece = window.gpt3enc.decode([id]);
+                        tokens.push({ id, text: piece });
+                    }
+                } catch (e) {
+                    console.warn('gpt-3-encoder not available', e);
+                    this.tokenizerEngine = 'byte';
+                    return this.runTokenizer();
+                }
+            } else {
+                // Fallback to bytes
+                const encoder = new TextEncoder();
+                const bytes = encoder.encode(text);
+                for (let i=0;i<bytes.length;i++) tokens.push({ id: bytes[i], text: `0x${bytes[i].toString(16).padStart(2,'0')}` });
+            }
+            this.tokenizerTokens = tokens;
+            // Counts
+            this.tokenizerCharCount = Array.from(text).length;
+            const wordMatches = text.trim().match(/[^\s]+/g) || [];
+            this.tokenizerWordCount = wordMatches.length;
+        }
+        ,
         generateTextPayload() {
             const base = String(this.tpBase || 'A');
             const count = Math.max(1, Math.min(10000, Number(this.tpRepeat) || 1));