diff --git a/modules/presets/index.js b/modules/presets/index.js
index 0f629ef22..c6fbd19db 100644
--- a/modules/presets/index.js
+++ b/modules/presets/index.js
@@ -380,6 +380,24 @@ export function presetIndex(context) {
         setFavorites(favs);
     };
 
+    all.removeFavorite = function(preset, geometry) {
+        var item = all.favoriteMatching(preset, geometry);
+        if (item) {
+            var items = all.getFavorites();
+            items.splice(items.indexOf(item), 1);
+            setFavorites(items);
+        }
+    };
+
+    all.removeRecent = function(preset, geometry) {
+        var item = all.recentMatching(preset, geometry);
+        if (item) {
+            var items = all.getRecents();
+            items.splice(items.indexOf(item), 1);
+            setRecents(items);
+        }
+    };
+
     all.favoriteMatching = function(preset, geometry) {
         var favs = all.getFavorites();
         for (var index in favs) {
@@ -439,14 +457,6 @@ export function presetIndex(context) {
         items.unshift(item);
         setRecents(items);
     };
-    all.removeRecent = function(preset, geometry) {
-        var item = all.recentMatching(preset, geometry);
-        if (item) {
-            var items = all.getRecents();
-            items.splice(items.indexOf(item), 1);
-            setRecents(items);
-        }
-    };
 
     return utilRebind(all, dispatch, 'on');
 }
diff --git a/modules/ui/modes.js b/modules/ui/modes.js
index e348d1340..78a0386ba 100644
--- a/modules/ui/modes.js
+++ b/modules/ui/modes.js
@@ -30,6 +30,22 @@ export function uiModes(context) {
         return context.editable() && mode && mode.id !== 'save';
     }
 
+    function toggleMode(d) {
+
+        if (!enabled(d)) return;
+
+        if (d.button === context.mode().button) {
+            context.enter(modeBrowse(context));
+        } else {
+            if (d.preset &&
+                // don't set a recent as most recent to avoid reordering buttons
+                !d.isRecent()) {
+                context.presets().setMostRecent(d.preset, d.geometry);
+            }
+            context.enter(d);
+        }
+    }
+
 
     return function(selection) {
         context
@@ -141,16 +157,7 @@ export function uiModes(context) {
 
                 if (mode.key) {
                     context.keybinding().on(mode.key, function() {
-                        if (!enabled(mode)) return;
-
-                        if (mode.button === context.mode().button) {
-                            context.enter(modeBrowse(context));
-                        } else {
-                            if (mode.preset && mode.isFavorite()) {
-                                context.presets().setMostRecent(mode.preset, mode.geometry);
-                            }
-                            context.enter(mode);
-                        }
+                        toggleMode(mode);
                     });
                 }
 
@@ -176,20 +183,12 @@ export function uiModes(context) {
                     return classes;
                 })
                 .on('click.mode-buttons', function(d) {
-                    if (!enabled(d)) return;
 
                     // When drawing, ignore accidental clicks on mode buttons - #4042
                     var currMode = context.mode().id;
                     if (/^draw/.test(currMode)) return;
 
-                    if (d.id === currMode) {
-                        context.enter(modeBrowse(context));
-                    } else {
-                        if (d.preset && d.isFavorite()) {
-                            context.presets().setMostRecent(d.preset, d.geometry);
-                        }
-                        context.enter(d);
-                    }
+                    toggleMode(d);
                 })
                 .call(tooltip()
                     .placement('bottom')
@@ -280,7 +279,9 @@ export function uiModes(context) {
                     if (y > 50) {
                         // dragged out of the top bar, remove
                         if (d.isFavorite()) {
-                            context.presets().toggleFavorite(d.preset, d.geometry);
+                            context.presets().removeFavorite(d.preset, d.geometry);
+                            // also remove this as a recent so it doesn't still appear
+                            context.presets().removeRecent(d.preset, d.geometry);
                         } else if (d.isRecent()) {
                             context.presets().removeRecent(d.preset, d.geometry);
                         }