diff --git a/CHANGES.md b/CHANGES.md
index dec7abc279..a5122abd0e 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -80,6 +80,7 @@
 - Retrieve variants with nested components [Taiga #10277](https://tree.taiga.io/project/penpot/us/10277)
 - Create variants in bulk from existing components [Taiga #7926](https://tree.taiga.io/project/penpot/us/7926)
 - Alternative ways of creating variants - Button Design Tab [Taiga #10316](https://tree.taiga.io/project/penpot/us/10316)
+- Fix problem with component swapping panel [Taiga #12175](https://tree.taiga.io/project/penpot/issue/12175)
 
 ### :bug: Bugs fixed
 
diff --git a/common/src/app/common/types/container.cljc b/common/src/app/common/types/container.cljc
index 751fe5518f..13daa4949c 100644
--- a/common/src/app/common/types/container.cljc
+++ b/common/src/app/common/types/container.cljc
@@ -76,11 +76,8 @@
 
 (defn get-shape
   [container shape-id]
-
-  (assert (check-container container))
   (assert (uuid? shape-id)
           "expected valid uuid for `shape-id`")
-
   (-> container
       (get :objects)
       (get shape-id)))
diff --git a/frontend/src/app/main/ui/workspace/sidebar/assets/components.cljs b/frontend/src/app/main/ui/workspace/sidebar/assets/components.cljs
index 7153f062b1..bedf4186e2 100644
--- a/frontend/src/app/main/ui/workspace/sidebar/assets/components.cljs
+++ b/frontend/src/app/main/ui/workspace/sidebar/assets/components.cljs
@@ -267,6 +267,7 @@
            [:div {:class (stl/css-case :component-group-drop-space true
                                        :component-group-drop-space-small (not dragging?))}])
 
+         ;; FIXME: This could be in the thousands. We need to think about paginate this
          (for [component components]
            [:> components-item*
             {:component component
diff --git a/frontend/src/app/main/ui/workspace/sidebar/assets/file_library.cljs b/frontend/src/app/main/ui/workspace/sidebar/assets/file_library.cljs
index a420140724..27cd620eae 100644
--- a/frontend/src/app/main/ui/workspace/sidebar/assets/file_library.cljs
+++ b/frontend/src/app/main/ui/workspace/sidebar/assets/file_library.cljs
@@ -363,13 +363,16 @@
          (fn []
            (st/emit! (dw/unselect-all-assets file-id))))
 
+        variants-counter
+        (mf/with-memo [library]
+          (-> (group-by :variant-id (ctkl/components-seq library))
+              (update-vals count)))
+
         count-variants
         (mf/use-fn
-         (mf/deps library)
+         (mf/deps variants-counter)
          (fn [variant-id]
-           (->> (ctkl/components-seq library)
-                (filterv #(= variant-id (:variant-id %)))
-                count)))]
+           (get variants-counter variant-id)))]
 
     [:div {:class (stl/css :tool-window)
            :on-context-menu dom/prevent-default
diff --git a/frontend/src/app/main/ui/workspace/sidebar/options/menus/component.cljs b/frontend/src/app/main/ui/workspace/sidebar/options/menus/component.cljs
index c4dab6bf87..6bb094b24b 100644
--- a/frontend/src/app/main/ui/workspace/sidebar/options/menus/component.cljs
+++ b/frontend/src/app/main/ui/workspace/sidebar/options/menus/component.cljs
@@ -631,7 +631,13 @@
                               (str/upper (tr "workspace.assets.local-library"))
                               (dm/get-in libraries [current-library-id :name]))
 
-        current-lib-data    (get-in libraries [current-library-id :data])
+        current-lib-data    (mf/with-memo [libraries]
+                              (get-in libraries [current-library-id :data]))
+
+        current-lib-counts  (mf/with-memo [current-lib-data]
+                              (-> (group-by :variant-id
+                                            (ctkl/components-seq current-lib-data))
+                                  (update-vals count)))
 
         components          (->> (get-in libraries [current-library-id :data :components])
                                  vals
@@ -640,9 +646,7 @@
                                  (map #(assoc % :full-name (cpn/merge-path-item-with-dot (:path %) (:name %)))))
 
         count-variants      (fn [component]
-                              (->> (ctkl/components-seq current-lib-data)
-                                   (filterv #(= (:variant-id component) (:variant-id %)))
-                                   count))
+                              (get current-lib-counts (:variant-id component)))
 
         get-subgroups       (fn [path]
                               (let [split-path (cpn/split-path path)]
@@ -767,6 +771,7 @@
 
        [:div {:class (stl/css-case :component-grid (:listing-thumbs? filters)
                                    :component-list (not (:listing-thumbs? filters)))}
+        ;; FIXME: This could be in the thousands. We need to think about paginate this
         (for [item items]
           (if (:id item)
             (let [data       (dm/get-in libraries [current-library-id :data])