From b72704e54b23e72c3b02c005244b238420ecf408 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 30 Sep 2025 12:18:19 +0200 Subject: [PATCH 01/15] :sparkles: Set explicit no-buffering for sse responses --- backend/src/app/http/sse.clj | 3 ++- docker/devenv/files/nginx.conf | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/app/http/sse.clj b/backend/src/app/http/sse.clj index 8d431cc93c..00defbd80e 100644 --- a/backend/src/app/http/sse.clj +++ b/backend/src/app/http/sse.clj @@ -44,7 +44,8 @@ (def default-headers {"Content-Type" "text/event-stream;charset=UTF-8" "Cache-Control" "no-cache, no-store, max-age=0, must-revalidate" - "Pragma" "no-cache"}) + "Pragma" "no-cache" + "X-Accel-Buffering" "no"}) (defn response [handler & {:keys [buf] :or {buf 32} :as opts}] diff --git a/docker/devenv/files/nginx.conf b/docker/devenv/files/nginx.conf index 7b217193d8..45f0dcc6bf 100644 --- a/docker/devenv/files/nginx.conf +++ b/docker/devenv/files/nginx.conf @@ -118,7 +118,6 @@ http { location /api { proxy_pass http://127.0.0.1:6060/api; - proxy_buffering off; proxy_http_version 1.1; } From 54bb879cb6a0b0f258bd5583534f4463826bb629 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 24 Sep 2025 09:58:44 +0200 Subject: [PATCH 02/15] :paperclip: Add several missing imports on repl related namespaces --- backend/dev/user.clj | 1 + backend/src/app/srepl/main.clj | 1 + 2 files changed, 2 insertions(+) diff --git a/backend/dev/user.clj b/backend/dev/user.clj index 04040776ef..f93e0b6f46 100644 --- a/backend/dev/user.clj +++ b/backend/dev/user.clj @@ -35,6 +35,7 @@ [app.util.blob :as blob] [clj-async-profiler.core :as prof] [clojure.contrib.humanize :as hum] + [clojure.datafy :refer [datafy]] [clojure.java.io :as io] [clojure.pprint :refer [pprint print-table]] [clojure.repl :refer :all] diff --git a/backend/src/app/srepl/main.clj b/backend/src/app/srepl/main.clj index 2ffe1caa41..d38757abd1 100644 --- a/backend/src/app/srepl/main.clj +++ b/backend/src/app/srepl/main.clj @@ -40,6 +40,7 @@ [app.util.blob :as blob] [app.util.pointer-map :as pmap] [app.worker :as wrk] + [clojure.datafy :refer [datafy]] [clojure.java.io :as io] [clojure.pprint :refer [print-table]] [clojure.stacktrace :as strace] From 0295f0f7c8d014b9628513ba2ee51cc22a3fcbf2 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 25 Sep 2025 09:56:39 +0200 Subject: [PATCH 03/15] :sparkles: Add better workspace file indexing strategy Improve file indexes initialization on workspace. Instead of initialize indexes for all pages only initialize indexes for the loaded page. --- common/src/app/common/files/indices.cljc | 56 +++-- frontend/src/app/main/data/changes.cljs | 2 +- frontend/src/app/main/data/workspace.cljs | 24 +-- .../main/data/workspace/drawing/common.cljs | 2 +- .../src/app/main/data/workspace/pages.cljs | 32 ++- .../app/main/data/workspace/selection.cljs | 2 +- frontend/src/app/main/snap.cljs | 6 +- .../app/main/ui/workspace/viewport/hooks.cljs | 2 +- frontend/src/app/worker.cljs | 2 - frontend/src/app/worker/index.cljs | 71 +++++-- frontend/src/app/worker/selection.cljs | 192 ++++++++++-------- .../{util/snap_data.cljs => worker/snap.cljs} | 2 +- frontend/src/app/worker/snaps.cljs | 40 ---- frontend/test/frontend_tests/runner.cljs | 16 +- ...p_data_test.cljs => worker_snap_test.cljs} | 160 +++++++-------- 15 files changed, 329 insertions(+), 280 deletions(-) rename frontend/src/app/{util/snap_data.cljs => worker/snap.cljs} (99%) delete mode 100644 frontend/src/app/worker/snaps.cljs rename frontend/test/frontend_tests/{util_snap_data_test.cljs => worker_snap_test.cljs} (71%) diff --git a/common/src/app/common/files/indices.cljc b/common/src/app/common/files/indices.cljc index 5dc13e3eaa..4e177f052c 100644 --- a/common/src/app/common/files/indices.cljc +++ b/common/src/app/common/files/indices.cljc @@ -6,14 +6,29 @@ (ns app.common.files.indices (:require + [app.common.data :as d] + [app.common.data.macros :as dm] [app.common.files.helpers :as cfh] [app.common.uuid :as uuid])) +(defn- generate-index + "An optimized algorithm for calculate parents index that walk from top + to down starting from a provided shape-id. Usefull when you want to + create an index for the whole objects or subpart of the tree." + [index objects shape-id parents] + (let [shape (get objects shape-id) + index (assoc index shape-id parents) + parents (cons shape-id parents)] + (reduce (fn [index shape-id] + (generate-index index objects shape-id parents)) + index + (:shapes shape)))) + (defn generate-child-all-parents-index "Creates an index where the key is the shape id and the value is a set with all the parents" ([objects] - (generate-child-all-parents-index objects (vals objects))) + (generate-index {} objects uuid/zero [])) ([objects shapes] (let [shape->entry @@ -24,24 +39,25 @@ (defn create-clip-index "Retrieves the mask information for an object" [objects parents-index] - (let [retrieve-clips + (let [get-clip-parents + (fn [shape] + (let [shape-id (dm/get-prop shape :id)] + (cond-> [] + (or (and (cfh/frame-shape? shape) + (not (:show-content shape)) + (not= uuid/zero shape-id)) + (cfh/bool-shape? shape)) + (conj shape) + + (:masked-group shape) + (conj (get objects (->> shape :shapes first)))))) + + xform + (comp (map (d/getf objects)) + (mapcat get-clip-parents)) + + populate-with-clips (fn [parents] - (let [lookup-object (fn [id] (get objects id)) - get-clip-parents - (fn [shape] - (cond-> [] - (or (and (= :frame (:type shape)) - (not (:show-content shape)) - (not= uuid/zero (:id shape))) - (cfh/bool-shape? shape)) - (conj shape) + (into [] xform parents))] - (:masked-group shape) - (conj (get objects (->> shape :shapes first)))))] - - (into [] - (comp (map lookup-object) - (mapcat get-clip-parents)) - parents)))] - (-> parents-index - (update-vals retrieve-clips)))) + (d/update-vals parents-index populate-with-clips))) diff --git a/frontend/src/app/main/data/changes.cljs b/frontend/src/app/main/data/changes.cljs index e8fc97862c..6f7799d088 100644 --- a/frontend/src/app/main/data/changes.cljs +++ b/frontend/src/app/main/data/changes.cljs @@ -52,7 +52,7 @@ (->> (rx/from changes) (rx/merge-map (fn [[page-id changes]] (log/debug :hint "update-indexes" :page-id page-id :changes (count changes)) - (mw/ask! {:cmd :index/update-page-index + (mw/ask! {:cmd :index/update :page-id page-id :changes changes}))) (rx/catch (fn [cause] diff --git a/frontend/src/app/main/data/workspace.cljs b/frontend/src/app/main/data/workspace.cljs index cf4921a8ce..88cefbbc0b 100644 --- a/frontend/src/app/main/data/workspace.cljs +++ b/frontend/src/app/main/data/workspace.cljs @@ -21,7 +21,6 @@ [app.common.types.component :as ctc] [app.common.types.fills :as types.fills] [app.common.types.shape :as cts] - [app.common.types.shape-tree :as ctst] [app.common.uuid :as uuid] [app.config :as cf] [app.main.data.changes :as dch] @@ -70,7 +69,6 @@ [app.main.features.pointer-map :as fpmap] [app.main.repo :as rp] [app.main.router :as rt] - [app.main.worker :as mw] [app.render-wasm :as wasm] [app.render-wasm.api :as api] [app.util.dom :as dom] @@ -158,18 +156,9 @@ (->> (fpmap/resolve-file file) (rx/map :data) (rx/map process-fills) - (rx/mapcat - (fn [{:keys [pages-index] :as data}] - (->> (rx/from (seq pages-index)) - (rx/mapcat - (fn [[id page]] - (let [page (update page :objects ctst/start-page-index)] - (->> (mw/ask! {:cmd :index/initialize-page-index :page page}) - (rx/map (fn [_] [id page])))))) - (rx/reduce conj {}) - (rx/map (fn [pages-index] - (let [data (assoc data :pages-index pages-index)] - (assoc file :data (d/removem (comp t/pointer? val) data)))))))))) + (rx/map + (fn [data] + (assoc file :data (d/removem (comp t/pointer? val) data)))))) (defn- check-libraries-synchronozation [file-id libraries] @@ -280,6 +269,8 @@ (ptk/reify ::fetch-bundle ptk/WatchEvent (watch [_ _ stream] + (log/debug :hint "fetch bundle" :file-id (dm/str file-id)) + (let [stopper-s (rx/filter (ptk/type? ::finalize-workspace) stream)] (->> (rx/zip (rp/cmd! :get-file {:id file-id :features features}) (get-file-object-thumbnails file-id)) @@ -288,6 +279,7 @@ (fn [[file thumbnails]] (->> (resolve-file file) (rx/map (fn [file] + (log/trace :hint "file resolved" :file-id file-id) {:file file :file-id file-id :features features @@ -357,6 +349,10 @@ (rx/map deref) (rx/mapcat (fn [{:keys [file]}] + (log/debug :hint "bundle fetched" + :team-id (dm/str team-id) + :file-id (dm/str file-id)) + (rx/of (dpj/initialize-project (:project-id file)) (dwn/initialize team-id file-id) (dwsl/initialize-shape-layout) diff --git a/frontend/src/app/main/data/workspace/drawing/common.cljs b/frontend/src/app/main/data/workspace/drawing/common.cljs index 70f826e593..ab0aed9480 100644 --- a/frontend/src/app/main/data/workspace/drawing/common.cljs +++ b/frontend/src/app/main/data/workspace/drawing/common.cljs @@ -83,7 +83,7 @@ (rx/of (dwsh/add-shape shape {:no-select? (= tool :curve)})) (if (cfh/frame-shape? shape) (rx/concat - (->> (mw/ask! {:cmd :selection/query + (->> (mw/ask! {:cmd :index/query-selection :page-id page-id :rect (:selrect shape) :include-frames? true diff --git a/frontend/src/app/main/data/workspace/pages.cljs b/frontend/src/app/main/data/workspace/pages.cljs index 145b77649b..a9c9d3354e 100644 --- a/frontend/src/app/main/data/workspace/pages.cljs +++ b/frontend/src/app/main/data/workspace/pages.cljs @@ -15,6 +15,7 @@ [app.common.types.components-list :as ctkl] [app.common.types.container :as ctn] [app.common.types.page :as ctp] + [app.common.types.shape-tree :as ctst] [app.common.uuid :as uuid] [app.config :as cf] [app.main.data.changes :as dch] @@ -29,6 +30,7 @@ [app.main.errors] [app.main.features :as features] [app.main.router :as rt] + [app.main.worker :as mw] [app.render-wasm.shape :as wasm.shape] [app.util.http :as http] [app.util.i18n :as i18n :refer [tr]] @@ -56,16 +58,21 @@ (some? metadata) (cf/resolve-file-media metadata) (some? fill-image) (cf/resolve-file-media fill-image)))))) +(defn- get-page-cache + [state file-id page-id] + (dm/get-in state [:workspace-cache [file-id page-id]])) (defn- initialize-page* "Second phase of page initialization, once we know the page is available in the state" - [file-id page-id page] + [file-id page-id] (ptk/reify ::initialize-page* ptk/UpdateEvent (update [_ state] - ;; selection; when user abandon the current page, the selection is lost - (let [local (dm/get-in state [:workspace-cache [file-id page-id]] default-workspace-local)] + (let [state (dsh/update-page state file-id page-id #(update % :objects ctst/start-page-index)) + page (dsh/lookup-page state file-id page-id) + local (or (get-page-cache state file-id page-id) default-workspace-local)] + (-> state (assoc :current-page-id page-id) (assoc :workspace-local (assoc local :selected (d/ordered-set))) @@ -75,11 +82,16 @@ (update :workspace-layout layout/load-layout-flags) (update :workspace-global layout/load-layout-state)))) - ptk/EffectEvent - (effect [_ _ _] - (let [uris (into #{} xf:collect-file-media (:objects page))] - (->> (rx/from uris) - (rx/subs! #(http/fetch-data-uri % false))))))) + ptk/WatchEvent + (watch [_ state _] + (let [page (dsh/lookup-page state file-id page-id) + uris (into #{} xf:collect-file-media (:objects page))] + (rx/merge + (->> (rx/from uris) + (rx/map #(http/fetch-data-uri % false)) + (rx/ignore)) + (->> (mw/ask! {:cmd :index/initialize :page page}) + (rx/ignore))))))) (defn initialize-page [file-id page-id] @@ -89,9 +101,9 @@ (ptk/reify ::initialize-page ptk/WatchEvent (watch [_ state _] - (if-let [page (dsh/lookup-page state file-id page-id)] + (if (dsh/lookup-page state file-id page-id) (rx/concat - (rx/of (initialize-page* file-id page-id page) + (rx/of (initialize-page* file-id page-id) (dwth/watch-state-changes file-id page-id) (dwl/watch-component-changes)) (let [profile (:profile state) diff --git a/frontend/src/app/main/data/workspace/selection.cljs b/frontend/src/app/main/data/workspace/selection.cljs index 498dbd2bad..0320573b15 100644 --- a/frontend/src/app/main/data/workspace/selection.cljs +++ b/frontend/src/app/main/data/workspace/selection.cljs @@ -344,7 +344,7 @@ (if (some? selrect) (->> (ask-worker - {:cmd :selection/query + {:cmd :index/query-selection :page-id page-id :rect selrect :include-frames? true diff --git a/frontend/src/app/main/snap.cljs b/frontend/src/app/main/snap.cljs index 090e624410..f485880536 100644 --- a/frontend/src/app/main/snap.cljs +++ b/frontend/src/app/main/snap.cljs @@ -84,7 +84,7 @@ (let [value (get point coord) vbox @refs/vbox ranges [[(- value (/ 0.5 zoom)) (+ value (/ 0.5 zoom))]]] - (->> (mw/ask! {:cmd :snaps/range-query + (->> (mw/ask! {:cmd :index/query-snap :page-id page-id :frame-id frame-id :axis coord @@ -101,7 +101,7 @@ (mapv #(vector (- % snap-accuracy) (+ % snap-accuracy)))) vbox @refs/vbox] - (->> (mw/ask! {:cmd :snaps/range-query + (->> (mw/ask! {:cmd :index/query-snap :page-id page-id :frame-id frame-id :axis coord @@ -217,7 +217,7 @@ (defn select-shapes-area [page-id frame-id selected objects area] - (->> (mw/ask! {:cmd :selection/query + (->> (mw/ask! {:cmd :index/query-selection :page-id page-id :frame-id frame-id :include-frames? true diff --git a/frontend/src/app/main/ui/workspace/viewport/hooks.cljs b/frontend/src/app/main/ui/workspace/viewport/hooks.cljs index b49699e7f0..f573e405bf 100644 --- a/frontend/src/app/main/ui/workspace/viewport/hooks.cljs +++ b/frontend/src/app/main/ui/workspace/viewport/hooks.cljs @@ -194,7 +194,7 @@ (if (mf/ref-val hover-disabled-ref) (rx/of nil) (->> (mw/ask-buffered! - {:cmd :selection/query + {:cmd :index/query-selection :page-id page-id :rect rect :include-frames? true diff --git a/frontend/src/app/worker.cljs b/frontend/src/app/worker.cljs index d5d5f18e44..89476d0408 100644 --- a/frontend/src/app/worker.cljs +++ b/frontend/src/app/worker.cljs @@ -16,8 +16,6 @@ [app.worker.import] [app.worker.index] [app.worker.messages :as wm] - [app.worker.selection] - [app.worker.snaps] [app.worker.thumbnails] [beicon.v2.core :as rx] [promesa.core :as p])) diff --git a/frontend/src/app/worker/index.cljs b/frontend/src/app/worker/index.cljs index 77c2b31f30..b89fad06ee 100644 --- a/frontend/src/app/worker/index.cljs +++ b/frontend/src/app/worker/index.cljs @@ -9,26 +9,69 @@ (:require [app.common.data.macros :as dm] [app.common.files.changes :as ch] + [app.common.geom.rect :as grc] + [app.common.logging :as log] + [app.common.time :as ct] [app.worker.impl :as impl] + [app.worker.selection :as selection] + [app.worker.snap :as snap] [okulary.core :as l])) +(log/set-level! :info) + (defonce state (l/atom {:pages-index {}})) -(defmethod impl/handler :index/initialize-page-index +(defmethod impl/handler :index/initialize [{:keys [page] :as message}] - (swap! state update :pages-index assoc (:id page) page) - (impl/handler (assoc message :cmd :selection/initialize-page-index)) - (impl/handler (assoc message :cmd :snaps/initialize-page-index))) + (let [tpoint (ct/tpoint-ms)] + (try + (swap! state update :pages-index assoc (:id page) page) + (swap! state update ::selection selection/add-page page) + (swap! state update ::snap snap/add-page page) -(defmethod impl/handler :index/update-page-index + (finally + (let [elapsed (tpoint)] + (log/dbg :hint "page indexed" :id (:id page) :elapsed elapsed ::log/sync? true)))) + nil)) + +(defmethod impl/handler :index/update [{:keys [page-id changes] :as message}] + (let [tpoint (ct/tpoint-ms)] + (try + (let [old-page (dm/get-in @state [:pages-index page-id]) + new-page (-> state + (swap! ch/process-changes changes false) + (dm/get-in [:pages-index page-id]))] - (let [old-page (dm/get-in @state [:pages-index page-id]) - new-page (-> state - (swap! ch/process-changes changes false) - (dm/get-in [:pages-index page-id])) - message (assoc message - :old-page old-page - :new-page new-page)] - (impl/handler (assoc message :cmd :selection/update-page-index)) - (impl/handler (assoc message :cmd :snaps/update-page-index)))) + (swap! state update ::snap snap/update-page old-page new-page) + (swap! state update ::selection selection/update-page old-page new-page)) + (finally + (let [elapsed (tpoint)] + (log/dbg :hint "page index updated" :id page-id :elapsed elapsed ::log/sync? true)))) + nil)) + +;; FIXME: schema + +(defmethod impl/handler :index/query-snap + [{:keys [page-id frame-id axis ranges bounds] :as message}] + (if-let [index (get @state ::snap)] + (let [match-bounds? + (fn [[_ data]] + (some #(or (= :guide (:type %)) + (= :layout (:type %)) + (grc/contains-point? bounds (:pt %))) data)) + + xform + (comp (mapcat #(snap/query index page-id frame-id axis %)) + (distinct) + (filter match-bounds?))] + (into [] xform ranges)) + [])) + +;; FIXME: schema + +(defmethod impl/handler :index/query-selection + [message] + (if-let [index (get @state ::selection)] + (selection/query index message) + [])) diff --git a/frontend/src/app/worker/selection.cljs b/frontend/src/app/worker/selection.cljs index f6a917b003..e48fbfc2c9 100644 --- a/frontend/src/app/worker/selection.cljs +++ b/frontend/src/app/worker/selection.cljs @@ -17,46 +17,62 @@ [app.common.types.modifiers :as ctm] [app.common.uuid :as uuid] [app.util.quadtree :as qdt] - [app.worker.impl :as impl] - [clojure.set :as set] - [okulary.core :as l])) + [clojure.set :as set])) -;; FIXME: performance shape & rect static props +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; IMPL +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (def ^:const padding-percent 0.10) -(defonce state (l/atom {})) +(defn- index-shape + "A reducing function that ads a shape to the index" + [objects parents-index clip-index index shape] + (let [bounds + (cond + (and ^boolean (cfh/text-shape? shape) + ^boolean (some? (:position-data shape)) + ^boolean (d/not-empty? (:position-data shape))) + (gst/shape->bounds shape) -(defn make-index-shape - [objects parents-index clip-parents-index] - (fn [index shape] - (let [{:keys [x y width height]} - (cond - (and ^boolean (cfh/text-shape? shape) - ^boolean (some? (:position-data shape)) - ^boolean (d/not-empty? (:position-data shape))) - (gst/shape->bounds shape) + :else + (grc/points->rect (:points shape))) - :else - (grc/points->rect (:points shape))) + bound + #js {:x (dm/get-prop bounds :x) + :y (dm/get-prop bounds :y) + :width (dm/get-prop bounds :width) + :height (dm/get-prop bounds :height)} - shape-bound #js {:x x :y y :width width :height height} + shape-id + (dm/get-prop shape :id) - parents (get parents-index (:id shape)) - clip-parents (get clip-parents-index (:id shape)) + frame-id + (dm/get-prop shape :frame-id) - frame (when (and (not= :frame (:type shape)) - (not= (:frame-id shape) uuid/zero)) - (get objects (:frame-id shape)))] - (qdt/insert index - (:id shape) - shape-bound - (assoc shape - :frame frame - :clip-parents clip-parents - :parents parents))))) + shape-type + (dm/get-prop shape :type) -(defn objects-bounds + parents + (get parents-index shape-id) + + clip-parents + (get clip-index shape-id) + + frame + (when (and (not= :frame shape-type) + (not= frame-id uuid/zero)) + (get objects frame-id))] + + (qdt/insert index + shape-id + bound + (assoc shape + :frame frame + :clip-parents clip-parents + :parents parents)))) + +(defn- objects-bounds "Calculates the bounds of the quadtree given a objects map." [objects] (-> objects @@ -64,7 +80,7 @@ vals gsh/shapes->rect)) -(defn add-padding-bounds +(defn- add-padding-bounds "Adds a padding to the bounds defined as a percent in the constant `padding-percent`. For a value of 0.1 will add a 20% width increase (2 x padding)" [bounds] @@ -81,41 +97,48 @@ (defn- create-index [objects] - (let [shapes (-> objects (dissoc uuid/zero) vals) - parents-index (cfi/generate-child-all-parents-index objects) - clip-parents-index (cfi/create-clip-index objects parents-index) - - root-shapes (cfh/get-immediate-children objects uuid/zero) - bounds (-> root-shapes gsh/shapes->rect add-padding-bounds) - - index-shape (make-index-shape objects parents-index clip-parents-index) - initial-quadtree (qdt/create (clj->js bounds)) - - index (reduce index-shape initial-quadtree shapes)] + (let [parents-index (cfi/generate-child-all-parents-index objects) + clip-index (cfi/create-clip-index objects parents-index) + root-shapes (cfh/get-immediate-children objects uuid/zero) + bounds (-> root-shapes gsh/shapes->rect add-padding-bounds) + index (reduce-kv #(index-shape objects parents-index clip-index %1 %3) + (qdt/create (clj->js bounds)) + (dissoc objects uuid/zero))] {:index index :bounds bounds})) +;; FIXME: optimize (defn- update-index [{index :index :as data} old-objects new-objects] - (let [changes? (fn [id] - (not= (get old-objects id) - (get new-objects id))) + (let [object-changed? + (fn [id] + (not= (get old-objects id) + (get new-objects id))) - changed-ids (into #{} - (comp (filter #(not= % uuid/zero)) - (filter changes?) - (mapcat #(into [%] (cfh/get-children-ids new-objects %)))) - (set/union (set (keys old-objects)) - (set (keys new-objects)))) + changed-ids + (into #{} + (comp (filter #(not= % uuid/zero)) + (filter object-changed?) + (mapcat #(into [%] (cfh/get-children-ids new-objects %)))) - shapes (->> changed-ids (mapv #(get new-objects %)) (filterv (comp not nil?))) - parents-index (cfi/generate-child-all-parents-index new-objects shapes) - clip-parents-index (cfi/create-clip-index new-objects parents-index) + (set/union (set (keys old-objects)) + (set (keys new-objects)))) - new-index (qdt/remove-all index changed-ids) + shapes + (->> changed-ids + (map #(get new-objects %)) + (filterv (comp not nil?))) - index-shape (make-index-shape new-objects parents-index clip-parents-index) - index (reduce index-shape new-index shapes)] + parents-index + (cfi/generate-child-all-parents-index new-objects shapes) + + clip-index + (cfi/create-clip-index new-objects parents-index) + + index + (reduce #(index-shape new-objects parents-index clip-index %1 %2) + (qdt/remove-all index changed-ids) + shapes)] (assoc data :index index))) @@ -231,35 +254,36 @@ (map :id)) result))) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; PUBLIC API +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(defmethod impl/handler :selection/initialize-page-index - [{:keys [page] :as message}] - (letfn [(add-page [state {:keys [id objects] :as page}] - (assoc state id (create-index objects)))] - (swap! state add-page page) - nil)) +(defn add-page + "Add a page index to the state" + [state {:keys [id objects] :as page}] + (assoc state id (create-index objects))) -(defmethod impl/handler :selection/update-page-index - [{:keys [page-id old-page new-page] :as message}] - (swap! state update page-id - (fn [index] - (let [old-objects (:objects old-page) - new-objects (:objects new-page) - old-bounds (:bounds index) - new-bounds (objects-bounds new-objects)] +(defn update-page + "Update page index on the state" + [state old-page new-page] + (let [page-id (get old-page :id)] + (update state page-id + (fn [index] + (let [old-objects (:objects old-page) + new-objects (:objects new-page) + old-bounds (:bounds index) + new-bounds (objects-bounds new-objects)] - ;; If the new bounds are contained within the old bounds - ;; we can update the index. Otherwise we need to - ;; re-create it. - (if (and (some? index) - (grc/contains-rect? old-bounds new-bounds)) - (update-index index old-objects new-objects) - (create-index new-objects))))) - nil) + ;; If the new bounds are contained within the old bounds + ;; we can update the index. Otherwise we need to + ;; re-create it. + (if (and (some? index) + (grc/contains-rect? old-bounds new-bounds)) + (update-index index old-objects new-objects) + (create-index new-objects))))))) -(defmethod impl/handler :selection/query - [{:keys [page-id rect frame-id full-frame? include-frames? ignore-groups? clip-children? using-selrect?] - :or {full-frame? false include-frames? false clip-children? true using-selrect? false} - :as message}] - (when-let [index (get @state page-id)] +(defn query + [index {:keys [page-id rect frame-id full-frame? include-frames? ignore-groups? clip-children? using-selrect?] + :or {full-frame? false include-frames? false clip-children? true using-selrect? false}}] + (when-let [index (get index page-id)] (query-index index rect frame-id full-frame? include-frames? ignore-groups? clip-children? using-selrect?))) diff --git a/frontend/src/app/util/snap_data.cljs b/frontend/src/app/worker/snap.cljs similarity index 99% rename from frontend/src/app/util/snap_data.cljs rename to frontend/src/app/worker/snap.cljs index d8fc89cea9..494c34adff 100644 --- a/frontend/src/app/util/snap_data.cljs +++ b/frontend/src/app/worker/snap.cljs @@ -4,7 +4,7 @@ ;; ;; Copyright (c) KALEIDOS INC -(ns app.util.snap-data +(ns app.worker.snap "Data structure that holds and retrieves the data to make the snaps. Internally is implemented with a balanced binary tree that queries by range. https://en.wikipedia.org/wiki/Range_tree" diff --git a/frontend/src/app/worker/snaps.cljs b/frontend/src/app/worker/snaps.cljs deleted file mode 100644 index 77bf5d8f5a..0000000000 --- a/frontend/src/app/worker/snaps.cljs +++ /dev/null @@ -1,40 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS INC - -(ns app.worker.snaps - (:require - [app.common.geom.rect :as grc] - [app.util.snap-data :as sd] - [app.worker.impl :as impl] - [okulary.core :as l])) - -(defonce state (l/atom {})) - -;; Public API -(defmethod impl/handler :snaps/initialize-page-index - [{:keys [page] :as message}] - (swap! state sd/add-page page) - nil) - -(defmethod impl/handler :snaps/update-page-index - [{:keys [old-page new-page] :as message}] - (swap! state sd/update-page old-page new-page) - nil) - -(defmethod impl/handler :snaps/range-query - [{:keys [page-id frame-id axis ranges bounds] :as message}] - (let [match-bounds? - (fn [[_ data]] - (some #(or (= :guide (:type %)) - (= :layout (:type %)) - (grc/contains-point? bounds (:pt %))) data))] - (->> (into [] - (comp (mapcat #(sd/query @state page-id frame-id axis %)) - (distinct)) - ranges) - (filter match-bounds?)))) - - diff --git a/frontend/test/frontend_tests/runner.cljs b/frontend/test/frontend_tests/runner.cljs index c3def3b7de..9c8db535e6 100644 --- a/frontend/test/frontend_tests/runner.cljs +++ b/frontend/test/frontend_tests/runner.cljs @@ -17,7 +17,7 @@ [frontend-tests.tokens.token-form-test] [frontend-tests.util-range-tree-test] [frontend-tests.util-simple-math-test] - [frontend-tests.util-snap-data-test])) + [frontend-tests.worker-snap-test])) (enable-console-print!) @@ -29,6 +29,8 @@ (defn init [] (t/run-tests + 'frontend-tests.basic-shapes-test + 'frontend-tests.data.workspace-colors-test 'frontend-tests.helpers-shapes-test 'frontend-tests.logic.comp-remove-swap-slots-test 'frontend-tests.logic.components-and-tokens @@ -36,13 +38,11 @@ 'frontend-tests.logic.frame-guides-test 'frontend-tests.logic.groups-test 'frontend-tests.plugins.context-shapes-test - 'frontend-tests.util-range-tree-test - 'frontend-tests.util-snap-data-test - 'frontend-tests.util-simple-math-test - 'frontend-tests.basic-shapes-test - 'frontend-tests.data.workspace-colors-test + 'frontend-tests.tokens.import-export-test 'frontend-tests.tokens.logic.token-actions-test 'frontend-tests.tokens.logic.token-data-test - 'frontend-tests.tokens.import-export-test 'frontend-tests.tokens.style-dictionary-test - 'frontend-tests.tokens.token-form-test)) + 'frontend-tests.tokens.token-form-test + 'frontend-tests.util-range-tree-test + 'frontend-tests.util-simple-math-test + 'frontend-tests.worker-snap-test)) diff --git a/frontend/test/frontend_tests/util_snap_data_test.cljs b/frontend/test/frontend_tests/worker_snap_test.cljs similarity index 71% rename from frontend/test/frontend_tests/util_snap_data_test.cljs rename to frontend/test/frontend_tests/worker_snap_test.cljs index 0ad546a672..280db50e56 100644 --- a/frontend/test/frontend_tests/util_snap_data_test.cljs +++ b/frontend/test/frontend_tests/worker_snap_test.cljs @@ -4,12 +4,12 @@ ;; ;; Copyright (c) KALEIDOS INC -(ns frontend-tests.util-snap-data-test +(ns frontend-tests.worker-snap-test (:require [app.common.files.builder :as fb] [app.common.types.shape :as cts] [app.common.uuid :as uuid] - [app.util.snap-data :as sd] + [app.worker.snap :as snap] [cljs.pprint :refer [pprint]] [cljs.test :as t :include-macros true])) @@ -21,9 +21,9 @@ (fn [] (uuid/custom 123456789 (swap! counter inc))))) -(t/deftest test-create-index +(t/deftest create-index (t/testing "Create empty data" - (let [data (sd/make-snap-data)] + (let [data (snap/make-snap-data)] (t/is (some? data)))) (t/testing "Add empty page (only root-frame)" @@ -32,8 +32,8 @@ (fb/add-page {:name "Page 1"}) (fb/get-current-page)) - data (-> (sd/make-snap-data) - (sd/add-page page))] + data (-> (snap/make-snap-data) + (snap/add-page page))] (t/is (some? data)))) (t/testing "Create simple shape on root" @@ -48,10 +48,10 @@ :height 100})) page (fb/get-current-page state) - data (-> (sd/make-snap-data) - (sd/add-page page)) + data (-> (snap/make-snap-data) + (snap/add-page page)) - result-x (sd/query data (:id page) uuid/zero :x [0 100])] + result-x (snap/query data (:id page) uuid/zero :x [0 100])] (t/is (some? data)) @@ -82,11 +82,11 @@ page (fb/get-current-page state) ;; frame-id (::fb/last-id file) - data (-> (sd/make-snap-data) - (sd/add-page page)) + data (-> (snap/make-snap-data) + (snap/add-page page)) - result-zero-x (sd/query data (:id page) uuid/zero :x [0 100]) - result-frame-x (sd/query data (:id page) frame-id :x [0 100])] + result-zero-x (snap/query data (:id page) uuid/zero :x [0 100]) + result-frame-x (snap/query data (:id page) frame-id :x [0 100])] (t/is (some? data)) (t/is (= (count result-zero-x) 3)) @@ -116,11 +116,11 @@ page (fb/get-current-page state) - data (-> (sd/make-snap-data) - (sd/add-page page)) + data (-> (snap/make-snap-data) + (snap/add-page page)) - result-zero-x (sd/query data (:id page) uuid/zero :x [0 100]) - result-frame-x (sd/query data (:id page) frame-id :x [0 100])] + result-zero-x (snap/query data (:id page) uuid/zero :x [0 100]) + result-frame-x (snap/query data (:id page) frame-id :x [0 100])] (t/is (some? data)) (t/is (= (count result-zero-x) 3)) @@ -137,13 +137,13 @@ frame-id (::fb/last-id state) page (fb/get-current-page state) - data (-> (sd/make-snap-data) - (sd/add-page page)) + data (-> (snap/make-snap-data) + (snap/add-page page)) - result-zero-x (sd/query data (:id page) uuid/zero :x [0 100]) - result-zero-y (sd/query data (:id page) uuid/zero :y [0 100]) - result-frame-x (sd/query data (:id page) frame-id :x [0 100]) - result-frame-y (sd/query data (:id page) frame-id :y [0 100])] + result-zero-x (snap/query data (:id page) uuid/zero :x [0 100]) + result-zero-y (snap/query data (:id page) uuid/zero :y [0 100]) + result-frame-x (snap/query data (:id page) frame-id :x [0 100]) + result-frame-y (snap/query data (:id page) frame-id :y [0 100])] (t/is (some? data)) ;; We can snap in the root @@ -168,13 +168,13 @@ page (fb/get-current-page state) - data (-> (sd/make-snap-data) - (sd/add-page page)) + data (-> (snap/make-snap-data) + (snap/add-page page)) - result-zero-x (sd/query data (:id page) uuid/zero :x [0 100]) - result-zero-y (sd/query data (:id page) uuid/zero :y [0 100]) - result-frame-x (sd/query data (:id page) frame-id :x [0 100]) - result-frame-y (sd/query data (:id page) frame-id :y [0 100])] + result-zero-x (snap/query data (:id page) uuid/zero :x [0 100]) + result-zero-y (snap/query data (:id page) uuid/zero :y [0 100]) + result-frame-x (snap/query data (:id page) frame-id :x [0 100]) + result-frame-y (snap/query data (:id page) frame-id :y [0 100])] (t/is (some? data)) ;; We can snap in the root @@ -185,7 +185,7 @@ (t/is (= (count result-frame-x) 1)) (t/is (= (count result-frame-y) 0))))) -(t/deftest test-update-index +(t/deftest update-index (t/testing "Create frame on root and then remove it." (let [state (-> (fb/create-state) (fb/add-file {:name "Test"}) @@ -200,17 +200,17 @@ shape-id (::fb/last-id state) page (fb/get-current-page state) - data (-> (sd/make-snap-data) - (sd/add-page page)) + data (-> (snap/make-snap-data) + (snap/add-page page)) state (-> state (fb/delete-shape shape-id)) new-page (fb/get-current-page state) - data (sd/update-page data page new-page) + data (snap/update-page data page new-page) - result-x (sd/query data (:id page) uuid/zero :x [0 100]) - result-y (sd/query data (:id page) uuid/zero :y [0 100])] + result-x (snap/query data (:id page) uuid/zero :x [0 100]) + result-y (snap/query data (:id page) uuid/zero :y [0 100])] (t/is (some? data)) (t/is (= (count result-x) 0)) @@ -231,16 +231,16 @@ page (fb/get-current-page state) ;; frame-id (::fb/last-id state) - data (-> (sd/make-snap-data) - (sd/add-page page)) + data (-> (snap/make-snap-data) + (snap/add-page page)) state (fb/delete-shape state shape-id) new-page (fb/get-current-page state) - data (sd/update-page data page new-page) + data (snap/update-page data page new-page) - result-x (sd/query data (:id page) uuid/zero :x [0 100]) - result-y (sd/query data (:id page) uuid/zero :y [0 100])] + result-x (snap/query data (:id page) uuid/zero :x [0 100]) + result-y (snap/query data (:id page) uuid/zero :y [0 100])] (t/is (some? data)) (t/is (= (count result-x) 0)) @@ -263,16 +263,16 @@ state (fb/close-board state) page (fb/get-current-page state) - data (-> (sd/make-snap-data) - (sd/add-page page)) + data (-> (snap/make-snap-data) + (snap/add-page page)) state (fb/delete-shape state shape-id) new-page (fb/get-current-page state) - data (sd/update-page data page new-page) + data (snap/update-page data page new-page) - result-zero-x (sd/query data (:id page) uuid/zero :x [0 100]) - result-frame-x (sd/query data (:id page) frame-id :x [0 100])] + result-zero-x (snap/query data (:id page) uuid/zero :x [0 100]) + result-frame-x (snap/query data (:id page) frame-id :x [0 100])] (t/is (some? data)) (t/is (= (count result-zero-x) 3)) @@ -291,18 +291,18 @@ frame-id (::fb/last-id state) page (fb/get-current-page state) - data (-> (sd/make-snap-data) - (sd/add-page page)) + data (-> (snap/make-snap-data) + (snap/add-page page)) new-page (-> (fb/delete-guide state guide-id) (fb/get-current-page)) - data (sd/update-page data page new-page) + data (snap/update-page data page new-page) - result-zero-x (sd/query data (:id page) uuid/zero :x [0 100]) - result-zero-y (sd/query data (:id page) uuid/zero :y [0 100]) - result-frame-x (sd/query data (:id page) frame-id :x [0 100]) - result-frame-y (sd/query data (:id page) frame-id :y [0 100])] + result-zero-x (snap/query data (:id page) uuid/zero :x [0 100]) + result-zero-y (snap/query data (:id page) uuid/zero :y [0 100]) + result-frame-x (snap/query data (:id page) frame-id :x [0 100]) + result-frame-y (snap/query data (:id page) frame-id :y [0 100])] (t/is (some? data)) ;; We can snap in the root @@ -325,17 +325,17 @@ guide-id (::fb/last-id file) page (fb/get-current-page file) - data (-> (sd/make-snap-data) (sd/add-page page)) + data (-> (snap/make-snap-data) (snap/add-page page)) new-page (-> (fb/delete-guide file guide-id) (fb/get-current-page)) - data (sd/update-page data page new-page) + data (snap/update-page data page new-page) - result-zero-x (sd/query data (:id page) uuid/zero :x [0 100]) - result-zero-y (sd/query data (:id page) uuid/zero :y [0 100]) - result-frame-x (sd/query data (:id page) frame-id :x [0 100]) - result-frame-y (sd/query data (:id page) frame-id :y [0 100])] + result-zero-x (snap/query data (:id page) uuid/zero :x [0 100]) + result-zero-y (snap/query data (:id page) uuid/zero :y [0 100]) + result-frame-x (snap/query data (:id page) frame-id :x [0 100]) + result-frame-y (snap/query data (:id page) frame-id :y [0 100])] (t/is (some? data)) ;; We can snap in the root (t/is (= (count result-zero-x) 0)) @@ -358,8 +358,8 @@ frame-id (::fb/last-id state) page (fb/get-current-page state) - data (-> (sd/make-snap-data) - (sd/add-page page)) + data (-> (snap/make-snap-data) + (snap/add-page page)) state (fb/update-shape state frame-id (fn [shape] @@ -370,12 +370,12 @@ new-page (fb/get-current-page state) - data (sd/update-page data page new-page) + data (snap/update-page data page new-page) - result-zero-x-1 (sd/query data (:id page) uuid/zero :x [0 100]) - result-frame-x-1 (sd/query data (:id page) frame-id :x [0 100]) - result-zero-x-2 (sd/query data (:id page) uuid/zero :x [200 300]) - result-frame-x-2 (sd/query data (:id page) frame-id :x [200 300])] + result-zero-x-1 (snap/query data (:id page) uuid/zero :x [0 100]) + result-frame-x-1 (snap/query data (:id page) frame-id :x [0 100]) + result-zero-x-2 (snap/query data (:id page) uuid/zero :x [200 300]) + result-frame-x-2 (snap/query data (:id page) frame-id :x [200 300])] (t/is (some? data)) (t/is (= (count result-zero-x-1) 0)) @@ -396,8 +396,8 @@ shape-id (::fb/last-id state) page (fb/get-current-page state) - data (-> (sd/make-snap-data) - (sd/add-page page)) + data (-> (snap/make-snap-data) + (snap/add-page page)) state (fb/update-shape state shape-id (fn [shape] @@ -408,10 +408,10 @@ new-page (fb/get-current-page state) ;; FIXME: update - data (sd/update-page data page new-page) + data (snap/update-page data page new-page) - result-zero-x-1 (sd/query data (:id page) uuid/zero :x [0 100]) - result-zero-x-2 (sd/query data (:id page) uuid/zero :x [200 300])] + result-zero-x-1 (snap/query data (:id page) uuid/zero :x [0 100]) + result-zero-x-2 (snap/query data (:id page) uuid/zero :x [200 300])] (t/is (some? data)) (t/is (= (count result-zero-x-1) 0)) @@ -432,22 +432,22 @@ frame-id (::fb/last-id state) page (fb/get-current-page state) - data (-> (sd/make-snap-data) (sd/add-page page)) + data (-> (snap/make-snap-data) (snap/add-page page)) new-page (-> (fb/update-guide state (assoc guide :position 150)) (fb/get-current-page)) - data (sd/update-page data page new-page) + data (snap/update-page data page new-page) - result-zero-x-1 (sd/query data (:id page) uuid/zero :x [0 100]) - result-zero-y-1 (sd/query data (:id page) uuid/zero :y [0 100]) - result-frame-x-1 (sd/query data (:id page) frame-id :x [0 100]) - result-frame-y-1 (sd/query data (:id page) frame-id :y [0 100]) + result-zero-x-1 (snap/query data (:id page) uuid/zero :x [0 100]) + result-zero-y-1 (snap/query data (:id page) uuid/zero :y [0 100]) + result-frame-x-1 (snap/query data (:id page) frame-id :x [0 100]) + result-frame-y-1 (snap/query data (:id page) frame-id :y [0 100]) - result-zero-x-2 (sd/query data (:id page) uuid/zero :x [0 200]) - result-zero-y-2 (sd/query data (:id page) uuid/zero :y [0 200]) - result-frame-x-2 (sd/query data (:id page) frame-id :x [0 200]) - result-frame-y-2 (sd/query data (:id page) frame-id :y [0 200])] + result-zero-x-2 (snap/query data (:id page) uuid/zero :x [0 200]) + result-zero-y-2 (snap/query data (:id page) uuid/zero :y [0 200]) + result-frame-x-2 (snap/query data (:id page) frame-id :x [0 200]) + result-frame-y-2 (snap/query data (:id page) frame-id :y [0 200])] (t/is (some? data)) From c70e7f38760f8594e384dd48879995794cf01528 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 25 Sep 2025 09:57:56 +0200 Subject: [PATCH 04/15] :sparkles: Add logging to frontend repo namespace --- frontend/src/app/main/repo.cljs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/main/repo.cljs b/frontend/src/app/main/repo.cljs index d863465b42..7294f823cf 100644 --- a/frontend/src/app/main/repo.cljs +++ b/frontend/src/app/main/repo.cljs @@ -8,6 +8,8 @@ (:require [app.common.data :as d] [app.common.exceptions :as ex] + [app.common.logging :as log] + [app.common.time :as ct] [app.common.transit :as t] [app.common.uri :as u] [app.config :as cf] @@ -17,6 +19,8 @@ [beicon.v2.core :as rx] [cuerdas.core :as str])) +(log/set-level! :info) + (defn handle-response [{:keys [status body headers uri] :as response}] (cond @@ -126,13 +130,21 @@ (select-keys params query-params) nil)) :response-type - (if stream? nil response-type)}] + (if stream? nil response-type)} + + tpoint + (ct/tpoint-ms)] + + (log/trc :hint "make request" :id id) (->> (http/fetch request) (rx/map http/response->map) (rx/mapcat (fn [{:keys [headers body] :as response}] + (log/trc :hint "response received" :id id :elapsed (tpoint)) + (let [ctype (get headers "content-type") - response-stream? (str/starts-with? ctype "text/event-stream")] + response-stream? (str/starts-with? ctype "text/event-stream") + tpoint (ct/tpoint-ms)] (when (and response-stream? (not stream?)) (ex/raise :type :internal @@ -148,6 +160,8 @@ (->> response (http/process-response-type response-type) (rx/map decode-fn) + (rx/tap (fn [_] + (log/trc :hint "response decoded" :id id :elapsed (tpoint)))) (rx/mapcat handle-response))))))))) (defmulti cmd! (fn [id _] id)) From 67661674e2fb86e56662dca536fbc3407a0ae787 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 25 Sep 2025 16:36:09 +0200 Subject: [PATCH 05/15] :sparkles: Make deleted fonts fixer to run with more granular stragegy Instead of running it on all the file, only run it to local library and the current page, reducing considerably the overhead of analyzing the whole file on each file load. It stills executes for page each time the page is loaded, and add some kind of local cache for not doing repeated work each time page loads is pending to be implemented in other commit. --- frontend/src/app/main/data/workspace.cljs | 4 +- .../data/workspace/fix_broken_shapes.cljs | 56 ------ .../data/workspace/fix_deleted_fonts.cljs | 164 ++++++++---------- .../src/app/main/data/workspace/pages.cljs | 2 + 4 files changed, 79 insertions(+), 147 deletions(-) delete mode 100644 frontend/src/app/main/data/workspace/fix_broken_shapes.cljs diff --git a/frontend/src/app/main/data/workspace.cljs b/frontend/src/app/main/data/workspace.cljs index 88cefbbc0b..048908202f 100644 --- a/frontend/src/app/main/data/workspace.cljs +++ b/frontend/src/app/main/data/workspace.cljs @@ -42,7 +42,6 @@ [app.main.data.workspace.common :as dwc] [app.main.data.workspace.drawing :as dwd] [app.main.data.workspace.edition :as dwe] - [app.main.data.workspace.fix-broken-shapes :as fbs] [app.main.data.workspace.fix-deleted-fonts :as fdf] [app.main.data.workspace.groups :as dwg] [app.main.data.workspace.guides :as dwgu] @@ -232,8 +231,7 @@ ptk/WatchEvent (watch [_ _ _] (rx/of (dp/check-open-plugin) - (fdf/fix-deleted-fonts) - (fbs/fix-broken-shapes))))) + (fdf/fix-deleted-fonts-for-local-library file-id))))) (defn- bundle-fetched [{:keys [file file-id thumbnails] :as bundle}] diff --git a/frontend/src/app/main/data/workspace/fix_broken_shapes.cljs b/frontend/src/app/main/data/workspace/fix_broken_shapes.cljs deleted file mode 100644 index 78f88dc823..0000000000 --- a/frontend/src/app/main/data/workspace/fix_broken_shapes.cljs +++ /dev/null @@ -1,56 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS INC - -(ns app.main.data.workspace.fix-broken-shapes - (:require - [app.main.data.changes :as dch] - [app.main.data.helpers :as dsh] - [beicon.v2.core :as rx] - [potok.v2.core :as ptk])) - -(defn- generate-broken-link-changes - [attr {:keys [objects id] :as container}] - (let [base {:type :fix-obj :fix :broken-children attr id} - contains? (partial contains? objects) - xform (comp - ;; FIXME: Ensure all obj have id field (this is needed - ;; because some bug adds an ephimeral shape with id ZERO, - ;; with a single attr `:shapes` having a vector of ids - ;; pointing to not existing shapes). That happens on - ;; components. THIS IS A WORKAOURD - (map (fn [[id obj]] - (if (some? (:id obj)) - obj - (assoc obj :id id)))) - - ;; Remove all valid shapes - (remove (fn [obj] - (every? contains? (:shapes obj)))) - - (map (fn [obj] - (assoc base :id (:id obj)))))] - - (sequence xform objects))) - -(defn fix-broken-shapes - [] - (ptk/reify ::fix-broken-shapes - ptk/WatchEvent - (watch [it state _] - (let [fdata (dsh/lookup-file-data state) - changes (concat - (mapcat (partial generate-broken-link-changes :page-id) - (vals (:pages-index fdata))) - (mapcat (partial generate-broken-link-changes :component-id) - (vals (:components fdata))))] - - (if (seq changes) - (rx/of (dch/commit-changes - {:origin it - :redo-changes (vec changes) - :undo-changes [] - :save-undo? false})) - (rx/empty)))))) diff --git a/frontend/src/app/main/data/workspace/fix_deleted_fonts.cljs b/frontend/src/app/main/data/workspace/fix_deleted_fonts.cljs index 1ffe8a1ba4..fb33a74dc7 100644 --- a/frontend/src/app/main/data/workspace/fix_deleted_fonts.cljs +++ b/frontend/src/app/main/data/workspace/fix_deleted_fonts.cljs @@ -14,8 +14,9 @@ [beicon.v2.core :as rx] [potok.v2.core :as ptk])) -;; This event will update the file so the texts with non existing custom fonts try to be fixed. -;; This can happen when: +;; This event will update the file so the texts with non existing +;; custom fonts try to be fixed. This can happen when: +;; ;; - Exporting/importing files to different teams or penpot instances ;; - Moving files from one team to another in the same instance ;; - Custom fonts are explicitly deleted in the team area @@ -23,112 +24,99 @@ (defn- calculate-alternative-font-id [value] (let [fonts (deref fonts/fontsdb)] - (->> (vals fonts) - (filter #(= (:family %) value)) - (first) - :id))) + (reduce-kv (fn [_ _ font] + (if (= (:family font) value) + (reduced (:id font)) + nil)) + nil + fonts))) (defn- has-invalid-font-family? [node] - (let [fonts (deref fonts/fontsdb) - font-family (:font-family node) - alternative-font-id (calculate-alternative-font-id font-family)] + (let [fonts (deref fonts/fontsdb) + font-family (:font-family node)] (and (some? font-family) - (nil? (get fonts (:font-id node))) - (some? alternative-font-id)))) + (nil? (get fonts (:font-id node)))))) -(defn- should-fix-deleted-font-shape? +(defn- shape-has-invalid-font-family?? [shape] - (let [text-nodes (txt/node-seq txt/is-text-node? (:content shape))] - (and (cfh/text-shape? shape) - (some has-invalid-font-family? text-nodes)))) - -(defn- should-fix-deleted-font-component? - [component] - (let [xf (comp (map val) - (filter should-fix-deleted-font-shape?))] - (first (sequence xf (:objects component))))) + (and (cfh/text-shape? shape) + (some has-invalid-font-family? + (txt/node-seq txt/is-text-node? (:content shape))))) (defn- fix-deleted-font [node] - (let [alternative-font-id (calculate-alternative-font-id (:font-family node))] - (cond-> node - (some? alternative-font-id) (assoc :font-id alternative-font-id)))) + (if-let [alternative-font-id (calculate-alternative-font-id (:font-family node))] + (assoc node :font-id alternative-font-id) + node)) -(defn- fix-deleted-font-shape +(defn- fix-shape-content [shape] - (let [transform (partial txt/transform-nodes has-invalid-font-family? fix-deleted-font)] - (update shape :content transform))) + (txt/transform-nodes has-invalid-font-family? fix-deleted-font + (:content shape))) -(defn- fix-deleted-font-component - [component] - (update component - :objects - (fn [objects] - (update-vals objects fix-deleted-font-shape)))) - -(defn fix-deleted-font-typography +(defn- fix-typography [typography] - (let [alternative-font-id (calculate-alternative-font-id (:font-family typography))] - (cond-> typography - (some? alternative-font-id) (assoc :font-id alternative-font-id)))) + (if-let [alternative-font-id (calculate-alternative-font-id (:font-family typography))] + (assoc typography :font-id alternative-font-id) + typography)) -(defn- generate-deleted-font-shape-changes +(defn- generate-page-changes [{:keys [objects id]}] - (sequence - (comp (map val) - (filter should-fix-deleted-font-shape?) - (map (fn [shape] - {:type :mod-obj - :id (:id shape) - :page-id id - :operations [{:type :set - :attr :content - :val (:content (fix-deleted-font-shape shape))} - {:type :set - :attr :position-data - :val nil}]}))) - objects)) + (reduce-kv (fn [changes shape-id shape] + (if (shape-has-invalid-font-family?? shape) + (conj changes {:type :mod-obj + :id shape-id + :page-id id + :operations [{:type :set + :attr :content + :val (fix-shape-content shape)} + {:type :set + :attr :position-data + :val nil}]}) + changes)) + [] + objects)) -(defn- generate-deleted-font-components-changes +(defn- generate-library-changes [fdata] - (sequence - (comp (map val) - (filter should-fix-deleted-font-component?) - (map (fn [component] - {:type :mod-component - :id (:id component) - :objects (-> (fix-deleted-font-component component) :objects)}))) - (:components fdata))) + (reduce-kv (fn [changes _ typography] + (if (has-invalid-font-family? typography) + (conj changes {:type :mod-typography + :typography (fix-typography typography)}) + changes)) + [] + (:typographies fdata))) -(defn- generate-deleted-font-typography-changes - [fdata] - (sequence - (comp (map val) - (filter has-invalid-font-family?) - (map (fn [typography] - {:type :mod-typography - :typography (fix-deleted-font-typography typography)}))) - (:typographies fdata))) - -(defn fix-deleted-fonts - [] - (ptk/reify ::fix-deleted-fonts +(defn fix-deleted-fonts-for-local-library + "Looks the file local library for deleted fonts and emit changes if + invalid but fixable typographyes found." + [file-id] + (ptk/reify ::fix-deleted-fonts-for-local-library ptk/WatchEvent (watch [it state _] - (let [fdata (dsh/lookup-file-data state) - pages (:pages-index fdata) - - shape-changes (mapcat generate-deleted-font-shape-changes (vals pages)) - components-changes (generate-deleted-font-components-changes fdata) - typography-changes (generate-deleted-font-typography-changes fdata) - changes (concat shape-changes - components-changes - typography-changes)] - (if (seq changes) + (let [fdata (dsh/lookup-file-data state file-id)] + (when-let [changes (-> (generate-library-changes fdata) + (not-empty))] (rx/of (dwc/commit-changes {:origin it - :redo-changes (vec changes) + :redo-changes changes :undo-changes [] - :save-undo? false})) - (rx/empty)))))) + :save-undo? false}))))))) + +;; FIXME: would be nice to not execute this code twice per page in the +;; same working session, maybe some local memoization can improve that + +(defn fix-deleted-fonts-for-page + [file-id page-id] + (ptk/reify ::fix-deleted-fonts-for-page + ptk/WatchEvent + (watch [it state _] + (let [page (dsh/lookup-page state file-id page-id)] + (when-let [changes (-> (generate-page-changes page) + (not-empty))] + (rx/of (dwc/commit-changes + {:origin it + :redo-changes changes + :undo-changes [] + :save-undo? false}))))))) diff --git a/frontend/src/app/main/data/workspace/pages.cljs b/frontend/src/app/main/data/workspace/pages.cljs index a9c9d3354e..5b91d10864 100644 --- a/frontend/src/app/main/data/workspace/pages.cljs +++ b/frontend/src/app/main/data/workspace/pages.cljs @@ -24,6 +24,7 @@ [app.main.data.helpers :as dsh] [app.main.data.persistence :as-alias dps] [app.main.data.workspace.drawing :as dwd] + [app.main.data.workspace.fix-deleted-fonts :as fdf] [app.main.data.workspace.layout :as layout] [app.main.data.workspace.libraries :as dwl] [app.main.data.workspace.thumbnails :as dwth] @@ -104,6 +105,7 @@ (if (dsh/lookup-page state file-id page-id) (rx/concat (rx/of (initialize-page* file-id page-id) + (fdf/fix-deleted-fonts-for-page file-id page-id) (dwth/watch-state-changes file-id page-id) (dwl/watch-component-changes)) (let [profile (:profile state) From c3eabbdb25401b55c5f9c94bb4d9a032ced1b1d4 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 6 Oct 2025 09:20:14 +0200 Subject: [PATCH 06/15] :paperclip: Enable the fdata/objects-map feature by default Using config flags --- common/src/app/common/flags.cljc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/src/app/common/flags.cljc b/common/src/app/common/flags.cljc index 7cc41c19f6..c8ce46085b 100644 --- a/common/src/app/common/flags.cljc +++ b/common/src/app/common/flags.cljc @@ -158,7 +158,8 @@ :enable-component-thumbnails :enable-render-wasm-dpr :enable-token-units - :enable-token-typography-types]) + :enable-token-typography-types + :enable-feature-fdata-objects-map]) (defn parse [& flags] From 8c91109c63713839bc313951c8cf4ac05106e224 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 6 Oct 2025 09:26:42 +0200 Subject: [PATCH 07/15] :books: Update changelog --- CHANGES.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 844714bfa3..5590657dfd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,13 @@ # CHANGELOG -## 2.10.0 (Unreleased) +## 2.10.1 (Unreleased) + +### :sparkles: New features & Enhancements + +- Improve workpace file loading [Github 7366](https://github.com/penpot/penpot/pull/7366) + + +## 2.10.0 ### :rocket: Epics and highlights From e38dd213079c4bf017ad15ac045a0a68322a0453 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 6 Oct 2025 10:56:04 +0200 Subject: [PATCH 08/15] :arrow_up: Update exported dependencies --- exporter/package.json | 2 +- exporter/yarn.lock | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/exporter/package.json b/exporter/package.json index ab394fe62d..3b9981ea6e 100644 --- a/exporter/package.json +++ b/exporter/package.json @@ -16,7 +16,7 @@ "inflation": "^2.1.0", "ioredis": "^5.6.1", "luxon": "^3.6.1", - "playwright": "^1.53.0", + "playwright": "^1.55.1", "raw-body": "^3.0.0", "svgo": "penpot/svgo#v3.1", "xml-js": "^1.6.11", diff --git a/exporter/yarn.lock b/exporter/yarn.lock index ab798ac2f5..8c11bacc43 100644 --- a/exporter/yarn.lock +++ b/exporter/yarn.lock @@ -557,7 +557,7 @@ __metadata: inflation: "npm:^2.1.0" ioredis: "npm:^5.6.1" luxon: "npm:^3.6.1" - playwright: "npm:^1.53.0" + playwright: "npm:^1.55.1" raw-body: "npm:^3.0.0" source-map-support: "npm:^0.5.21" svgo: "penpot/svgo#v3.1" @@ -1101,27 +1101,27 @@ __metadata: languageName: node linkType: hard -"playwright-core@npm:1.53.0": - version: 1.53.0 - resolution: "playwright-core@npm:1.53.0" +"playwright-core@npm:1.55.1": + version: 1.55.1 + resolution: "playwright-core@npm:1.55.1" bin: playwright-core: cli.js - checksum: 10c0/fda0cf76115b15b1ca5cbc69e14185904e5c85e9e7cddb0a48121e69d681c638ac497e8a103985976cae260aa02e9c03ea27d6cd0b5f3d3ca914d4c7fd96f930 + checksum: 10c0/39837a8c1232ec27486eac8c3fcacc0b090acc64310f7f9004b06715370fc426f944e3610fe8c29f17cd3d68280ed72c75f660c02aa5b5cf0eb34bab0031308f languageName: node linkType: hard -"playwright@npm:^1.53.0": - version: 1.53.0 - resolution: "playwright@npm:1.53.0" +"playwright@npm:^1.55.1": + version: 1.55.1 + resolution: "playwright@npm:1.55.1" dependencies: fsevents: "npm:2.3.2" - playwright-core: "npm:1.53.0" + playwright-core: "npm:1.55.1" dependenciesMeta: fsevents: optional: true bin: playwright: cli.js - checksum: 10c0/8d995114808b92f2005bd12ff5e494cdc3fa2d484f4d85a3e54be1fb99e88ae3e34b24792d83bb987462c73e553a0fa37a2a70264afbf67894b51c1498cf5a11 + checksum: 10c0/b84a97b0d764403df512f5bbb10c7343974e151a28202cc06f90883a13e8a45f4491a0597f0ae5fb03a026746cbc0d200f0f32195bfaa381aee5ca5770626771 languageName: node linkType: hard From d5b743c604a4d782a0585baeb5e74893ce8833a6 Mon Sep 17 00:00:00 2001 From: Alonso Torres Date: Mon, 6 Oct 2025 11:19:33 +0200 Subject: [PATCH 09/15] :bug: Fix problem with text in plugins (#7446) --- CHANGES.md | 4 ++++ frontend/src/app/plugins/api.cljs | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 5590657dfd..dcb57cb0d4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,10 @@ - Improve workpace file loading [Github 7366](https://github.com/penpot/penpot/pull/7366) +### :bug: Bugs fixed + +- Fix regression with text shapes creation with Plugins API [Taiga #12244](https://tree.taiga.io/project/penpot/issue/12244) + ## 2.10.0 diff --git a/frontend/src/app/plugins/api.cljs b/frontend/src/app/plugins/api.cljs index cc65f10883..0b8350b0e2 100644 --- a/frontend/src/app/plugins/api.cljs +++ b/frontend/src/app/plugins/api.cljs @@ -338,8 +338,9 @@ :else (let [page (dsh/lookup-page @st/state) shape (-> (cts/setup-shape {:type :text :x 0 :y 0 :grow-type :auto-width}) - (txt/change-text text) + (update :content txt/change-text text) (assoc :position-data nil)) + changes (-> (cb/empty-changes) (cb/with-page page) From 9244501c6e36cf00e9838d4bf34317aa272d8bfa Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 6 Oct 2025 09:56:44 +0200 Subject: [PATCH 10/15] :sparkles: Add variants/v1 feature to default features emited by sdk --- common/src/app/common/files/builder.cljc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/src/app/common/files/builder.cljc b/common/src/app/common/files/builder.cljc index 181824c672..3a3f734d61 100644 --- a/common/src/app/common/files/builder.cljc +++ b/common/src/app/common/files/builder.cljc @@ -200,7 +200,8 @@ "layout/grid" "components/v2" "plugins/runtime" - "design-tokens/v1"}) + "design-tokens/v1" + "variants/v1"}) ;; WORKAROUND: the same as features (def available-migrations From b30cb0e0848de8266d95cd5c914a19dfcd59a3ef Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 6 Oct 2025 10:28:00 +0200 Subject: [PATCH 11/15] :sparkles: Allow pass variant related attrs on add-component change --- common/src/app/common/files/changes.cljc | 8 ++++++-- common/src/app/common/types/components_list.cljc | 6 ++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/common/src/app/common/files/changes.cljc b/common/src/app/common/files/changes.cljc index 73f48f5661..62810a0e02 100644 --- a/common/src/app/common/files/changes.cljc +++ b/common/src/app/common/files/changes.cljc @@ -320,14 +320,18 @@ [:shapes {:optional true} [:vector {:gen/max 3} ::sm/any]] [:path {:optional true} :string] [:main-instance-id ::sm/uuid] - [:main-instance-page ::sm/uuid]]] + [:main-instance-page ::sm/uuid] + ;; Only used by external processes (like Penpot SDK) + [:variant-id {:optional true} ::sm/uuid] + [:variant-properties {:optional true} [:vector ctv/schema:variant-property]]]] [:mod-component [:map {:title "ModCompoenentChange"} [:type [:= :mod-component]] [:id ::sm/uuid] - [:shapes {:optional true} [:vector {:gen/max 3} ::sm/any]] [:name {:optional true} :string] + [:path {:optional true} :string] + [:shapes {:optional true} [:vector {:gen/max 3} ::sm/any]] [:variant-id {:optional true} ::sm/uuid] [:variant-properties {:optional true} [:vector ctv/schema:variant-property]]]] diff --git a/common/src/app/common/types/components_list.cljc b/common/src/app/common/types/components_list.cljc index fd75341681..68d8e72ef0 100644 --- a/common/src/app/common/types/components_list.cljc +++ b/common/src/app/common/types/components_list.cljc @@ -35,7 +35,7 @@ (defn add-component [fdata {:keys [id name path main-instance-id main-instance-page annotation variant-id variant-properties]}] - (let [fdata (update fdata :components assoc id (touch {:id id :name name :path path}))] + (let [fdata (update fdata :components assoc id (touch {:id id :name name :path path}))] (cond-> (update-in fdata [:components id] assoc :main-instance-id main-instance-id :main-instance-page main-instance-page) annotation (update-in [:components id] assoc :annotation annotation) variant-id (update-in [:components id] assoc :variant-id variant-id) @@ -83,9 +83,11 @@ (nil? variant-properties) (dissoc :variant-properties)) + + ;; The set of properties that doesn't mark a component as touched diff (set/difference (ctk/diff-components component new-comp) - #{:annotation :modified-at :variant-id :variant-properties})] ;; The set of properties that doesn't mark a component as touched + #{:annotation :modified-at :variant-id :variant-properties})] (if (empty? diff) new-comp From 588eb0b4fa7e38c57e68db85f77d6130816b6dea Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 6 Oct 2025 10:28:24 +0200 Subject: [PATCH 12/15] :arrow_up: Update shadow-cljs dependency on sdk --- library/deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/deps.edn b/library/deps.edn index 0b2473bf81..9edca59650 100644 --- a/library/deps.edn +++ b/library/deps.edn @@ -21,7 +21,7 @@ :dev {:extra-paths ["dev"] :extra-deps - {thheller/shadow-cljs {:mvn/version "3.1.7"} + {thheller/shadow-cljs {:mvn/version "3.2.1"} com.bhauman/rebel-readline {:mvn/version "RELEASE"} org.clojure/tools.namespace {:mvn/version "RELEASE"} criterium/criterium {:mvn/version "RELEASE"}}} From 6b0d0a302f57889036c543d08ea31403978ba811 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 6 Oct 2025 11:52:08 +0200 Subject: [PATCH 13/15] :sparkles: Enable variant attrs on SDK addComponent method --- common/src/app/common/files/builder.cljc | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/common/src/app/common/files/builder.cljc b/common/src/app/common/files/builder.cljc index 3a3f734d61..6dbaf47efd 100644 --- a/common/src/app/common/files/builder.cljc +++ b/common/src/app/common/files/builder.cljc @@ -8,11 +8,11 @@ "Internal implementation of file builder. Mainly used as base impl for penpot library" (:require + ;; [app.common.features :as cfeat] [app.common.data :as d] [app.common.data.macros :as dm] [app.common.exceptions :as ex] [app.common.files.changes :as ch] - ;; [app.common.features :as cfeat] [app.common.files.helpers :as cph] [app.common.files.migrations :as fmig] [app.common.geom.shapes :as gsh] @@ -26,6 +26,7 @@ [app.common.types.path :as types.path] [app.common.types.shape :as types.shape] [app.common.types.typography :as types.typography] + [app.common.types.variant :as types.variant] [app.common.uuid :as uuid] [cuerdas.core :as str])) @@ -126,10 +127,12 @@ [:map [:component-id ::sm/uuid] [:file-id {:optional true} ::sm/uuid] + [:page-id {:optional true} ::sm/uuid] + [:frame-id {:optional true} ::sm/uuid] [:name {:optional true} ::sm/text] [:path {:optional true} ::sm/text] - [:frame-id {:optional true} ::sm/uuid] - [:page-id {:optional true} ::sm/uuid]]) + [:variant-id {:optional true} ::sm/uuid] + [:variant-properties {:optional true} [:vector types.variant/schema:variant-property]]]) (def ^:private check-add-component (sm/check-fn schema:add-component @@ -444,7 +447,7 @@ (defn add-component [state params] - (let [{:keys [component-id file-id page-id frame-id name path]} + (let [{:keys [component-id file-id page-id frame-id name path variant-id variant-properties]} (-> (check-add-component params) (update :component-id default-uuid)) @@ -464,7 +467,9 @@ :name (or name "anonmous") :path path :main-instance-id frame-id - :main-instance-page page-id}) + :main-instance-page page-id + :variant-id variant-id + :variant-properties variant-properties}) change2 {:type :mod-obj From 987dea80484a1e764be9aeb3c0e9c518ecefcc8d Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 6 Oct 2025 11:53:26 +0200 Subject: [PATCH 14/15] :books: Update changelog --- library/CHANGES.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/library/CHANGES.md b/library/CHANGES.md index 22cd12ea2e..71fa68ffb8 100644 --- a/library/CHANGES.md +++ b/library/CHANGES.md @@ -1,5 +1,11 @@ # CHANGELOG +## 1.0.10 + +- Enable variant/v1 feature by default +- Add variant attrs handling to addComponent method + + ## 1.0.9 - Fix dependencies declaration on package.json From 613acd5b29b9221b5156e8383d34e4ff60934515 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 6 Oct 2025 11:56:23 +0200 Subject: [PATCH 15/15] :paperclip: Update version on library/package.json file --- library/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/package.json b/library/package.json index 13851f6d6d..c2d967f1b4 100644 --- a/library/package.json +++ b/library/package.json @@ -1,6 +1,6 @@ { "name": "@penpot/library", - "version": "1.0.9", + "version": "1.0.10", "license": "MPL-2.0", "author": "Kaleidos INC", "packageManager": "yarn@4.9.1+sha512.f95ce356460e05be48d66401c1ae64ef84d163dd689964962c6888a9810865e39097a5e9de748876c2e0bf89b232d583c33982773e9903ae7a76257270986538",