From eea65b12dd6e41d093889e3d608bef7610cf9966 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 25 Sep 2025 09:51:57 +0200 Subject: [PATCH 01/12] :sparkles: Add minor improvement to cljs impl logging Mainly reduce the emmited code, that will contribute to reduce the bundle size and also adds timestamp to the default output. --- common/src/app/common/logging.cljc | 58 +++++++++++++++++------------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/common/src/app/common/logging.cljc b/common/src/app/common/logging.cljc index 7a4df8ebef..e678152b12 100644 --- a/common/src/app/common/logging.cljc +++ b/common/src/app/common/logging.cljc @@ -49,6 +49,7 @@ [app.common.exceptions :as ex] [app.common.pprint :as pp] [app.common.schema :as sm] + [app.common.time :as ct] [app.common.uuid :as uuid] [cuerdas.core :as str] [promesa.exec :as px] @@ -221,36 +222,42 @@ #?(:clj (inst-ms (java.time.Instant/now)) :cljs (js/Date.now))) +(defn emit-log + [props cause context logger level sync?] + (let [props (cond-> props sync? deref) + ts (current-timestamp) + gcontext *context* + logfn (fn [] + (let [props (if sync? props (deref props)) + props (into (d/ordered-map) props) + context (if (and (empty? gcontext) + (empty? context)) + {} + (d/without-nils (merge gcontext context))) + + lrecord {::id (uuid/next) + ::timestamp ts + ::message (delay (build-message props)) + ::props props + ::context context + ::level level + ::logger logger} + lrecord (cond-> lrecord + (some? cause) + (assoc ::cause cause + ::trace (delay (build-stack-trace cause))))] + (swap! log-record (constantly lrecord))))] + (if sync? + (logfn) + (px/exec! *default-executor* logfn)))) + (defmacro log! "Emit a new log record to the global log-record state (asynchronously). " [& props] (let [{:keys [::level ::logger ::context ::sync? cause] :or {sync? false}} props props (into [] msg-props-xf props)] `(when (enabled? ~logger ~level) - (let [props# (cond-> (delay ~props) ~sync? deref) - ts# (current-timestamp) - context# *context* - logfn# (fn [] - (let [props# (if ~sync? props# (deref props#)) - props# (into (d/ordered-map) props#) - cause# ~cause - context# (d/without-nils - (merge context# ~context)) - lrecord# {::id (uuid/next) - ::timestamp ts# - ::message (delay (build-message props#)) - ::props props# - ::context context# - ::level ~level - ::logger ~logger} - lrecord# (cond-> lrecord# - (some? cause#) - (assoc ::cause cause# - ::trace (delay (build-stack-trace cause#))))] - (swap! log-record (constantly lrecord#))))] - (if ~sync? - (logfn#) - (px/exec! *default-executor* logfn#)))))) + (emit-log (delay ~props) ~cause ~context ~logger ~level ~sync?)))) #?(:clj (defn slf4j-log-handler @@ -276,7 +283,8 @@ (when (enabled? logger level) (let [hstyles (str/ffmt "font-weight: 600; color: %" (level->color level)) mstyles (str/ffmt "font-weight: 300; color: %" (level->color level)) - header (str/concat "%c" (level->name level) " [" logger "] ") + ts (ct/format-inst (ct/now) "kk:mm:ss.SSSS") + header (str/concat "%c" (level->name level) " " ts " [" logger "] ") message (str/concat header "%c" @message)] (js/console.group message hstyles mstyles) From 255f5af2e38bb7bf6c56bfae0ba9d16b23027cc0 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 24 Sep 2025 09:54:26 +0200 Subject: [PATCH 02/12] :sparkles: Add several enhacements to buffer namespace The changes are just for completenes. --- common/src/app/common/buffer.cljc | 82 ++++++++++++++++++++++++++++--- 1 file changed, 74 insertions(+), 8 deletions(-) diff --git a/common/src/app/common/buffer.cljc b/common/src/app/common/buffer.cljc index c04bd46239..07cf5e6853 100644 --- a/common/src/app/common/buffer.cljc +++ b/common/src/app/common/buffer.cljc @@ -50,6 +50,13 @@ (let [target (with-meta target {:tag 'java.nio.ByteBuffer})] `(long (.getInt ~target (unchecked-int ~offset)))))) +(defmacro read-long + [target offset] + (if (:ns &env) + `(.getInt64 ~target ~offset true) + (let [target (with-meta target {:tag 'java.nio.ByteBuffer})] + `(.getLong ~target (unchecked-int ~offset))))) + (defmacro read-float [target offset] (if (:ns &env) @@ -75,6 +82,40 @@ (finally (.order ~target ByteOrder/LITTLE_ENDIAN)))))) +(defmacro read-bytes + "Get a byte array from buffer. It is potentially unsafe because on + JS/CLJS it returns a subarray without doing any copy of data." + [target offset size] + (if (:ns &env) + `(new js/Uint8Array + (.-buffer ~target) + (+ (.-byteOffset ~target) ~offset) + ~size) + (let [target (with-meta target {:tag 'java.nio.ByteBuffer}) + bbuf (with-meta (gensym "bbuf") {:tag bytes})] + `(let [~bbuf (byte-array ~size)] + (.get ~target + (unchecked-int ~offset) + ~bbuf + 0 + ~size) + ~bbuf)))) + +;; FIXME: implement in cljs +(defmacro write-bytes + ([target offset src size] + `(write-bytes ~target ~offset ~src 0 ~size)) + ([target offset src src-offset size] + (if (:ns &env) + (throw (ex-info "not implemented" {})) + (let [target (with-meta target {:tag 'java.nio.ByteBuffer}) + src (with-meta src {:tag 'bytes})] + `(.put ~target + (unchecked-int ~offset) + ~src + (unchecked-int ~src-offset) + (unchecked-int ~size)))))) + (defmacro write-byte [target offset value] (if (:ns &env) @@ -144,13 +185,15 @@ (.setUint32 ~target (+ ~offset 12) (aget barray# 3) true)) (let [target (with-meta target {:tag 'java.nio.ByteBuffer}) - value (with-meta value {:tag 'java.util.UUID})] - `(try - (.order ~target ByteOrder/BIG_ENDIAN) - (.putLong ~target (unchecked-int (+ ~offset 0)) (.getMostSignificantBits ~value)) - (.putLong ~target (unchecked-int (+ ~offset 8)) (.getLeastSignificantBits ~value)) - (finally - (.order ~target ByteOrder/LITTLE_ENDIAN)))))) + value (with-meta value {:tag 'java.util.UUID}) + prev (with-meta (gensym "prev-") {:tag 'java.nio.ByteOrder})] + `(let [~prev (.order ~target)] + (try + (.order ~target ByteOrder/BIG_ENDIAN) + (.putLong ~target (unchecked-int (+ ~offset 0)) (.getMostSignificantBits ~value)) + (.putLong ~target (unchecked-int (+ ~offset 8)) (.getLeastSignificantBits ~value)) + (finally + (.order ~target ~prev))))))) (defn wrap [data] @@ -160,7 +203,7 @@ (defn allocate [size] - #?(:clj (let [buffer (ByteBuffer/allocate (int size))] + #?(:clj (let [buffer (ByteBuffer/allocate (unchecked-int size))] (.order buffer ByteOrder/LITTLE_ENDIAN)) :cljs (new js/DataView (new js/ArrayBuffer size)))) @@ -181,6 +224,14 @@ (.set dst-view src-view) (js/DataView. dst-buff)))) +;; FIXME: cljs impl +#?(:clj + (defn copy-bytes + [src src-offset size dst dst-offset] + (let [tmp (byte-array size)] + (.get ^ByteBuffer src src-offset tmp 0 size) + (.put ^ByteBuffer dst dst-offset tmp 0 size)))) + (defn equals? [buffer-a buffer-b] #?(:clj @@ -208,3 +259,18 @@ [o] #?(:clj (instance? ByteBuffer o) :cljs (instance? js/DataView o))) + +(defn slice + [buffer offset size] + #?(:cljs + (let [offset (+ (.-byteOffset buffer) offset)] + (new js/DataView (.-buffer buffer) offset size)) + + :clj + (-> (.slice ^ByteBuffer buffer (unchecked-int offset) (unchecked-int size)) + (.order ByteOrder/LITTLE_ENDIAN)))) + +(defn size + [o] + #?(:cljs (.-byteLength ^js o) + :clj (.capacity ^ByteBuffer o))) From 1d54fe2e248b9f489b6c6ab4aa6697ae770da266 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 25 Sep 2025 09:55:34 +0200 Subject: [PATCH 03/12] :sparkles: Add support for emit messages without waiting response on worker --- frontend/src/app/main/worker.cljs | 11 ++++++++++- frontend/src/app/util/worker.cljs | 25 +++++++++++++++++++------ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/frontend/src/app/main/worker.cljs b/frontend/src/app/main/worker.cljs index a310f566e8..aae9000556 100644 --- a/frontend/src/app/main/worker.cljs +++ b/frontend/src/app/main/worker.cljs @@ -21,7 +21,6 @@ :config {:public-uri cf/public-uri :build-data cf/build-date :version cf/version}}) - (set! instance worker))) (defn ask! @@ -34,6 +33,16 @@ (uw/ask! instance message transfer) (rx/empty)))) +(defn emit! + ([message] + (if instance + (uw/emit! instance message) + (rx/empty))) + ([message transfer] + (if instance + (uw/emit! instance message transfer) + (rx/empty)))) + (defn ask-buffered! ([message] (if instance diff --git a/frontend/src/app/util/worker.cljs b/frontend/src/app/util/worker.cljs index f1b27cc7c6..bf1b44c30b 100644 --- a/frontend/src/app/util/worker.cljs +++ b/frontend/src/app/util/worker.cljs @@ -19,7 +19,7 @@ ([worker message] (send-message! worker message nil)) - ([worker {sender-id :sender-id :as message} {:keys [many?] :or {many? false}}] + ([worker {sender-id :sender-id :as message} {:keys [many? ignore-response?] :or {many? false ignore-response? false}}] (let [take-messages (fn [ob] (if many? @@ -34,11 +34,13 @@ (if (some? instance) (do (.postMessage instance data transfer) - (->> (:stream worker) - (rx/filter #(= (:reply-to %) sender-id)) - (take-messages) - (rx/filter (complement :dropped)) - (rx/map handle-response))) + (if (not ignore-response?) + (->> (:stream worker) + (rx/filter #(= (:reply-to %) sender-id)) + (take-messages) + (rx/filter (complement :dropped)) + (rx/map handle-response)) + (rx/empty))) (rx/empty))))) (defn ask! @@ -51,6 +53,17 @@ :payload message :transfer transfer}))) +(defn emit! + ([worker message] + (emit! worker message nil)) + ([worker message transfer] + (send-message! + worker + {:sender-id (uuid/next) + :payload message + :transfer transfer} + {:ignore-response? true}))) + (defn ask-many! ([worker message] (ask-many! worker message nil)) From 0346c48b0304b318e078136c258f2fb626d30ee9 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 25 Sep 2025 10:49:30 +0200 Subject: [PATCH 04/12] :sparkles: Add several minor enhacements to features subsystem Mainly fixes the team non-inheritable features handling and removes unnecesary/duplicate checks. --- backend/src/app/rpc/commands/files_create.clj | 9 +++++---- backend/src/app/rpc/commands/files_update.clj | 9 ++++----- backend/src/app/rpc/commands/teams.clj | 2 +- common/src/app/common/features.cljc | 17 ++++++++--------- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/backend/src/app/rpc/commands/files_create.clj b/backend/src/app/rpc/commands/files_create.clj index bcae300d30..8f4b9b428e 100644 --- a/backend/src/app/rpc/commands/files_create.clj +++ b/backend/src/app/rpc/commands/files_create.clj @@ -112,14 +112,15 @@ ;; FIXME: IMPORTANT: this code can have race conditions, because ;; we have no locks for updating team so, creating two files ;; concurrently can lead to lost team features updating - (when-let [features (-> features (set/difference (:features team)) (set/difference cfeat/no-team-inheritable-features) (not-empty))] - (let [features (->> features - (set/union (:features team)) - (db/create-array conn "text"))] + (let [features (-> features + (set/union (:features team)) + (set/difference cfeat/no-team-inheritable-features) + (into-array))] + (db/update! conn :team {:features features} {:id (:id team)} diff --git a/backend/src/app/rpc/commands/files_update.clj b/backend/src/app/rpc/commands/files_update.clj index f21860cbea..6bc82afeaa 100644 --- a/backend/src/app/rpc/commands/files_update.clj +++ b/backend/src/app/rpc/commands/files_update.clj @@ -158,7 +158,6 @@ tpoint (ct/tpoint)] - (when (not= (:vern params) (:vern file)) (ex/raise :type :validation @@ -181,15 +180,15 @@ (set/difference (:features team)) (set/difference cfeat/no-team-inheritable-features) (not-empty))] - (let [features (->> features - (set/union (:features team)) - (db/create-array conn "text"))] + (let [features (-> features + (set/union (:features team)) + (set/difference cfeat/no-team-inheritable-features) + (into-array))] (db/update! conn :team {:features features} {:id (:id team)} {::db/return-keys false}))) - (mtx/run! metrics {:id :update-file-changes :inc (count changes)}) (binding [l/*context* (some-> (meta params) diff --git a/backend/src/app/rpc/commands/teams.clj b/backend/src/app/rpc/commands/teams.clj index 1c65e8e516..18f1f1b5f5 100644 --- a/backend/src/app/rpc/commands/teams.clj +++ b/backend/src/app/rpc/commands/teams.clj @@ -503,7 +503,7 @@ (let [features (-> (cfeat/get-enabled-features cf/flags) (set/difference cfeat/frontend-only-features) - (cfeat/check-client-features! (:features params))) + (set/difference cfeat/no-team-inheritable-features)) params (-> params (assoc :profile-id profile-id) (assoc :features features)) diff --git a/common/src/app/common/features.cljc b/common/src/app/common/features.cljc index d08b73a97f..f9b5dd1a16 100644 --- a/common/src/app/common/features.cljc +++ b/common/src/app/common/features.cljc @@ -68,11 +68,6 @@ "design-tokens/v1" "variants/v1"}) -;; A set of features that should not be propagated to team on creating -;; or modifying a file -(def no-team-inheritable-features - #{"fdata/path-data"}) - ;; A set of features which only affects on frontend and can be enabled ;; and disabled freely by the user any time. This features does not ;; persist on file features field but can be permanently enabled on @@ -87,8 +82,14 @@ ;; Features that are mainly backend only or there are a proper ;; fallback when frontend reports no support for it (def backend-only-features - #{"fdata/objects-map" - "fdata/pointer-map"}) + #{"fdata/pointer-map" + "fdata/objects-map"}) + +;; A set of features that should not be propagated to team on creating +;; or modifying a file or creating or modifying a team +(def no-team-inheritable-features + #{"fdata/path-data" + "fdata/shape-data-type"}) ;; This is a set of features that does not require an explicit ;; migration like components/v2 or the migration is not mandatory to @@ -226,8 +227,6 @@ :hint (str/ffmt "enabled feature '%' not present in file (missing migration)" not-supported))) - (check-supported-features! file-features) - ;; Components v1 is deprecated (when-not (contains? file-features "components/v2") (ex/raise :type :restriction From 42d03a0325dff89ec8176ca2bab17da56387e3fc Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 24 Sep 2025 09:58:44 +0200 Subject: [PATCH 05/12] :paperclip: Add several missing imports on repl related namespaces --- backend/dev/user.clj | 1 + backend/src/app/srepl/main.clj | 1 + 2 files changed, 2 insertions(+) diff --git a/backend/dev/user.clj b/backend/dev/user.clj index 04040776ef..f93e0b6f46 100644 --- a/backend/dev/user.clj +++ b/backend/dev/user.clj @@ -35,6 +35,7 @@ [app.util.blob :as blob] [clj-async-profiler.core :as prof] [clojure.contrib.humanize :as hum] + [clojure.datafy :refer [datafy]] [clojure.java.io :as io] [clojure.pprint :refer [pprint print-table]] [clojure.repl :refer :all] diff --git a/backend/src/app/srepl/main.clj b/backend/src/app/srepl/main.clj index 2ffe1caa41..d38757abd1 100644 --- a/backend/src/app/srepl/main.clj +++ b/backend/src/app/srepl/main.clj @@ -40,6 +40,7 @@ [app.util.blob :as blob] [app.util.pointer-map :as pmap] [app.worker :as wrk] + [clojure.datafy :refer [datafy]] [clojure.java.io :as io] [clojure.pprint :refer [print-table]] [clojure.stacktrace :as strace] From 528315b75c5e66cc569b3e617c96317c80f66767 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 24 Sep 2025 09:59:18 +0200 Subject: [PATCH 06/12] :paperclip: Add not-empty generator to schema generator ns --- common/src/app/common/schema/generators.cljc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/src/app/common/schema/generators.cljc b/common/src/app/common/schema/generators.cljc index 72b03f6730..16eab9b687 100644 --- a/common/src/app/common/schema/generators.cljc +++ b/common/src/app/common/schema/generators.cljc @@ -5,7 +5,7 @@ ;; Copyright (c) KALEIDOS INC (ns app.common.schema.generators - (:refer-clojure :exclude [set subseq uuid filter map let boolean vector keyword int double]) + (:refer-clojure :exclude [set subseq uuid filter map let boolean vector keyword int double not-empty]) #?(:cljs (:require-macros [app.common.schema.generators])) (:require [app.common.math :as mth] @@ -146,3 +146,5 @@ (def any (tg/one-of [text boolean double int keyword])) + +(def not-empty tg/not-empty) From fdf70ae9c192b90a8c3bfe6c9295618b1c4dba08 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 24 Sep 2025 10:00:03 +0200 Subject: [PATCH 07/12] :sparkles: Fix docstring on common.weak ns function --- common/src/app/common/weak.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/app/common/weak.cljc b/common/src/app/common/weak.cljc index 87ae6cbc4b..f9efeb73e7 100644 --- a/common/src/app/common/weak.cljc +++ b/common/src/app/common/weak.cljc @@ -22,7 +22,7 @@ #?(:cljs (defn weak-map - "Create a WeakMap like instance what uses clojure equality + "Create a WeakMap-like instance what uses clojure equality semantics." [] (new wm/WeakEqMap #js {:hash hash :equals =}))) From 7fefe6dbc8e4886113c4a68250672640a403a158 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 24 Sep 2025 10:00:54 +0200 Subject: [PATCH 08/12] :tada: Add multiplatform impl of ObjectsMap The new type get influentiated by the ObjectsMap impl on backend code but with simplier implementation that no longer restricts keys to UUID type but preserves the same performance characteristics. This type encodes and decodes correctly both in fressian (backend) and transit (backend and frontend). This is an initial implementation and several memory usage optimizations are still missing. --- common/src/app/common/types/objects_map.cljc | 521 ++++++++++++++++++ common/test/common_tests/runner.cljc | 6 +- .../common_tests/types/objects_map_test.cljc | 133 +++++ 3 files changed, 658 insertions(+), 2 deletions(-) create mode 100644 common/src/app/common/types/objects_map.cljc create mode 100644 common/test/common_tests/types/objects_map_test.cljc diff --git a/common/src/app/common/types/objects_map.cljc b/common/src/app/common/types/objects_map.cljc new file mode 100644 index 0000000000..d08330765c --- /dev/null +++ b/common/src/app/common/types/objects_map.cljc @@ -0,0 +1,521 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns app.common.types.objects-map + "Implements a specialized map-like data structure for store an UUID => + OBJECT mappings. The main purpose of this data structure is be able + to serialize it on fressian as byte-array and have the ability to + decode each field separatelly without the need to decode the whole + map from the byte-array. + + It works transparently, so no aditional dynamic vars are needed. It + only works by reference equality and the hash-code is calculated + properly from each value." + + (:require + #?(:clj [app.common.fressian :as fres]) + #?(:clj [clojure.data.json :as json]) + [app.common.transit :as t] + [clojure.core :as c] + [clojure.core.protocols :as cp]) + #?(:clj + (:import + clojure.lang.Murmur3 + clojure.lang.RT + java.util.Iterator))) + +#?(:clj (set! *warn-on-reflection* true)) + +(declare create) +(declare ^:private do-compact) + +(defprotocol IObjectsMap + (^:no-doc compact [this]) + (^:no-doc get-data [this] "retrieve internal data") + (^:no-doc -hash-for-key [this key] "retrieve a hash for a key")) + +#?(:cljs + (deftype ObjectsMapEntry [key omap] + c/IMapEntry + (-key [_] key) + (-val [_] (get omap key)) + + c/IHash + (-hash [_] + (-hash-for-key omap key)) + + c/IEquiv + (-equiv [this other] + (and (c/map-entry? other) + (= (key this) + (key other)) + (= (val this) + (val other)))) + + c/ISequential + c/ISeqable + (-seq [this] + (cons key (lazy-seq (cons (c/-val this) nil)))) + + c/ICounted + (-count [_] 2) + + c/IIndexed + (-nth [node n] + (cond (== n 0) key + (== n 1) (c/-val node) + :else (throw (js/Error. "Index out of bounds")))) + + (-nth [node n not-found] + (cond (== n 0) key + (== n 1) (c/-val node) + :else not-found)) + + c/ILookup + (-lookup [node k] + (c/-nth node k nil)) + (-lookup [node k not-found] + (c/-nth node k not-found)) + + c/IFn + (-invoke [node k] + (c/-nth node k)) + + (-invoke [node k not-found] + (c/-nth node k not-found)) + + c/IPrintWithWriter + (-pr-writer [this writer opts] + (c/pr-sequential-writer + writer + (fn [item w _] + (c/-write w (pr-str item))) + "[" ", " "]" + opts + this))) + + :clj + (deftype ObjectsMapEntry [key omap] + clojure.lang.IMapEntry + (key [_] key) + (getKey [_] key) + + (val [_] + (get omap key)) + (getValue [_] + (get omap key)) + + clojure.lang.Indexed + (nth [node n] + (cond + (== n 0) key + (== n 1) (val node) + :else (throw (IllegalArgumentException. "Index out of bounds")))) + + (nth [node n not-found] + (cond + (== n 0) key + (== n 1) (val node) + :else not-found)) + + clojure.lang.IPersistentCollection + (empty [_] []) + (count [_] 2) + (seq [this] + (cons key (lazy-seq (cons (val this) nil)))) + (cons [this item] + (.cons ^clojure.lang.IPersistentCollection (vec this) item)) + + clojure.lang.IHashEq + (hasheq [_] + (-hash-for-key omap key)))) + +#?(:cljs + (deftype ObjectMapIterator [iterator omap] + Object + (hasNext [_] + (.hasNext ^js iterator)) + + (next [_] + (let [entry (.next iterator)] + (ObjectsMapEntry. (key entry) omap))) + + (remove [_] + (js/Error. "Unsupported operation"))) + + :clj + (deftype ObjectsMapIterator [^Iterator iterator omap] + Iterator + (hasNext [_] + (.hasNext iterator)) + + (next [_] + (let [entry (.next iterator)] + (ObjectsMapEntry. (key entry) omap))))) + +#?(:cljs + (deftype ObjectsMap [metadata cache + ^:mutable data + ^:mutable modified + ^:mutable hash] + Object + (toString [this] + (pr-str* this)) + (equiv [this other] + (c/-equiv this other)) + (keys [this] + (c/es6-iterator (keys this))) + (entries [this] + (c/es6-entries-iterator (seq this))) + (values [this] + (es6-iterator (vals this))) + (has [this k] + (c/contains? this k)) + (get [this k not-found] + (c/-lookup this k not-found)) + (forEach [this f] + (run! (fn [[k v]] (f v k)) this)) + + cp/Datafiable + (datafy [_] + {:data data + :cache cache + :modified modified + :hash hash}) + + IObjectsMap + (compact [this] + (when modified + (do-compact data cache + (fn [data'] + (set! (.-modified this) false) + (set! (.-data this) data')))) + this) + + (get-data [this] + (compact this) + data) + + (-hash-for-key [this key] + (if (c/-contains-key? cache key) + (c/-hash (c/-lookup cache key)) + (c/-hash (c/-lookup this key)))) + + c/IWithMeta + (-with-meta [this new-meta] + (if (identical? new-meta meta) + this + (ObjectsMap. new-meta + cache + data + modified + hash))) + + c/IMeta + (-meta [_] metadata) + + c/ICloneable + (-clone [this] + (compact this) + (ObjectsMap. metadata {} data false nil)) + + c/IIterable + (-iterator [this] + (c/seq-iter this)) + + c/ICollection + (-conj [this entry] + (cond + (map-entry? entry) + (c/-assoc this (c/-key entry) (c/-val entry)) + + (vector? entry) + (c/-assoc this (c/-nth entry 0) (c/-nth entry 1)) + + :else + (loop [ret this es (seq entry)] + (if (nil? es) + ret + (let [e (first es)] + (if (vector? e) + (recur (c/-assoc ret (c/-nth e 0) (c/-nth e 1)) + (next es)) + (throw (js/Error. "conj on a map takes map entries or seqables of map entries")))))))) + + c/IEmptyableCollection + (-empty [_] + (create)) + + c/IEquiv + (-equiv [this other] + (equiv-map this other)) + + c/IHash + (-hash [this] + (when-not hash + (set! hash (hash-unordered-coll this))) + hash) + + c/ISeqable + (-seq [this] + (->> (keys data) + (map (fn [id] (new ObjectsMapEntry id this))) + (seq))) + + c/ICounted + (-count [_] + (c/-count data)) + + c/ILookup + (-lookup [this k] + (or (c/-lookup cache k) + (if (c/-contains-key? data k) + (let [v (c/-lookup data k) + v (t/decode-str v)] + (set! (.-cache this) (c/-assoc cache k v)) + v) + (do + (set! (.-cache this) (assoc cache key nil)) + nil)))) + + (-lookup [this k not-found] + (if (c/-contains-key? data k) + (c/-lookup this k) + not-found)) + + c/IAssociative + (-assoc [_ k v] + (ObjectsMap. metadata + (c/-assoc cache k v) + (c/-assoc data k nil) + true + nil)) + + (-contains-key? [_ k] + (c/-contains-key? data k)) + + c/IFind + (-find [this k] + (when (c/-contains-key? data k) + (new ObjectsMapEntry k this))) + + c/IMap + (-dissoc [_ k] + (ObjectsMap. metadata + (c/-dissoc cache k) + (c/-dissoc data k) + true + nil)) + + c/IKVReduce + (-kv-reduce [this f init] + (c/-kv-reduce data + (fn [init k _] + (f init k (c/-lookup this k))) + init)) + + c/IFn + (-invoke [this k] + (c/-lookup this k)) + (-invoke [this k not-found] + (c/-lookup this k not-found)) + + c/IPrintWithWriter + (-pr-writer [this writer opts] + (c/pr-sequential-writer + writer + (fn [item w _] + (c/-write w (pr-str (c/-key item))) + (c/-write w \space) + (c/-write w (pr-str (c/-val item)))) + "#penpot/objects-map {" ", " "}" + opts + (seq this)))) + + :clj + (deftype ObjectsMap [metadata cache + ^:unsynchronized-mutable data + ^:unsynchronized-mutable modified + ^:unsynchronized-mutable hash] + + Object + (hashCode [this] + (.hasheq ^clojure.lang.IHashEq this)) + + cp/Datafiable + (datafy [_] + {:data data + :cache cache + :modified modified + :hash hash}) + + IObjectsMap + (compact [this] + (locking this + (when modified + (do-compact data cache + (fn [data'] + (set! (.-modified this) false) + (set! (.-data this) data'))))) + this) + + (get-data [this] + (compact this) + data) + + (-hash-for-key [this key] + (if (contains? cache key) + (c/hash (get cache key)) + (c/hash (get this key)))) + + json/JSONWriter + (-write [this writter options] + (json/-write (into {} this) writter options)) + + clojure.lang.IHashEq + (hasheq [this] + (when-not hash + (set! hash (Murmur3/hashUnordered this))) + hash) + + clojure.lang.Seqable + (seq [this] + (RT/chunkIteratorSeq (.iterator ^Iterable this))) + + java.lang.Iterable + (iterator [this] + (ObjectsMapIterator. (.iterator ^Iterable data) this)) + + clojure.lang.IPersistentCollection + (equiv [this other] + (and (instance? ObjectsMap other) + (= (count this) (count other)) + (reduce-kv (fn [_ id _] + (let [this-val (get this id) + other-val (get other id) + result (= this-val other-val)] + (or result + (reduced false)))) + true + data))) + + clojure.lang.IPersistentMap + (cons [this o] + (if (map-entry? o) + (assoc this (key o) (val o)) + (if (vector? o) + (assoc this (nth o 0) (nth o 1)) + (throw (UnsupportedOperationException. "invalid arguments to cons"))))) + + (empty [_] + (create)) + + (containsKey [_ key] + (.containsKey ^clojure.lang.IPersistentMap data key)) + + (entryAt [this key] + (ObjectsMapEntry. this key)) + + (valAt [this key] + (or (get cache key) + (locking this + (if (contains? data key) + (let [value (get data key) + value (t/decode-str value)] + (set! (.-cache this) (assoc cache key value)) + value) + (do + (set! (.-cache this) (assoc cache key nil)) + nil))))) + + (valAt [this key not-found] + (if (.containsKey ^clojure.lang.IPersistentMap data key) + (.valAt this key) + not-found)) + + (assoc [_ key val] + (ObjectsMap. metadata + (assoc cache key val) + (assoc data key nil) + true + nil)) + + + (assocEx [_ _ _] + (throw (UnsupportedOperationException. "method not implemented"))) + + (without [_ key] + (ObjectsMap. metadata + (dissoc cache key) + (dissoc data key) + true + nil)) + + clojure.lang.Counted + (count [_] + (count data)))) + +#?(:cljs (es6-iterable ObjectsMap)) + + +(defn- do-compact + [data cache update-fn] + (let [new-data + (persistent! + (reduce-kv (fn [data id obj] + (if (nil? obj) + (assoc! data id (t/encode-str (get cache id))) + data)) + (transient data) + data))] + (update-fn new-data) + nil)) + +(defn from-data + [data] + (ObjectsMap. {} {} + data + false + nil)) + +(defn objects-map? + [o] + (instance? ObjectsMap o)) + +(defn create + ([] (from-data {})) + ([other] + (cond + (objects-map? other) + (-> other get-data from-data) + + :else + (throw #?(:clj (UnsupportedOperationException. "invalid arguments") + :cljs (js/Error. "invalid arguments")))))) + +(defn wrap + [objects] + (if (instance? ObjectsMap objects) + objects + (->> objects + (into (create)) + (compact)))) + +#?(:clj + (fres/add-handlers! + {:name "penpot/objects-map/v2" + :class ObjectsMap + :wfn (fn [n w o] + (fres/write-tag! w n) + (fres/write-object! w (get-data o))) + :rfn (fn [r] + (-> r fres/read-object! from-data))})) + +(t/add-handlers! + {:id "penpot/objects-map/v2" + :class ObjectsMap + :wfn get-data + :rfn from-data}) diff --git a/common/test/common_tests/runner.cljc b/common/test/common_tests/runner.cljc index c09ae65416..a657e096f0 100644 --- a/common/test/common_tests/runner.cljc +++ b/common/test/common_tests/runner.cljc @@ -41,6 +41,7 @@ [common-tests.types.components-test] [common-tests.types.fill-test] [common-tests.types.modifiers-test] + [common-tests.types.objects-map-test] [common-tests.types.path-data-test] [common-tests.types.shape-decode-encode-test] [common-tests.types.shape-interactions-test] @@ -90,9 +91,10 @@ 'common-tests.time-test 'common-tests.types.absorb-assets-test 'common-tests.types.components-test - 'common-tests.types.modifiers-test - 'common-tests.types.path-data-test 'common-tests.types.fill-test + 'common-tests.types.modifiers-test + 'common-tests.types.objects-map-test + 'common-tests.types.path-data-test 'common-tests.types.shape-decode-encode-test 'common-tests.types.shape-interactions-test 'common-tests.types.tokens-lib-test diff --git a/common/test/common_tests/types/objects_map_test.cljc b/common/test/common_tests/types/objects_map_test.cljc new file mode 100644 index 0000000000..a326d89b19 --- /dev/null +++ b/common/test/common_tests/types/objects_map_test.cljc @@ -0,0 +1,133 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns common-tests.types.objects-map-test + (:require + #?(:clj [app.common.fressian :as fres]) + [app.common.json :as json] + [app.common.pprint :as pp] + [app.common.schema :as sm] + [app.common.schema.generators :as sg] + [app.common.schema.test :as smt] + [app.common.transit :as transit] + [app.common.types.objects-map :as omap] + [app.common.types.path :as path] + [app.common.types.plugins :refer [schema:plugin-data]] + [app.common.types.shape :as cts] + [app.common.uuid :as uuid] + [clojure.datafy :refer [datafy]] + [clojure.test :as t])) + +(t/deftest basic-operations + (t/testing "assoc" + (let [id (uuid/custom 0 1) + id' (uuid/custom 0 2) + obj (-> (omap/create) (assoc id {:foo 1}))] + (t/is (not= id id')) + (t/is (not (contains? obj id'))) + (t/is (contains? obj id)))) + + (t/testing "assoc-with-non-uuid-keys" + (let [obj (-> (omap/create) + (assoc :a {:foo 1}) + (assoc :b {:bar 1}))] + (t/is (not (contains? obj :c))) + (t/is (contains? obj :a)) + (t/is (contains? obj :b)))) + + (t/testing "dissoc" + (let [id (uuid/custom 0 1) + obj (-> (omap/create) (assoc id {:foo 1}))] + (t/is (contains? obj id)) + (let [obj (dissoc obj id)] + (t/is (not (contains? obj id)))))) + + (t/testing "seq" + (let [id (uuid/custom 0 1) + obj (-> (omap/create) (assoc id 1))] + (t/is (contains? obj id)) + (let [[entry] (seq obj)] + (t/is (map-entry? entry)) + (t/is (= (key entry) id)) + (t/is (= (val entry) 1))))) + + (t/testing "cons & count" + (let [obj (into (omap/create) [[uuid/zero 1]])] + (t/is (contains? obj uuid/zero)) + (t/is (= 1 (count obj))) + (t/is (omap/objects-map? obj)))) + + (t/testing "wrap" + (let [obj1 (omap/wrap {}) + tmp (omap/create) + obj2 (omap/wrap tmp)] + (t/is (omap/objects-map? obj1)) + (t/is (omap/objects-map? obj2)) + (t/is (identical? tmp obj2)) + (t/is (= 0 (count obj1))) + (t/is (= 0 (count obj2)))))) + +(t/deftest internal-state + (t/testing "modified & compact" + (let [obj (-> (omap/create) + (assoc :a 1) + (assoc :b 2))] + (t/is (= 2 (count obj))) + (t/is (-> obj datafy :modified)) + (let [obj (omap/compact obj)] + (t/is (not (-> obj datafy :modified)))))) + + (t/testing "create from other" + (let [obj1 (-> (omap/create) + (assoc :a {:foo 1}) + (assoc :b {:bar 2})) + obj2 (omap/create obj1)] + + (t/is (not (identical? obj1 obj2))) + (t/is (= obj1 obj2)) + (t/is (= (hash obj1) (hash obj2))) + (t/is (= (get obj1 :a) (get obj2 :a))) + (t/is (= (get obj1 :b) (get obj2 :b)))))) + +(t/deftest creation-and-duplication + (smt/check! + (smt/for [data (->> (sg/map-of (sg/uuid) (sg/generator cts/schema:shape)) + (sg/not-empty))] + (let [obj1 (omap/wrap data) + obj2 (omap/create obj1)] + (and (= (hash obj1) (hash obj2)) + (= obj1 obj2)))) + {:num 100})) + +#?(:clj + (t/deftest fressian-encode-decode + (smt/check! + (smt/for [data (->> (sg/map-of (sg/uuid) (sg/generator cts/schema:shape)) + (sg/not-empty) + (sg/fmap omap/wrap) + (sg/fmap (fn [o] {:objects o})))] + + (let [res (-> data fres/encode fres/decode)] + (and (contains? res :objects) + (omap/objects-map? (:objects res)) + (= res data)))) + {:num 100}))) + +(t/deftest transit-encode-decode + (smt/check! + (smt/for [data (->> (sg/map-of (sg/uuid) (sg/generator cts/schema:shape)) + (sg/not-empty) + (sg/fmap omap/wrap) + (sg/fmap (fn [o] {:objects o})))] + (let [res (-> data transit/encode-str transit/decode-str)] + ;; (app.common.pprint/pprint data) + ;; (app.common.pprint/pprint res) + (and (every? (fn [[k v]] + (= v (get-in data [:objects k]))) + (:objects res)) + (omap/objects-map? (:objects data)) + (omap/objects-map? (:objects res))))) + {:num 100})) From e1519f0ee4979ab8485bd6fe819498a48e514fe3 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 25 Sep 2025 09:42:57 +0200 Subject: [PATCH 09/12] :sparkles: Integrate objects-map usage on backend and frontend --- backend/src/app/features/fdata.clj | 11 +++++------ backend/src/app/rpc/commands/files.clj | 26 ++++++++++++++++++-------- frontend/src/app/main.cljs | 1 + frontend/src/app/worker.cljs | 1 + 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/backend/src/app/features/fdata.clj b/backend/src/app/features/fdata.clj index 7c24a477b9..b6facdadba 100644 --- a/backend/src/app/features/fdata.clj +++ b/backend/src/app/features/fdata.clj @@ -12,12 +12,13 @@ [app.common.files.helpers :as cfh] [app.common.files.migrations :as fmg] [app.common.logging :as l] + [app.common.types.objects-map :as omap] [app.common.types.path :as path] [app.db :as db] [app.db.sql :as-alias sql] [app.storage :as sto] [app.util.blob :as blob] - [app.util.objects-map :as omap] + [app.util.objects-map :as omap.legacy] [app.util.pointer-map :as pmap])) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -36,10 +37,7 @@ [file & _opts] (let [update-page (fn [page] - (if (and (pmap/pointer-map? page) - (not (pmap/loaded? page))) - page - (update page :objects omap/wrap))) + (update page :objects omap/wrap)) update-data (fn [fdata] @@ -58,7 +56,8 @@ (fn [page] (update page :objects (fn [objects] - (if (omap/objects-map? objects) + (if (or (omap/objects-map? objects) + (omap.legacy/objects-map? objects)) (update-fn objects) objects))))) fdata)) diff --git a/backend/src/app/rpc/commands/files.clj b/backend/src/app/rpc/commands/files.clj index ab020338e8..d9d3f0bfd3 100644 --- a/backend/src/app/rpc/commands/files.clj +++ b/backend/src/app/rpc/commands/files.clj @@ -336,14 +336,24 @@ (cfeat/check-client-features! (:features params)) (cfeat/check-file-features! (:features file))) - ;; This operation is needed for backward comapatibility with frontends that - ;; does not support pointer-map resolution mechanism; this just resolves the - ;; pointers on backend and return a complete file. - (if (and (contains? (:features file) "fdata/pointer-map") - (not (contains? (:features params) "fdata/pointer-map"))) - (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)] - (update file :data feat.fdata/process-pointers deref)) - file)))) + (as-> file file + ;; This operation is needed for backward comapatibility with + ;; frontends that does not support pointer-map resolution + ;; mechanism; this just resolves the pointers on backend and + ;; return a complete file + (if (and (contains? (:features file) "fdata/pointer-map") + (not (contains? (:features params) "fdata/pointer-map"))) + (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)] + (update file :data feat.fdata/process-pointers deref)) + file) + + ;; This operation is needed for backward comapatibility with + ;; frontends that does not support objects-map mechanism; this + ;; just converts all objects map instaces to plain maps + (if (and (contains? (:features file) "fdata/objects-map") + (not (contains? (:features params) "fdata/objects-map"))) + (update file :data feat.fdata/process-objects (partial into {})) + file))))) ;; --- COMMAND QUERY: get-file-fragment (by id) diff --git a/frontend/src/app/main.cljs b/frontend/src/app/main.cljs index ee08002475..06fde42430 100644 --- a/frontend/src/app/main.cljs +++ b/frontend/src/app/main.cljs @@ -8,6 +8,7 @@ (:require [app.common.data.macros :as dm] [app.common.logging :as log] + [app.common.types.objects-map] [app.common.uuid :as uuid] [app.config :as cf] [app.main.data.auth :as da] diff --git a/frontend/src/app/worker.cljs b/frontend/src/app/worker.cljs index dce2f952d1..d5d5f18e44 100644 --- a/frontend/src/app/worker.cljs +++ b/frontend/src/app/worker.cljs @@ -9,6 +9,7 @@ [app.common.data.macros :as dm] [app.common.logging :as log] [app.common.schema :as sm] + [app.common.types.objects-map] [app.util.object :as obj] [app.worker.export] [app.worker.impl :as impl] From b9030fcc73b314e45efd281e3f6d7bc93b7d7de3 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 25 Sep 2025 09:56:39 +0200 Subject: [PATCH 10/12] :sparkles: Add better workspace file indexing strategy Improve file indexes initialization on workspace. Instead of initialize indexes for all pages only initialize indexes for the loaded page. --- common/src/app/common/files/indices.cljc | 56 +++-- frontend/src/app/main/data/changes.cljs | 2 +- frontend/src/app/main/data/workspace.cljs | 26 +-- .../main/data/workspace/drawing/common.cljs | 2 +- .../src/app/main/data/workspace/pages.cljs | 32 ++- .../app/main/data/workspace/selection.cljs | 2 +- frontend/src/app/main/snap.cljs | 6 +- .../app/main/ui/workspace/viewport/hooks.cljs | 2 +- frontend/src/app/worker.cljs | 2 - frontend/src/app/worker/index.cljs | 71 +++++-- frontend/src/app/worker/selection.cljs | 192 ++++++++++-------- .../{util/snap_data.cljs => worker/snap.cljs} | 2 +- frontend/src/app/worker/snaps.cljs | 40 ---- frontend/test/frontend_tests/runner.cljs | 16 +- ...p_data_test.cljs => worker_snap_test.cljs} | 160 +++++++-------- 15 files changed, 330 insertions(+), 281 deletions(-) rename frontend/src/app/{util/snap_data.cljs => worker/snap.cljs} (99%) delete mode 100644 frontend/src/app/worker/snaps.cljs rename frontend/test/frontend_tests/{util_snap_data_test.cljs => worker_snap_test.cljs} (71%) diff --git a/common/src/app/common/files/indices.cljc b/common/src/app/common/files/indices.cljc index 5dc13e3eaa..4e177f052c 100644 --- a/common/src/app/common/files/indices.cljc +++ b/common/src/app/common/files/indices.cljc @@ -6,14 +6,29 @@ (ns app.common.files.indices (:require + [app.common.data :as d] + [app.common.data.macros :as dm] [app.common.files.helpers :as cfh] [app.common.uuid :as uuid])) +(defn- generate-index + "An optimized algorithm for calculate parents index that walk from top + to down starting from a provided shape-id. Usefull when you want to + create an index for the whole objects or subpart of the tree." + [index objects shape-id parents] + (let [shape (get objects shape-id) + index (assoc index shape-id parents) + parents (cons shape-id parents)] + (reduce (fn [index shape-id] + (generate-index index objects shape-id parents)) + index + (:shapes shape)))) + (defn generate-child-all-parents-index "Creates an index where the key is the shape id and the value is a set with all the parents" ([objects] - (generate-child-all-parents-index objects (vals objects))) + (generate-index {} objects uuid/zero [])) ([objects shapes] (let [shape->entry @@ -24,24 +39,25 @@ (defn create-clip-index "Retrieves the mask information for an object" [objects parents-index] - (let [retrieve-clips + (let [get-clip-parents + (fn [shape] + (let [shape-id (dm/get-prop shape :id)] + (cond-> [] + (or (and (cfh/frame-shape? shape) + (not (:show-content shape)) + (not= uuid/zero shape-id)) + (cfh/bool-shape? shape)) + (conj shape) + + (:masked-group shape) + (conj (get objects (->> shape :shapes first)))))) + + xform + (comp (map (d/getf objects)) + (mapcat get-clip-parents)) + + populate-with-clips (fn [parents] - (let [lookup-object (fn [id] (get objects id)) - get-clip-parents - (fn [shape] - (cond-> [] - (or (and (= :frame (:type shape)) - (not (:show-content shape)) - (not= uuid/zero (:id shape))) - (cfh/bool-shape? shape)) - (conj shape) + (into [] xform parents))] - (:masked-group shape) - (conj (get objects (->> shape :shapes first)))))] - - (into [] - (comp (map lookup-object) - (mapcat get-clip-parents)) - parents)))] - (-> parents-index - (update-vals retrieve-clips)))) + (d/update-vals parents-index populate-with-clips))) diff --git a/frontend/src/app/main/data/changes.cljs b/frontend/src/app/main/data/changes.cljs index e8fc97862c..6f7799d088 100644 --- a/frontend/src/app/main/data/changes.cljs +++ b/frontend/src/app/main/data/changes.cljs @@ -52,7 +52,7 @@ (->> (rx/from changes) (rx/merge-map (fn [[page-id changes]] (log/debug :hint "update-indexes" :page-id page-id :changes (count changes)) - (mw/ask! {:cmd :index/update-page-index + (mw/ask! {:cmd :index/update :page-id page-id :changes changes}))) (rx/catch (fn [cause] diff --git a/frontend/src/app/main/data/workspace.cljs b/frontend/src/app/main/data/workspace.cljs index cf4921a8ce..8a19ef1871 100644 --- a/frontend/src/app/main/data/workspace.cljs +++ b/frontend/src/app/main/data/workspace.cljs @@ -21,7 +21,6 @@ [app.common.types.component :as ctc] [app.common.types.fills :as types.fills] [app.common.types.shape :as cts] - [app.common.types.shape-tree :as ctst] [app.common.uuid :as uuid] [app.config :as cf] [app.main.data.changes :as dch] @@ -70,7 +69,6 @@ [app.main.features.pointer-map :as fpmap] [app.main.repo :as rp] [app.main.router :as rt] - [app.main.worker :as mw] [app.render-wasm :as wasm] [app.render-wasm.api :as api] [app.util.dom :as dom] @@ -84,7 +82,7 @@ [cuerdas.core :as str] [potok.v2.core :as ptk])) -(log/set-level! :debug) +(log/set-level! :trace) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Workspace Initialization @@ -158,18 +156,9 @@ (->> (fpmap/resolve-file file) (rx/map :data) (rx/map process-fills) - (rx/mapcat - (fn [{:keys [pages-index] :as data}] - (->> (rx/from (seq pages-index)) - (rx/mapcat - (fn [[id page]] - (let [page (update page :objects ctst/start-page-index)] - (->> (mw/ask! {:cmd :index/initialize-page-index :page page}) - (rx/map (fn [_] [id page])))))) - (rx/reduce conj {}) - (rx/map (fn [pages-index] - (let [data (assoc data :pages-index pages-index)] - (assoc file :data (d/removem (comp t/pointer? val) data)))))))))) + (rx/map + (fn [data] + (assoc file :data (d/removem (comp t/pointer? val) data)))))) (defn- check-libraries-synchronozation [file-id libraries] @@ -280,6 +269,8 @@ (ptk/reify ::fetch-bundle ptk/WatchEvent (watch [_ _ stream] + (log/debug :hint "fetch bundle" :file-id (dm/str file-id)) + (let [stopper-s (rx/filter (ptk/type? ::finalize-workspace) stream)] (->> (rx/zip (rp/cmd! :get-file {:id file-id :features features}) (get-file-object-thumbnails file-id)) @@ -288,6 +279,7 @@ (fn [[file thumbnails]] (->> (resolve-file file) (rx/map (fn [file] + (log/trace :hint "file resolved" :file-id file-id) {:file file :file-id file-id :features features @@ -357,6 +349,10 @@ (rx/map deref) (rx/mapcat (fn [{:keys [file]}] + (log/debug :hint "bundle fetched" + :team-id (dm/str team-id) + :file-id (dm/str file-id)) + (rx/of (dpj/initialize-project (:project-id file)) (dwn/initialize team-id file-id) (dwsl/initialize-shape-layout) diff --git a/frontend/src/app/main/data/workspace/drawing/common.cljs b/frontend/src/app/main/data/workspace/drawing/common.cljs index 70f826e593..ab0aed9480 100644 --- a/frontend/src/app/main/data/workspace/drawing/common.cljs +++ b/frontend/src/app/main/data/workspace/drawing/common.cljs @@ -83,7 +83,7 @@ (rx/of (dwsh/add-shape shape {:no-select? (= tool :curve)})) (if (cfh/frame-shape? shape) (rx/concat - (->> (mw/ask! {:cmd :selection/query + (->> (mw/ask! {:cmd :index/query-selection :page-id page-id :rect (:selrect shape) :include-frames? true diff --git a/frontend/src/app/main/data/workspace/pages.cljs b/frontend/src/app/main/data/workspace/pages.cljs index 145b77649b..a9c9d3354e 100644 --- a/frontend/src/app/main/data/workspace/pages.cljs +++ b/frontend/src/app/main/data/workspace/pages.cljs @@ -15,6 +15,7 @@ [app.common.types.components-list :as ctkl] [app.common.types.container :as ctn] [app.common.types.page :as ctp] + [app.common.types.shape-tree :as ctst] [app.common.uuid :as uuid] [app.config :as cf] [app.main.data.changes :as dch] @@ -29,6 +30,7 @@ [app.main.errors] [app.main.features :as features] [app.main.router :as rt] + [app.main.worker :as mw] [app.render-wasm.shape :as wasm.shape] [app.util.http :as http] [app.util.i18n :as i18n :refer [tr]] @@ -56,16 +58,21 @@ (some? metadata) (cf/resolve-file-media metadata) (some? fill-image) (cf/resolve-file-media fill-image)))))) +(defn- get-page-cache + [state file-id page-id] + (dm/get-in state [:workspace-cache [file-id page-id]])) (defn- initialize-page* "Second phase of page initialization, once we know the page is available in the state" - [file-id page-id page] + [file-id page-id] (ptk/reify ::initialize-page* ptk/UpdateEvent (update [_ state] - ;; selection; when user abandon the current page, the selection is lost - (let [local (dm/get-in state [:workspace-cache [file-id page-id]] default-workspace-local)] + (let [state (dsh/update-page state file-id page-id #(update % :objects ctst/start-page-index)) + page (dsh/lookup-page state file-id page-id) + local (or (get-page-cache state file-id page-id) default-workspace-local)] + (-> state (assoc :current-page-id page-id) (assoc :workspace-local (assoc local :selected (d/ordered-set))) @@ -75,11 +82,16 @@ (update :workspace-layout layout/load-layout-flags) (update :workspace-global layout/load-layout-state)))) - ptk/EffectEvent - (effect [_ _ _] - (let [uris (into #{} xf:collect-file-media (:objects page))] - (->> (rx/from uris) - (rx/subs! #(http/fetch-data-uri % false))))))) + ptk/WatchEvent + (watch [_ state _] + (let [page (dsh/lookup-page state file-id page-id) + uris (into #{} xf:collect-file-media (:objects page))] + (rx/merge + (->> (rx/from uris) + (rx/map #(http/fetch-data-uri % false)) + (rx/ignore)) + (->> (mw/ask! {:cmd :index/initialize :page page}) + (rx/ignore))))))) (defn initialize-page [file-id page-id] @@ -89,9 +101,9 @@ (ptk/reify ::initialize-page ptk/WatchEvent (watch [_ state _] - (if-let [page (dsh/lookup-page state file-id page-id)] + (if (dsh/lookup-page state file-id page-id) (rx/concat - (rx/of (initialize-page* file-id page-id page) + (rx/of (initialize-page* file-id page-id) (dwth/watch-state-changes file-id page-id) (dwl/watch-component-changes)) (let [profile (:profile state) diff --git a/frontend/src/app/main/data/workspace/selection.cljs b/frontend/src/app/main/data/workspace/selection.cljs index 498dbd2bad..0320573b15 100644 --- a/frontend/src/app/main/data/workspace/selection.cljs +++ b/frontend/src/app/main/data/workspace/selection.cljs @@ -344,7 +344,7 @@ (if (some? selrect) (->> (ask-worker - {:cmd :selection/query + {:cmd :index/query-selection :page-id page-id :rect selrect :include-frames? true diff --git a/frontend/src/app/main/snap.cljs b/frontend/src/app/main/snap.cljs index 090e624410..f485880536 100644 --- a/frontend/src/app/main/snap.cljs +++ b/frontend/src/app/main/snap.cljs @@ -84,7 +84,7 @@ (let [value (get point coord) vbox @refs/vbox ranges [[(- value (/ 0.5 zoom)) (+ value (/ 0.5 zoom))]]] - (->> (mw/ask! {:cmd :snaps/range-query + (->> (mw/ask! {:cmd :index/query-snap :page-id page-id :frame-id frame-id :axis coord @@ -101,7 +101,7 @@ (mapv #(vector (- % snap-accuracy) (+ % snap-accuracy)))) vbox @refs/vbox] - (->> (mw/ask! {:cmd :snaps/range-query + (->> (mw/ask! {:cmd :index/query-snap :page-id page-id :frame-id frame-id :axis coord @@ -217,7 +217,7 @@ (defn select-shapes-area [page-id frame-id selected objects area] - (->> (mw/ask! {:cmd :selection/query + (->> (mw/ask! {:cmd :index/query-selection :page-id page-id :frame-id frame-id :include-frames? true diff --git a/frontend/src/app/main/ui/workspace/viewport/hooks.cljs b/frontend/src/app/main/ui/workspace/viewport/hooks.cljs index b49699e7f0..f573e405bf 100644 --- a/frontend/src/app/main/ui/workspace/viewport/hooks.cljs +++ b/frontend/src/app/main/ui/workspace/viewport/hooks.cljs @@ -194,7 +194,7 @@ (if (mf/ref-val hover-disabled-ref) (rx/of nil) (->> (mw/ask-buffered! - {:cmd :selection/query + {:cmd :index/query-selection :page-id page-id :rect rect :include-frames? true diff --git a/frontend/src/app/worker.cljs b/frontend/src/app/worker.cljs index d5d5f18e44..89476d0408 100644 --- a/frontend/src/app/worker.cljs +++ b/frontend/src/app/worker.cljs @@ -16,8 +16,6 @@ [app.worker.import] [app.worker.index] [app.worker.messages :as wm] - [app.worker.selection] - [app.worker.snaps] [app.worker.thumbnails] [beicon.v2.core :as rx] [promesa.core :as p])) diff --git a/frontend/src/app/worker/index.cljs b/frontend/src/app/worker/index.cljs index 77c2b31f30..b89fad06ee 100644 --- a/frontend/src/app/worker/index.cljs +++ b/frontend/src/app/worker/index.cljs @@ -9,26 +9,69 @@ (:require [app.common.data.macros :as dm] [app.common.files.changes :as ch] + [app.common.geom.rect :as grc] + [app.common.logging :as log] + [app.common.time :as ct] [app.worker.impl :as impl] + [app.worker.selection :as selection] + [app.worker.snap :as snap] [okulary.core :as l])) +(log/set-level! :info) + (defonce state (l/atom {:pages-index {}})) -(defmethod impl/handler :index/initialize-page-index +(defmethod impl/handler :index/initialize [{:keys [page] :as message}] - (swap! state update :pages-index assoc (:id page) page) - (impl/handler (assoc message :cmd :selection/initialize-page-index)) - (impl/handler (assoc message :cmd :snaps/initialize-page-index))) + (let [tpoint (ct/tpoint-ms)] + (try + (swap! state update :pages-index assoc (:id page) page) + (swap! state update ::selection selection/add-page page) + (swap! state update ::snap snap/add-page page) -(defmethod impl/handler :index/update-page-index + (finally + (let [elapsed (tpoint)] + (log/dbg :hint "page indexed" :id (:id page) :elapsed elapsed ::log/sync? true)))) + nil)) + +(defmethod impl/handler :index/update [{:keys [page-id changes] :as message}] + (let [tpoint (ct/tpoint-ms)] + (try + (let [old-page (dm/get-in @state [:pages-index page-id]) + new-page (-> state + (swap! ch/process-changes changes false) + (dm/get-in [:pages-index page-id]))] - (let [old-page (dm/get-in @state [:pages-index page-id]) - new-page (-> state - (swap! ch/process-changes changes false) - (dm/get-in [:pages-index page-id])) - message (assoc message - :old-page old-page - :new-page new-page)] - (impl/handler (assoc message :cmd :selection/update-page-index)) - (impl/handler (assoc message :cmd :snaps/update-page-index)))) + (swap! state update ::snap snap/update-page old-page new-page) + (swap! state update ::selection selection/update-page old-page new-page)) + (finally + (let [elapsed (tpoint)] + (log/dbg :hint "page index updated" :id page-id :elapsed elapsed ::log/sync? true)))) + nil)) + +;; FIXME: schema + +(defmethod impl/handler :index/query-snap + [{:keys [page-id frame-id axis ranges bounds] :as message}] + (if-let [index (get @state ::snap)] + (let [match-bounds? + (fn [[_ data]] + (some #(or (= :guide (:type %)) + (= :layout (:type %)) + (grc/contains-point? bounds (:pt %))) data)) + + xform + (comp (mapcat #(snap/query index page-id frame-id axis %)) + (distinct) + (filter match-bounds?))] + (into [] xform ranges)) + [])) + +;; FIXME: schema + +(defmethod impl/handler :index/query-selection + [message] + (if-let [index (get @state ::selection)] + (selection/query index message) + [])) diff --git a/frontend/src/app/worker/selection.cljs b/frontend/src/app/worker/selection.cljs index f6a917b003..e48fbfc2c9 100644 --- a/frontend/src/app/worker/selection.cljs +++ b/frontend/src/app/worker/selection.cljs @@ -17,46 +17,62 @@ [app.common.types.modifiers :as ctm] [app.common.uuid :as uuid] [app.util.quadtree :as qdt] - [app.worker.impl :as impl] - [clojure.set :as set] - [okulary.core :as l])) + [clojure.set :as set])) -;; FIXME: performance shape & rect static props +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; IMPL +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (def ^:const padding-percent 0.10) -(defonce state (l/atom {})) +(defn- index-shape + "A reducing function that ads a shape to the index" + [objects parents-index clip-index index shape] + (let [bounds + (cond + (and ^boolean (cfh/text-shape? shape) + ^boolean (some? (:position-data shape)) + ^boolean (d/not-empty? (:position-data shape))) + (gst/shape->bounds shape) -(defn make-index-shape - [objects parents-index clip-parents-index] - (fn [index shape] - (let [{:keys [x y width height]} - (cond - (and ^boolean (cfh/text-shape? shape) - ^boolean (some? (:position-data shape)) - ^boolean (d/not-empty? (:position-data shape))) - (gst/shape->bounds shape) + :else + (grc/points->rect (:points shape))) - :else - (grc/points->rect (:points shape))) + bound + #js {:x (dm/get-prop bounds :x) + :y (dm/get-prop bounds :y) + :width (dm/get-prop bounds :width) + :height (dm/get-prop bounds :height)} - shape-bound #js {:x x :y y :width width :height height} + shape-id + (dm/get-prop shape :id) - parents (get parents-index (:id shape)) - clip-parents (get clip-parents-index (:id shape)) + frame-id + (dm/get-prop shape :frame-id) - frame (when (and (not= :frame (:type shape)) - (not= (:frame-id shape) uuid/zero)) - (get objects (:frame-id shape)))] - (qdt/insert index - (:id shape) - shape-bound - (assoc shape - :frame frame - :clip-parents clip-parents - :parents parents))))) + shape-type + (dm/get-prop shape :type) -(defn objects-bounds + parents + (get parents-index shape-id) + + clip-parents + (get clip-index shape-id) + + frame + (when (and (not= :frame shape-type) + (not= frame-id uuid/zero)) + (get objects frame-id))] + + (qdt/insert index + shape-id + bound + (assoc shape + :frame frame + :clip-parents clip-parents + :parents parents)))) + +(defn- objects-bounds "Calculates the bounds of the quadtree given a objects map." [objects] (-> objects @@ -64,7 +80,7 @@ vals gsh/shapes->rect)) -(defn add-padding-bounds +(defn- add-padding-bounds "Adds a padding to the bounds defined as a percent in the constant `padding-percent`. For a value of 0.1 will add a 20% width increase (2 x padding)" [bounds] @@ -81,41 +97,48 @@ (defn- create-index [objects] - (let [shapes (-> objects (dissoc uuid/zero) vals) - parents-index (cfi/generate-child-all-parents-index objects) - clip-parents-index (cfi/create-clip-index objects parents-index) - - root-shapes (cfh/get-immediate-children objects uuid/zero) - bounds (-> root-shapes gsh/shapes->rect add-padding-bounds) - - index-shape (make-index-shape objects parents-index clip-parents-index) - initial-quadtree (qdt/create (clj->js bounds)) - - index (reduce index-shape initial-quadtree shapes)] + (let [parents-index (cfi/generate-child-all-parents-index objects) + clip-index (cfi/create-clip-index objects parents-index) + root-shapes (cfh/get-immediate-children objects uuid/zero) + bounds (-> root-shapes gsh/shapes->rect add-padding-bounds) + index (reduce-kv #(index-shape objects parents-index clip-index %1 %3) + (qdt/create (clj->js bounds)) + (dissoc objects uuid/zero))] {:index index :bounds bounds})) +;; FIXME: optimize (defn- update-index [{index :index :as data} old-objects new-objects] - (let [changes? (fn [id] - (not= (get old-objects id) - (get new-objects id))) + (let [object-changed? + (fn [id] + (not= (get old-objects id) + (get new-objects id))) - changed-ids (into #{} - (comp (filter #(not= % uuid/zero)) - (filter changes?) - (mapcat #(into [%] (cfh/get-children-ids new-objects %)))) - (set/union (set (keys old-objects)) - (set (keys new-objects)))) + changed-ids + (into #{} + (comp (filter #(not= % uuid/zero)) + (filter object-changed?) + (mapcat #(into [%] (cfh/get-children-ids new-objects %)))) - shapes (->> changed-ids (mapv #(get new-objects %)) (filterv (comp not nil?))) - parents-index (cfi/generate-child-all-parents-index new-objects shapes) - clip-parents-index (cfi/create-clip-index new-objects parents-index) + (set/union (set (keys old-objects)) + (set (keys new-objects)))) - new-index (qdt/remove-all index changed-ids) + shapes + (->> changed-ids + (map #(get new-objects %)) + (filterv (comp not nil?))) - index-shape (make-index-shape new-objects parents-index clip-parents-index) - index (reduce index-shape new-index shapes)] + parents-index + (cfi/generate-child-all-parents-index new-objects shapes) + + clip-index + (cfi/create-clip-index new-objects parents-index) + + index + (reduce #(index-shape new-objects parents-index clip-index %1 %2) + (qdt/remove-all index changed-ids) + shapes)] (assoc data :index index))) @@ -231,35 +254,36 @@ (map :id)) result))) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; PUBLIC API +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(defmethod impl/handler :selection/initialize-page-index - [{:keys [page] :as message}] - (letfn [(add-page [state {:keys [id objects] :as page}] - (assoc state id (create-index objects)))] - (swap! state add-page page) - nil)) +(defn add-page + "Add a page index to the state" + [state {:keys [id objects] :as page}] + (assoc state id (create-index objects))) -(defmethod impl/handler :selection/update-page-index - [{:keys [page-id old-page new-page] :as message}] - (swap! state update page-id - (fn [index] - (let [old-objects (:objects old-page) - new-objects (:objects new-page) - old-bounds (:bounds index) - new-bounds (objects-bounds new-objects)] +(defn update-page + "Update page index on the state" + [state old-page new-page] + (let [page-id (get old-page :id)] + (update state page-id + (fn [index] + (let [old-objects (:objects old-page) + new-objects (:objects new-page) + old-bounds (:bounds index) + new-bounds (objects-bounds new-objects)] - ;; If the new bounds are contained within the old bounds - ;; we can update the index. Otherwise we need to - ;; re-create it. - (if (and (some? index) - (grc/contains-rect? old-bounds new-bounds)) - (update-index index old-objects new-objects) - (create-index new-objects))))) - nil) + ;; If the new bounds are contained within the old bounds + ;; we can update the index. Otherwise we need to + ;; re-create it. + (if (and (some? index) + (grc/contains-rect? old-bounds new-bounds)) + (update-index index old-objects new-objects) + (create-index new-objects))))))) -(defmethod impl/handler :selection/query - [{:keys [page-id rect frame-id full-frame? include-frames? ignore-groups? clip-children? using-selrect?] - :or {full-frame? false include-frames? false clip-children? true using-selrect? false} - :as message}] - (when-let [index (get @state page-id)] +(defn query + [index {:keys [page-id rect frame-id full-frame? include-frames? ignore-groups? clip-children? using-selrect?] + :or {full-frame? false include-frames? false clip-children? true using-selrect? false}}] + (when-let [index (get index page-id)] (query-index index rect frame-id full-frame? include-frames? ignore-groups? clip-children? using-selrect?))) diff --git a/frontend/src/app/util/snap_data.cljs b/frontend/src/app/worker/snap.cljs similarity index 99% rename from frontend/src/app/util/snap_data.cljs rename to frontend/src/app/worker/snap.cljs index d8fc89cea9..494c34adff 100644 --- a/frontend/src/app/util/snap_data.cljs +++ b/frontend/src/app/worker/snap.cljs @@ -4,7 +4,7 @@ ;; ;; Copyright (c) KALEIDOS INC -(ns app.util.snap-data +(ns app.worker.snap "Data structure that holds and retrieves the data to make the snaps. Internally is implemented with a balanced binary tree that queries by range. https://en.wikipedia.org/wiki/Range_tree" diff --git a/frontend/src/app/worker/snaps.cljs b/frontend/src/app/worker/snaps.cljs deleted file mode 100644 index 77bf5d8f5a..0000000000 --- a/frontend/src/app/worker/snaps.cljs +++ /dev/null @@ -1,40 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS INC - -(ns app.worker.snaps - (:require - [app.common.geom.rect :as grc] - [app.util.snap-data :as sd] - [app.worker.impl :as impl] - [okulary.core :as l])) - -(defonce state (l/atom {})) - -;; Public API -(defmethod impl/handler :snaps/initialize-page-index - [{:keys [page] :as message}] - (swap! state sd/add-page page) - nil) - -(defmethod impl/handler :snaps/update-page-index - [{:keys [old-page new-page] :as message}] - (swap! state sd/update-page old-page new-page) - nil) - -(defmethod impl/handler :snaps/range-query - [{:keys [page-id frame-id axis ranges bounds] :as message}] - (let [match-bounds? - (fn [[_ data]] - (some #(or (= :guide (:type %)) - (= :layout (:type %)) - (grc/contains-point? bounds (:pt %))) data))] - (->> (into [] - (comp (mapcat #(sd/query @state page-id frame-id axis %)) - (distinct)) - ranges) - (filter match-bounds?)))) - - diff --git a/frontend/test/frontend_tests/runner.cljs b/frontend/test/frontend_tests/runner.cljs index 1b0f413364..11dc224dfa 100644 --- a/frontend/test/frontend_tests/runner.cljs +++ b/frontend/test/frontend_tests/runner.cljs @@ -18,7 +18,7 @@ [frontend-tests.tokens.token-form-test] [frontend-tests.util-range-tree-test] [frontend-tests.util-simple-math-test] - [frontend-tests.util-snap-data-test])) + [frontend-tests.worker-snap-test])) (enable-console-print!) @@ -30,6 +30,8 @@ (defn init [] (t/run-tests + 'frontend-tests.basic-shapes-test + 'frontend-tests.data.workspace-colors-test 'frontend-tests.helpers-shapes-test 'frontend-tests.logic.comp-remove-swap-slots-test 'frontend-tests.logic.components-and-tokens @@ -38,13 +40,11 @@ 'frontend-tests.logic.groups-test 'frontend-tests.logic.pasting-in-containers-test 'frontend-tests.plugins.context-shapes-test - 'frontend-tests.util-range-tree-test - 'frontend-tests.util-snap-data-test - 'frontend-tests.util-simple-math-test - 'frontend-tests.basic-shapes-test - 'frontend-tests.data.workspace-colors-test + 'frontend-tests.tokens.import-export-test 'frontend-tests.tokens.logic.token-actions-test 'frontend-tests.tokens.logic.token-data-test - 'frontend-tests.tokens.import-export-test 'frontend-tests.tokens.style-dictionary-test - 'frontend-tests.tokens.token-form-test)) + 'frontend-tests.tokens.token-form-test + 'frontend-tests.util-range-tree-test + 'frontend-tests.util-simple-math-test + 'frontend-tests.worker-snap-test)) diff --git a/frontend/test/frontend_tests/util_snap_data_test.cljs b/frontend/test/frontend_tests/worker_snap_test.cljs similarity index 71% rename from frontend/test/frontend_tests/util_snap_data_test.cljs rename to frontend/test/frontend_tests/worker_snap_test.cljs index 0ad546a672..280db50e56 100644 --- a/frontend/test/frontend_tests/util_snap_data_test.cljs +++ b/frontend/test/frontend_tests/worker_snap_test.cljs @@ -4,12 +4,12 @@ ;; ;; Copyright (c) KALEIDOS INC -(ns frontend-tests.util-snap-data-test +(ns frontend-tests.worker-snap-test (:require [app.common.files.builder :as fb] [app.common.types.shape :as cts] [app.common.uuid :as uuid] - [app.util.snap-data :as sd] + [app.worker.snap :as snap] [cljs.pprint :refer [pprint]] [cljs.test :as t :include-macros true])) @@ -21,9 +21,9 @@ (fn [] (uuid/custom 123456789 (swap! counter inc))))) -(t/deftest test-create-index +(t/deftest create-index (t/testing "Create empty data" - (let [data (sd/make-snap-data)] + (let [data (snap/make-snap-data)] (t/is (some? data)))) (t/testing "Add empty page (only root-frame)" @@ -32,8 +32,8 @@ (fb/add-page {:name "Page 1"}) (fb/get-current-page)) - data (-> (sd/make-snap-data) - (sd/add-page page))] + data (-> (snap/make-snap-data) + (snap/add-page page))] (t/is (some? data)))) (t/testing "Create simple shape on root" @@ -48,10 +48,10 @@ :height 100})) page (fb/get-current-page state) - data (-> (sd/make-snap-data) - (sd/add-page page)) + data (-> (snap/make-snap-data) + (snap/add-page page)) - result-x (sd/query data (:id page) uuid/zero :x [0 100])] + result-x (snap/query data (:id page) uuid/zero :x [0 100])] (t/is (some? data)) @@ -82,11 +82,11 @@ page (fb/get-current-page state) ;; frame-id (::fb/last-id file) - data (-> (sd/make-snap-data) - (sd/add-page page)) + data (-> (snap/make-snap-data) + (snap/add-page page)) - result-zero-x (sd/query data (:id page) uuid/zero :x [0 100]) - result-frame-x (sd/query data (:id page) frame-id :x [0 100])] + result-zero-x (snap/query data (:id page) uuid/zero :x [0 100]) + result-frame-x (snap/query data (:id page) frame-id :x [0 100])] (t/is (some? data)) (t/is (= (count result-zero-x) 3)) @@ -116,11 +116,11 @@ page (fb/get-current-page state) - data (-> (sd/make-snap-data) - (sd/add-page page)) + data (-> (snap/make-snap-data) + (snap/add-page page)) - result-zero-x (sd/query data (:id page) uuid/zero :x [0 100]) - result-frame-x (sd/query data (:id page) frame-id :x [0 100])] + result-zero-x (snap/query data (:id page) uuid/zero :x [0 100]) + result-frame-x (snap/query data (:id page) frame-id :x [0 100])] (t/is (some? data)) (t/is (= (count result-zero-x) 3)) @@ -137,13 +137,13 @@ frame-id (::fb/last-id state) page (fb/get-current-page state) - data (-> (sd/make-snap-data) - (sd/add-page page)) + data (-> (snap/make-snap-data) + (snap/add-page page)) - result-zero-x (sd/query data (:id page) uuid/zero :x [0 100]) - result-zero-y (sd/query data (:id page) uuid/zero :y [0 100]) - result-frame-x (sd/query data (:id page) frame-id :x [0 100]) - result-frame-y (sd/query data (:id page) frame-id :y [0 100])] + result-zero-x (snap/query data (:id page) uuid/zero :x [0 100]) + result-zero-y (snap/query data (:id page) uuid/zero :y [0 100]) + result-frame-x (snap/query data (:id page) frame-id :x [0 100]) + result-frame-y (snap/query data (:id page) frame-id :y [0 100])] (t/is (some? data)) ;; We can snap in the root @@ -168,13 +168,13 @@ page (fb/get-current-page state) - data (-> (sd/make-snap-data) - (sd/add-page page)) + data (-> (snap/make-snap-data) + (snap/add-page page)) - result-zero-x (sd/query data (:id page) uuid/zero :x [0 100]) - result-zero-y (sd/query data (:id page) uuid/zero :y [0 100]) - result-frame-x (sd/query data (:id page) frame-id :x [0 100]) - result-frame-y (sd/query data (:id page) frame-id :y [0 100])] + result-zero-x (snap/query data (:id page) uuid/zero :x [0 100]) + result-zero-y (snap/query data (:id page) uuid/zero :y [0 100]) + result-frame-x (snap/query data (:id page) frame-id :x [0 100]) + result-frame-y (snap/query data (:id page) frame-id :y [0 100])] (t/is (some? data)) ;; We can snap in the root @@ -185,7 +185,7 @@ (t/is (= (count result-frame-x) 1)) (t/is (= (count result-frame-y) 0))))) -(t/deftest test-update-index +(t/deftest update-index (t/testing "Create frame on root and then remove it." (let [state (-> (fb/create-state) (fb/add-file {:name "Test"}) @@ -200,17 +200,17 @@ shape-id (::fb/last-id state) page (fb/get-current-page state) - data (-> (sd/make-snap-data) - (sd/add-page page)) + data (-> (snap/make-snap-data) + (snap/add-page page)) state (-> state (fb/delete-shape shape-id)) new-page (fb/get-current-page state) - data (sd/update-page data page new-page) + data (snap/update-page data page new-page) - result-x (sd/query data (:id page) uuid/zero :x [0 100]) - result-y (sd/query data (:id page) uuid/zero :y [0 100])] + result-x (snap/query data (:id page) uuid/zero :x [0 100]) + result-y (snap/query data (:id page) uuid/zero :y [0 100])] (t/is (some? data)) (t/is (= (count result-x) 0)) @@ -231,16 +231,16 @@ page (fb/get-current-page state) ;; frame-id (::fb/last-id state) - data (-> (sd/make-snap-data) - (sd/add-page page)) + data (-> (snap/make-snap-data) + (snap/add-page page)) state (fb/delete-shape state shape-id) new-page (fb/get-current-page state) - data (sd/update-page data page new-page) + data (snap/update-page data page new-page) - result-x (sd/query data (:id page) uuid/zero :x [0 100]) - result-y (sd/query data (:id page) uuid/zero :y [0 100])] + result-x (snap/query data (:id page) uuid/zero :x [0 100]) + result-y (snap/query data (:id page) uuid/zero :y [0 100])] (t/is (some? data)) (t/is (= (count result-x) 0)) @@ -263,16 +263,16 @@ state (fb/close-board state) page (fb/get-current-page state) - data (-> (sd/make-snap-data) - (sd/add-page page)) + data (-> (snap/make-snap-data) + (snap/add-page page)) state (fb/delete-shape state shape-id) new-page (fb/get-current-page state) - data (sd/update-page data page new-page) + data (snap/update-page data page new-page) - result-zero-x (sd/query data (:id page) uuid/zero :x [0 100]) - result-frame-x (sd/query data (:id page) frame-id :x [0 100])] + result-zero-x (snap/query data (:id page) uuid/zero :x [0 100]) + result-frame-x (snap/query data (:id page) frame-id :x [0 100])] (t/is (some? data)) (t/is (= (count result-zero-x) 3)) @@ -291,18 +291,18 @@ frame-id (::fb/last-id state) page (fb/get-current-page state) - data (-> (sd/make-snap-data) - (sd/add-page page)) + data (-> (snap/make-snap-data) + (snap/add-page page)) new-page (-> (fb/delete-guide state guide-id) (fb/get-current-page)) - data (sd/update-page data page new-page) + data (snap/update-page data page new-page) - result-zero-x (sd/query data (:id page) uuid/zero :x [0 100]) - result-zero-y (sd/query data (:id page) uuid/zero :y [0 100]) - result-frame-x (sd/query data (:id page) frame-id :x [0 100]) - result-frame-y (sd/query data (:id page) frame-id :y [0 100])] + result-zero-x (snap/query data (:id page) uuid/zero :x [0 100]) + result-zero-y (snap/query data (:id page) uuid/zero :y [0 100]) + result-frame-x (snap/query data (:id page) frame-id :x [0 100]) + result-frame-y (snap/query data (:id page) frame-id :y [0 100])] (t/is (some? data)) ;; We can snap in the root @@ -325,17 +325,17 @@ guide-id (::fb/last-id file) page (fb/get-current-page file) - data (-> (sd/make-snap-data) (sd/add-page page)) + data (-> (snap/make-snap-data) (snap/add-page page)) new-page (-> (fb/delete-guide file guide-id) (fb/get-current-page)) - data (sd/update-page data page new-page) + data (snap/update-page data page new-page) - result-zero-x (sd/query data (:id page) uuid/zero :x [0 100]) - result-zero-y (sd/query data (:id page) uuid/zero :y [0 100]) - result-frame-x (sd/query data (:id page) frame-id :x [0 100]) - result-frame-y (sd/query data (:id page) frame-id :y [0 100])] + result-zero-x (snap/query data (:id page) uuid/zero :x [0 100]) + result-zero-y (snap/query data (:id page) uuid/zero :y [0 100]) + result-frame-x (snap/query data (:id page) frame-id :x [0 100]) + result-frame-y (snap/query data (:id page) frame-id :y [0 100])] (t/is (some? data)) ;; We can snap in the root (t/is (= (count result-zero-x) 0)) @@ -358,8 +358,8 @@ frame-id (::fb/last-id state) page (fb/get-current-page state) - data (-> (sd/make-snap-data) - (sd/add-page page)) + data (-> (snap/make-snap-data) + (snap/add-page page)) state (fb/update-shape state frame-id (fn [shape] @@ -370,12 +370,12 @@ new-page (fb/get-current-page state) - data (sd/update-page data page new-page) + data (snap/update-page data page new-page) - result-zero-x-1 (sd/query data (:id page) uuid/zero :x [0 100]) - result-frame-x-1 (sd/query data (:id page) frame-id :x [0 100]) - result-zero-x-2 (sd/query data (:id page) uuid/zero :x [200 300]) - result-frame-x-2 (sd/query data (:id page) frame-id :x [200 300])] + result-zero-x-1 (snap/query data (:id page) uuid/zero :x [0 100]) + result-frame-x-1 (snap/query data (:id page) frame-id :x [0 100]) + result-zero-x-2 (snap/query data (:id page) uuid/zero :x [200 300]) + result-frame-x-2 (snap/query data (:id page) frame-id :x [200 300])] (t/is (some? data)) (t/is (= (count result-zero-x-1) 0)) @@ -396,8 +396,8 @@ shape-id (::fb/last-id state) page (fb/get-current-page state) - data (-> (sd/make-snap-data) - (sd/add-page page)) + data (-> (snap/make-snap-data) + (snap/add-page page)) state (fb/update-shape state shape-id (fn [shape] @@ -408,10 +408,10 @@ new-page (fb/get-current-page state) ;; FIXME: update - data (sd/update-page data page new-page) + data (snap/update-page data page new-page) - result-zero-x-1 (sd/query data (:id page) uuid/zero :x [0 100]) - result-zero-x-2 (sd/query data (:id page) uuid/zero :x [200 300])] + result-zero-x-1 (snap/query data (:id page) uuid/zero :x [0 100]) + result-zero-x-2 (snap/query data (:id page) uuid/zero :x [200 300])] (t/is (some? data)) (t/is (= (count result-zero-x-1) 0)) @@ -432,22 +432,22 @@ frame-id (::fb/last-id state) page (fb/get-current-page state) - data (-> (sd/make-snap-data) (sd/add-page page)) + data (-> (snap/make-snap-data) (snap/add-page page)) new-page (-> (fb/update-guide state (assoc guide :position 150)) (fb/get-current-page)) - data (sd/update-page data page new-page) + data (snap/update-page data page new-page) - result-zero-x-1 (sd/query data (:id page) uuid/zero :x [0 100]) - result-zero-y-1 (sd/query data (:id page) uuid/zero :y [0 100]) - result-frame-x-1 (sd/query data (:id page) frame-id :x [0 100]) - result-frame-y-1 (sd/query data (:id page) frame-id :y [0 100]) + result-zero-x-1 (snap/query data (:id page) uuid/zero :x [0 100]) + result-zero-y-1 (snap/query data (:id page) uuid/zero :y [0 100]) + result-frame-x-1 (snap/query data (:id page) frame-id :x [0 100]) + result-frame-y-1 (snap/query data (:id page) frame-id :y [0 100]) - result-zero-x-2 (sd/query data (:id page) uuid/zero :x [0 200]) - result-zero-y-2 (sd/query data (:id page) uuid/zero :y [0 200]) - result-frame-x-2 (sd/query data (:id page) frame-id :x [0 200]) - result-frame-y-2 (sd/query data (:id page) frame-id :y [0 200])] + result-zero-x-2 (snap/query data (:id page) uuid/zero :x [0 200]) + result-zero-y-2 (snap/query data (:id page) uuid/zero :y [0 200]) + result-frame-x-2 (snap/query data (:id page) frame-id :x [0 200]) + result-frame-y-2 (snap/query data (:id page) frame-id :y [0 200])] (t/is (some? data)) From 27e311277a8691bf8b992f1871b70e84c279f701 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 25 Sep 2025 09:57:56 +0200 Subject: [PATCH 11/12] :sparkles: Add logging to frontend repo namespace --- frontend/src/app/main/repo.cljs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/main/repo.cljs b/frontend/src/app/main/repo.cljs index d863465b42..7294f823cf 100644 --- a/frontend/src/app/main/repo.cljs +++ b/frontend/src/app/main/repo.cljs @@ -8,6 +8,8 @@ (:require [app.common.data :as d] [app.common.exceptions :as ex] + [app.common.logging :as log] + [app.common.time :as ct] [app.common.transit :as t] [app.common.uri :as u] [app.config :as cf] @@ -17,6 +19,8 @@ [beicon.v2.core :as rx] [cuerdas.core :as str])) +(log/set-level! :info) + (defn handle-response [{:keys [status body headers uri] :as response}] (cond @@ -126,13 +130,21 @@ (select-keys params query-params) nil)) :response-type - (if stream? nil response-type)}] + (if stream? nil response-type)} + + tpoint + (ct/tpoint-ms)] + + (log/trc :hint "make request" :id id) (->> (http/fetch request) (rx/map http/response->map) (rx/mapcat (fn [{:keys [headers body] :as response}] + (log/trc :hint "response received" :id id :elapsed (tpoint)) + (let [ctype (get headers "content-type") - response-stream? (str/starts-with? ctype "text/event-stream")] + response-stream? (str/starts-with? ctype "text/event-stream") + tpoint (ct/tpoint-ms)] (when (and response-stream? (not stream?)) (ex/raise :type :internal @@ -148,6 +160,8 @@ (->> response (http/process-response-type response-type) (rx/map decode-fn) + (rx/tap (fn [_] + (log/trc :hint "response decoded" :id id :elapsed (tpoint)))) (rx/mapcat handle-response))))))))) (defmulti cmd! (fn [id _] id)) From f32112544eaed3ff8473e881d0b0b78d544ca148 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 25 Sep 2025 16:36:09 +0200 Subject: [PATCH 12/12] :sparkles: Make deleted fonts fixer to run with more granular stragegy Instead of running it on all the file, only run it to local library and the current page, reducing considerably the overhead of analyzing the whole file on each file load. It stills executes for page each time the page is loaded, and add some kind of local cache for not doing repeated work each time page loads is pending to be implemented in other commit. --- frontend/src/app/main/data/workspace.cljs | 4 +- .../data/workspace/fix_broken_shapes.cljs | 56 ------ .../data/workspace/fix_deleted_fonts.cljs | 164 ++++++++---------- .../src/app/main/data/workspace/pages.cljs | 2 + 4 files changed, 79 insertions(+), 147 deletions(-) delete mode 100644 frontend/src/app/main/data/workspace/fix_broken_shapes.cljs diff --git a/frontend/src/app/main/data/workspace.cljs b/frontend/src/app/main/data/workspace.cljs index 8a19ef1871..b516e9e75a 100644 --- a/frontend/src/app/main/data/workspace.cljs +++ b/frontend/src/app/main/data/workspace.cljs @@ -42,7 +42,6 @@ [app.main.data.workspace.common :as dwc] [app.main.data.workspace.drawing :as dwd] [app.main.data.workspace.edition :as dwe] - [app.main.data.workspace.fix-broken-shapes :as fbs] [app.main.data.workspace.fix-deleted-fonts :as fdf] [app.main.data.workspace.groups :as dwg] [app.main.data.workspace.guides :as dwgu] @@ -232,8 +231,7 @@ ptk/WatchEvent (watch [_ _ _] (rx/of (dp/check-open-plugin) - (fdf/fix-deleted-fonts) - (fbs/fix-broken-shapes))))) + (fdf/fix-deleted-fonts-for-local-library file-id))))) (defn- bundle-fetched [{:keys [file file-id thumbnails] :as bundle}] diff --git a/frontend/src/app/main/data/workspace/fix_broken_shapes.cljs b/frontend/src/app/main/data/workspace/fix_broken_shapes.cljs deleted file mode 100644 index 78f88dc823..0000000000 --- a/frontend/src/app/main/data/workspace/fix_broken_shapes.cljs +++ /dev/null @@ -1,56 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS INC - -(ns app.main.data.workspace.fix-broken-shapes - (:require - [app.main.data.changes :as dch] - [app.main.data.helpers :as dsh] - [beicon.v2.core :as rx] - [potok.v2.core :as ptk])) - -(defn- generate-broken-link-changes - [attr {:keys [objects id] :as container}] - (let [base {:type :fix-obj :fix :broken-children attr id} - contains? (partial contains? objects) - xform (comp - ;; FIXME: Ensure all obj have id field (this is needed - ;; because some bug adds an ephimeral shape with id ZERO, - ;; with a single attr `:shapes` having a vector of ids - ;; pointing to not existing shapes). That happens on - ;; components. THIS IS A WORKAOURD - (map (fn [[id obj]] - (if (some? (:id obj)) - obj - (assoc obj :id id)))) - - ;; Remove all valid shapes - (remove (fn [obj] - (every? contains? (:shapes obj)))) - - (map (fn [obj] - (assoc base :id (:id obj)))))] - - (sequence xform objects))) - -(defn fix-broken-shapes - [] - (ptk/reify ::fix-broken-shapes - ptk/WatchEvent - (watch [it state _] - (let [fdata (dsh/lookup-file-data state) - changes (concat - (mapcat (partial generate-broken-link-changes :page-id) - (vals (:pages-index fdata))) - (mapcat (partial generate-broken-link-changes :component-id) - (vals (:components fdata))))] - - (if (seq changes) - (rx/of (dch/commit-changes - {:origin it - :redo-changes (vec changes) - :undo-changes [] - :save-undo? false})) - (rx/empty)))))) diff --git a/frontend/src/app/main/data/workspace/fix_deleted_fonts.cljs b/frontend/src/app/main/data/workspace/fix_deleted_fonts.cljs index 1ffe8a1ba4..fb33a74dc7 100644 --- a/frontend/src/app/main/data/workspace/fix_deleted_fonts.cljs +++ b/frontend/src/app/main/data/workspace/fix_deleted_fonts.cljs @@ -14,8 +14,9 @@ [beicon.v2.core :as rx] [potok.v2.core :as ptk])) -;; This event will update the file so the texts with non existing custom fonts try to be fixed. -;; This can happen when: +;; This event will update the file so the texts with non existing +;; custom fonts try to be fixed. This can happen when: +;; ;; - Exporting/importing files to different teams or penpot instances ;; - Moving files from one team to another in the same instance ;; - Custom fonts are explicitly deleted in the team area @@ -23,112 +24,99 @@ (defn- calculate-alternative-font-id [value] (let [fonts (deref fonts/fontsdb)] - (->> (vals fonts) - (filter #(= (:family %) value)) - (first) - :id))) + (reduce-kv (fn [_ _ font] + (if (= (:family font) value) + (reduced (:id font)) + nil)) + nil + fonts))) (defn- has-invalid-font-family? [node] - (let [fonts (deref fonts/fontsdb) - font-family (:font-family node) - alternative-font-id (calculate-alternative-font-id font-family)] + (let [fonts (deref fonts/fontsdb) + font-family (:font-family node)] (and (some? font-family) - (nil? (get fonts (:font-id node))) - (some? alternative-font-id)))) + (nil? (get fonts (:font-id node)))))) -(defn- should-fix-deleted-font-shape? +(defn- shape-has-invalid-font-family?? [shape] - (let [text-nodes (txt/node-seq txt/is-text-node? (:content shape))] - (and (cfh/text-shape? shape) - (some has-invalid-font-family? text-nodes)))) - -(defn- should-fix-deleted-font-component? - [component] - (let [xf (comp (map val) - (filter should-fix-deleted-font-shape?))] - (first (sequence xf (:objects component))))) + (and (cfh/text-shape? shape) + (some has-invalid-font-family? + (txt/node-seq txt/is-text-node? (:content shape))))) (defn- fix-deleted-font [node] - (let [alternative-font-id (calculate-alternative-font-id (:font-family node))] - (cond-> node - (some? alternative-font-id) (assoc :font-id alternative-font-id)))) + (if-let [alternative-font-id (calculate-alternative-font-id (:font-family node))] + (assoc node :font-id alternative-font-id) + node)) -(defn- fix-deleted-font-shape +(defn- fix-shape-content [shape] - (let [transform (partial txt/transform-nodes has-invalid-font-family? fix-deleted-font)] - (update shape :content transform))) + (txt/transform-nodes has-invalid-font-family? fix-deleted-font + (:content shape))) -(defn- fix-deleted-font-component - [component] - (update component - :objects - (fn [objects] - (update-vals objects fix-deleted-font-shape)))) - -(defn fix-deleted-font-typography +(defn- fix-typography [typography] - (let [alternative-font-id (calculate-alternative-font-id (:font-family typography))] - (cond-> typography - (some? alternative-font-id) (assoc :font-id alternative-font-id)))) + (if-let [alternative-font-id (calculate-alternative-font-id (:font-family typography))] + (assoc typography :font-id alternative-font-id) + typography)) -(defn- generate-deleted-font-shape-changes +(defn- generate-page-changes [{:keys [objects id]}] - (sequence - (comp (map val) - (filter should-fix-deleted-font-shape?) - (map (fn [shape] - {:type :mod-obj - :id (:id shape) - :page-id id - :operations [{:type :set - :attr :content - :val (:content (fix-deleted-font-shape shape))} - {:type :set - :attr :position-data - :val nil}]}))) - objects)) + (reduce-kv (fn [changes shape-id shape] + (if (shape-has-invalid-font-family?? shape) + (conj changes {:type :mod-obj + :id shape-id + :page-id id + :operations [{:type :set + :attr :content + :val (fix-shape-content shape)} + {:type :set + :attr :position-data + :val nil}]}) + changes)) + [] + objects)) -(defn- generate-deleted-font-components-changes +(defn- generate-library-changes [fdata] - (sequence - (comp (map val) - (filter should-fix-deleted-font-component?) - (map (fn [component] - {:type :mod-component - :id (:id component) - :objects (-> (fix-deleted-font-component component) :objects)}))) - (:components fdata))) + (reduce-kv (fn [changes _ typography] + (if (has-invalid-font-family? typography) + (conj changes {:type :mod-typography + :typography (fix-typography typography)}) + changes)) + [] + (:typographies fdata))) -(defn- generate-deleted-font-typography-changes - [fdata] - (sequence - (comp (map val) - (filter has-invalid-font-family?) - (map (fn [typography] - {:type :mod-typography - :typography (fix-deleted-font-typography typography)}))) - (:typographies fdata))) - -(defn fix-deleted-fonts - [] - (ptk/reify ::fix-deleted-fonts +(defn fix-deleted-fonts-for-local-library + "Looks the file local library for deleted fonts and emit changes if + invalid but fixable typographyes found." + [file-id] + (ptk/reify ::fix-deleted-fonts-for-local-library ptk/WatchEvent (watch [it state _] - (let [fdata (dsh/lookup-file-data state) - pages (:pages-index fdata) - - shape-changes (mapcat generate-deleted-font-shape-changes (vals pages)) - components-changes (generate-deleted-font-components-changes fdata) - typography-changes (generate-deleted-font-typography-changes fdata) - changes (concat shape-changes - components-changes - typography-changes)] - (if (seq changes) + (let [fdata (dsh/lookup-file-data state file-id)] + (when-let [changes (-> (generate-library-changes fdata) + (not-empty))] (rx/of (dwc/commit-changes {:origin it - :redo-changes (vec changes) + :redo-changes changes :undo-changes [] - :save-undo? false})) - (rx/empty)))))) + :save-undo? false}))))))) + +;; FIXME: would be nice to not execute this code twice per page in the +;; same working session, maybe some local memoization can improve that + +(defn fix-deleted-fonts-for-page + [file-id page-id] + (ptk/reify ::fix-deleted-fonts-for-page + ptk/WatchEvent + (watch [it state _] + (let [page (dsh/lookup-page state file-id page-id)] + (when-let [changes (-> (generate-page-changes page) + (not-empty))] + (rx/of (dwc/commit-changes + {:origin it + :redo-changes changes + :undo-changes [] + :save-undo? false}))))))) diff --git a/frontend/src/app/main/data/workspace/pages.cljs b/frontend/src/app/main/data/workspace/pages.cljs index a9c9d3354e..5b91d10864 100644 --- a/frontend/src/app/main/data/workspace/pages.cljs +++ b/frontend/src/app/main/data/workspace/pages.cljs @@ -24,6 +24,7 @@ [app.main.data.helpers :as dsh] [app.main.data.persistence :as-alias dps] [app.main.data.workspace.drawing :as dwd] + [app.main.data.workspace.fix-deleted-fonts :as fdf] [app.main.data.workspace.layout :as layout] [app.main.data.workspace.libraries :as dwl] [app.main.data.workspace.thumbnails :as dwth] @@ -104,6 +105,7 @@ (if (dsh/lookup-page state file-id page-id) (rx/concat (rx/of (initialize-page* file-id page-id) + (fdf/fix-deleted-fonts-for-page file-id page-id) (dwth/watch-state-changes file-id page-id) (dwl/watch-component-changes)) (let [profile (:profile state)