From bf719b587f3e6dbbe2efe000d5fe6d51d0de675a Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 25 Jun 2025 19:17:58 +0200 Subject: [PATCH 1/8] :sparkles: Add better shadow cleaning migration --- common/src/app/common/files/migrations.cljc | 45 ++++++++++++--------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/common/src/app/common/files/migrations.cljc b/common/src/app/common/files/migrations.cljc index 11a99507fb..96352618ba 100644 --- a/common/src/app/common/files/migrations.cljc +++ b/common/src/app/common/files/migrations.cljc @@ -1337,32 +1337,37 @@ (update :pages-index d/update-vals update-container) (d/update-when :components d/update-vals update-container)))) -(defmethod migrate-data "0004-clean-shadow-and-colors" +(defmethod migrate-data "0004-clean-shadow-color" [data _] - (letfn [(clean-shadow [shadow] - (update shadow :color (fn [color] - (let [ref-id (get color :id) - ref-file (get color :file-id)] - (-> (d/without-qualified color) - (select-keys [:opacity :color :gradient :image :ref-id :ref-file]) - (cond-> ref-id - (assoc :ref-id ref-id)) - (cond-> ref-file - (assoc :ref-file ref-file))))))) + (let [decode-color (sm/decoder types.color/schema:color sm/json-transformer) - (update-object [object] - (d/update-when object :shadow #(mapv clean-shadow %))) + clean-shadow-color + (fn [color] + (let [ref-id (get color :id) + ref-file (get color :file-id)] + (-> (d/without-qualified color) + (select-keys [:opacity :color :gradient :image :ref-id :ref-file]) + (cond-> ref-id + (assoc :ref-id ref-id)) + (cond-> ref-file + (assoc :ref-file ref-file)) + (decode-color)))) - (update-container [container] - (d/update-when container :objects d/update-vals update-object)) + clean-shadow + (fn [shadow] + (update shadow :color clean-shadow-color)) - (clean-library-color [color] - (dissoc color :file-id))] + update-object + (fn [object] + (d/update-when object :shadow #(mapv clean-shadow %))) + + update-container + (fn [container] + (d/update-when container :objects d/update-vals update-object))] (-> data (update :pages-index d/update-vals update-container) - (d/update-when :components d/update-vals update-container) - (d/update-when :colors d/update-vals clean-library-color)))) + (d/update-when :components d/update-vals update-container)))) (defmethod migrate-data "0005-deprecate-image-type" [data _] @@ -1554,7 +1559,7 @@ "0002-clean-shape-interactions" "0003-fix-root-shape" "0003-convert-path-content" - "0004-clean-shadow-and-colors" + "0004-clean-shadow-color" "0005-deprecate-image-type" "0006-fix-old-texts-fills" "0007-clear-invalid-strokes-and-fills-v2" From b52e2fa6819fbbebd30e4a94f5240d7645d86050 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 25 Jun 2025 19:24:18 +0200 Subject: [PATCH 2/8] :bug: Add missing version field on get-team-shared-files internal query --- backend/src/app/rpc/commands/files.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/app/rpc/commands/files.clj b/backend/src/app/rpc/commands/files.clj index cfd9b4a947..83140780fd 100644 --- a/backend/src/app/rpc/commands/files.clj +++ b/backend/src/app/rpc/commands/files.clj @@ -560,6 +560,7 @@ f.created_at, f.modified_at, f.name, + f.version, f.is_shared, ft.media_id, p.team_id From 207974fe6c99cb1118797784bc757a0aa40b9cfb Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 25 Jun 2025 19:26:43 +0200 Subject: [PATCH 3/8] :sparkles: Add minor improvement to color cleaning migration --- common/src/app/common/files/migrations.cljc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/common/src/app/common/files/migrations.cljc b/common/src/app/common/files/migrations.cljc index 96352618ba..800c8552f7 100644 --- a/common/src/app/common/files/migrations.cljc +++ b/common/src/app/common/files/migrations.cljc @@ -1485,7 +1485,7 @@ (update :pages-index d/update-vals update-container) (d/update-when :components d/update-vals update-container)))) -(defmethod migrate-data "0008-fix-library-colors-v3" +(defmethod migrate-data "0008-fix-library-colors-v4" [data _] (letfn [(clear-color-opacity [color] (if (and (contains? color :opacity) @@ -1496,7 +1496,8 @@ (clear-color [color] (-> color (select-keys types.color/library-color-attrs) - (clear-color-opacity)))] + (clear-color-opacity) + (d/without-nils)))] (d/update-when data :colors d/update-vals clear-color))) @@ -1563,4 +1564,4 @@ "0005-deprecate-image-type" "0006-fix-old-texts-fills" "0007-clear-invalid-strokes-and-fills-v2" - "0008-fix-library-colors-v3"])) + "0008-fix-library-colors-v4"])) From 26239a15f2ddb540d46455f74eb66491c3c77249 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 25 Jun 2025 20:13:35 +0200 Subject: [PATCH 4/8] :paperclip: Add missing changes on lost-colors fix script --- backend/src/app/srepl/fixes/lost_colors.clj | 63 +++++++++++++-------- 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/backend/src/app/srepl/fixes/lost_colors.clj b/backend/src/app/srepl/fixes/lost_colors.clj index 6ea8bb23ae..d1aea9a1d1 100644 --- a/backend/src/app/srepl/fixes/lost_colors.clj +++ b/backend/src/app/srepl/fixes/lost_colors.clj @@ -23,7 +23,7 @@ AND label IS NOT NULL AND data IS NOT NULL ORDER BY created_at DESC - LIMIT 1") + LIMIT 2") (defn get-affected-migration [conn file-id] @@ -33,11 +33,12 @@ (defn get-last-valid-snapshot [conn migration] - (when-let [snapshot (db/exec-one! conn [sql:get-matching-snapshot - (:file-id migration) - (:created-at migration)])] - (let [snapshot (assoc snapshot :id (:file-id snapshot))] - (bfc/decode-file h/*system* snapshot)))) + (let [[snapshot] (db/exec! conn [sql:get-matching-snapshot + (:file-id migration) + (:created-at migration)])] + (when snapshot + (let [snapshot (assoc snapshot :id (:file-id snapshot))] + (bfc/decode-file h/*system* snapshot))))) (defn restore-color [{:keys [data] :as snapshot} color] @@ -47,25 +48,41 @@ (defn restore-missing-colors [{:keys [id] :as file} & _opts] - (when-let [colors (-> file :data :colors not-empty)] - (when-let [migration (get-affected-migration h/*system* id)] - (when-let [snapshot (get-last-valid-snapshot h/*system* migration)] - (let [colors (reduce-kv (fn [colors color-id color] - (if-let [result (restore-color snapshot color)] - (do - (l/inf :hint "restored color" :file-id (str id) :color-id (str color-id)) - (assoc colors color-id result)) - (do - (l/wrn :hint "ignoring color" :file-id (str id) :color (pr-str color)) - colors))) - colors - colors) - file (-> file - (update :data assoc :colors colors) - (update :migrations disj "0008-fix-library-colors-v2"))] + (l/inf :hint "process file" :file-id (str id) :name (:name file) :has-colors (-> file :data :colors not-empty boolean)) + (if-let [colors (-> file :data :colors not-empty)] + (let [migration (get-affected-migration h/*system* id)] + (if-let [snapshot (get-last-valid-snapshot h/*system* migration)] + (do + (l/inf :hint "using snapshot" :snapshot (:label snapshot)) + (let [colors (reduce-kv (fn [colors color-id color] + (if-let [result (restore-color snapshot color)] + (do + (l/inf :hint "restored color" :file-id (str id) :color-id (str color-id)) + (assoc colors color-id result)) + (do + (l/wrn :hint "ignoring color" :file-id (str id) :color (pr-str color)) + colors))) + colors + colors) + file (-> file + (update :data assoc :colors colors) + (update :migrations disj "0008-fix-library-colors-v2"))] + (db/delete! h/*system* :file-migration + {:name "0008-fix-library-colors-v2" + :file-id (:id file)}) + file)) + + (do (db/delete! h/*system* :file-migration {:name "0008-fix-library-colors-v2" :file-id (:id file)}) + nil))) + + (do + (db/delete! h/*system* :file-migration + {:name "0008-fix-library-colors-v2" + :file-id (:id file)}) + nil))) + - file))))) From 77ef26b2076eacbf24b5ad763e31a547f7e6458e Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 25 Jun 2025 20:14:05 +0200 Subject: [PATCH 5/8] :paperclip: Add srepl script for validate file schema --- backend/src/app/srepl/main.clj | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/backend/src/app/srepl/main.clj b/backend/src/app/srepl/main.clj index 915499bc52..2566fd5f0c 100644 --- a/backend/src/app/srepl/main.clj +++ b/backend/src/app/srepl/main.clj @@ -17,6 +17,7 @@ [app.common.files.validate :as cfv] [app.common.logging :as l] [app.common.pprint :as p] + [app.common.schema :as sm] [app.common.spec :as us] [app.common.uuid :as uuid] [app.config :as cf] @@ -395,6 +396,22 @@ libs (bfc/get-resolved-file-libraries system file)] (cfv/validate-file file libs)))))) +(defn validate-file-schema + "Validate structure, referencial integrity and semantic coherence of + all contents of a file. Returns a list of errors." + [file-id] + (let [file-id (h/parse-uuid file-id)] + (db/tx-run! (assoc main/system ::db/rollback true) + (fn [system] + (try + (let [file (bfc/get-file system file-id)] + (cfv/validate-file-schema! file) + (println "OK")) + (catch Exception cause + (if-let [explain (-> cause ex-data ::sm/explain)] + (println (sm/humanize-explain explain)) + (ex/print-throwable cause)))))))) + (defn repair-file! "Repair the list of errors detected by validation." [file-id & {:keys [rollback?] :or {rollback? true} :as opts}] From 17bfed137ce02bc9c35c58eae2f404b21cc3c42b Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 25 Jun 2025 20:14:33 +0200 Subject: [PATCH 6/8] :paperclip: Add better defaults for text processing on old migrations --- common/src/app/common/files/migrations.cljc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/src/app/common/files/migrations.cljc b/common/src/app/common/files/migrations.cljc index 800c8552f7..564fe4fc55 100644 --- a/common/src/app/common/files/migrations.cljc +++ b/common/src/app/common/files/migrations.cljc @@ -855,7 +855,7 @@ (update-object [object] (if (cfh/text-shape? object) - (update object :content #(txt/transform-nodes identity update-text-node %)) + (update object :content #(txt/transform-nodes txt/is-content-node? update-text-node %)) object)) (update-container [container] @@ -1105,7 +1105,7 @@ ;; The text shape also can has fills on the text ;; fragments so we need to fix fills there (cond-> (cfh/text-shape? object) - (update :content (partial txt/transform-nodes identity fix-fills))))) + (update :content (partial txt/transform-nodes txt/is-content-node? fix-fills))))) (update-container [container] (d/update-when container :objects d/update-vals update-object))] @@ -1407,7 +1407,7 @@ (update-object [object] (if (cfh/text-shape? object) - (update object :content (partial txt/transform-nodes identity fix-fills)) + (update object :content (partial txt/transform-nodes txt/is-content-node? fix-fills)) object)) (update-container [container] From 00390a1349019aa2284915c183c1a83f29f82229 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 25 Jun 2025 20:25:03 +0200 Subject: [PATCH 7/8] :bug: Add correct is-text-node? predicate to text processing methods --- common/src/app/common/text.cljc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/src/app/common/text.cljc b/common/src/app/common/text.cljc index 84cc27b218..9e635f6a03 100644 --- a/common/src/app/common/text.cljc +++ b/common/src/app/common/text.cljc @@ -130,8 +130,8 @@ (defn is-text-node? [node] - (and (string? (:text node)) - (not= (:text node) ""))) + (and (nil? (:type node)) + (string? (:text node)))) (defn is-paragraph-set-node? [node] From 3831b3034e1402391afe0c459282db69b0ec5536 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 26 Jun 2025 08:55:09 +0200 Subject: [PATCH 8/8] :bug: Fix boolean shape migration that causes issues on import --- common/src/app/common/files/migrations.cljc | 25 ++++++++++----------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/common/src/app/common/files/migrations.cljc b/common/src/app/common/files/migrations.cljc index 564fe4fc55..94949fbc1e 100644 --- a/common/src/app/common/files/migrations.cljc +++ b/common/src/app/common/files/migrations.cljc @@ -1277,22 +1277,21 @@ (update :pages-index d/update-vals update-container) (d/update-when :components d/update-vals update-container)))) -(defmethod migrate-data "0002-normalize-bool-content" +(defmethod migrate-data "0002-normalize-bool-content-v2" [data _] (letfn [(update-object [object] - ;; NOTE: we still preserve the previous value for possible - ;; rollback, we still need to perform an other migration - ;; for properly delete the bool-content prop from shapes - ;; once the know the migration was OK - (if (and (cfh/bool-shape? object) - (not (contains? object :content))) - (if-let [content (:bool-content object)] - (assoc object :content content) - object) + (if (cfh/bool-shape? object) + (if (contains? object :content) + (dissoc object :bool-content) + (let [content (:bool-content object)] + (-> object + (assoc :content content) + (dissoc :bool-content)))) + (dissoc object :bool-content :bool-type))) (update-container [container] - (d/update-when container :objects update-vals update-object))] + (d/update-when container :objects d/update-vals update-object))] (-> data (update :pages-index d/update-vals update-container) @@ -1331,7 +1330,7 @@ object)) (update-container [container] - (d/update-when container :objects update-vals update-object))] + (d/update-when container :objects d/update-vals update-object))] (-> data (update :pages-index d/update-vals update-container) @@ -1556,7 +1555,7 @@ "legacy-66" "legacy-67" "0001-remove-tokens-from-groups" - "0002-normalize-bool-content" + "0002-normalize-bool-content-v2" "0002-clean-shape-interactions" "0003-fix-root-shape" "0003-convert-path-content"