diff --git a/backend/src/app/rpc/commands/files.clj b/backend/src/app/rpc/commands/files.clj index cfd9b4a947..83140780fd 100644 --- a/backend/src/app/rpc/commands/files.clj +++ b/backend/src/app/rpc/commands/files.clj @@ -560,6 +560,7 @@ f.created_at, f.modified_at, f.name, + f.version, f.is_shared, ft.media_id, p.team_id diff --git a/backend/src/app/srepl/fixes/lost_colors.clj b/backend/src/app/srepl/fixes/lost_colors.clj index 6ea8bb23ae..d1aea9a1d1 100644 --- a/backend/src/app/srepl/fixes/lost_colors.clj +++ b/backend/src/app/srepl/fixes/lost_colors.clj @@ -23,7 +23,7 @@ AND label IS NOT NULL AND data IS NOT NULL ORDER BY created_at DESC - LIMIT 1") + LIMIT 2") (defn get-affected-migration [conn file-id] @@ -33,11 +33,12 @@ (defn get-last-valid-snapshot [conn migration] - (when-let [snapshot (db/exec-one! conn [sql:get-matching-snapshot - (:file-id migration) - (:created-at migration)])] - (let [snapshot (assoc snapshot :id (:file-id snapshot))] - (bfc/decode-file h/*system* snapshot)))) + (let [[snapshot] (db/exec! conn [sql:get-matching-snapshot + (:file-id migration) + (:created-at migration)])] + (when snapshot + (let [snapshot (assoc snapshot :id (:file-id snapshot))] + (bfc/decode-file h/*system* snapshot))))) (defn restore-color [{:keys [data] :as snapshot} color] @@ -47,25 +48,41 @@ (defn restore-missing-colors [{:keys [id] :as file} & _opts] - (when-let [colors (-> file :data :colors not-empty)] - (when-let [migration (get-affected-migration h/*system* id)] - (when-let [snapshot (get-last-valid-snapshot h/*system* migration)] - (let [colors (reduce-kv (fn [colors color-id color] - (if-let [result (restore-color snapshot color)] - (do - (l/inf :hint "restored color" :file-id (str id) :color-id (str color-id)) - (assoc colors color-id result)) - (do - (l/wrn :hint "ignoring color" :file-id (str id) :color (pr-str color)) - colors))) - colors - colors) - file (-> file - (update :data assoc :colors colors) - (update :migrations disj "0008-fix-library-colors-v2"))] + (l/inf :hint "process file" :file-id (str id) :name (:name file) :has-colors (-> file :data :colors not-empty boolean)) + (if-let [colors (-> file :data :colors not-empty)] + (let [migration (get-affected-migration h/*system* id)] + (if-let [snapshot (get-last-valid-snapshot h/*system* migration)] + (do + (l/inf :hint "using snapshot" :snapshot (:label snapshot)) + (let [colors (reduce-kv (fn [colors color-id color] + (if-let [result (restore-color snapshot color)] + (do + (l/inf :hint "restored color" :file-id (str id) :color-id (str color-id)) + (assoc colors color-id result)) + (do + (l/wrn :hint "ignoring color" :file-id (str id) :color (pr-str color)) + colors))) + colors + colors) + file (-> file + (update :data assoc :colors colors) + (update :migrations disj "0008-fix-library-colors-v2"))] + (db/delete! h/*system* :file-migration + {:name "0008-fix-library-colors-v2" + :file-id (:id file)}) + file)) + + (do (db/delete! h/*system* :file-migration {:name "0008-fix-library-colors-v2" :file-id (:id file)}) + nil))) + + (do + (db/delete! h/*system* :file-migration + {:name "0008-fix-library-colors-v2" + :file-id (:id file)}) + nil))) + - file))))) diff --git a/backend/src/app/srepl/main.clj b/backend/src/app/srepl/main.clj index 915499bc52..2566fd5f0c 100644 --- a/backend/src/app/srepl/main.clj +++ b/backend/src/app/srepl/main.clj @@ -17,6 +17,7 @@ [app.common.files.validate :as cfv] [app.common.logging :as l] [app.common.pprint :as p] + [app.common.schema :as sm] [app.common.spec :as us] [app.common.uuid :as uuid] [app.config :as cf] @@ -395,6 +396,22 @@ libs (bfc/get-resolved-file-libraries system file)] (cfv/validate-file file libs)))))) +(defn validate-file-schema + "Validate structure, referencial integrity and semantic coherence of + all contents of a file. Returns a list of errors." + [file-id] + (let [file-id (h/parse-uuid file-id)] + (db/tx-run! (assoc main/system ::db/rollback true) + (fn [system] + (try + (let [file (bfc/get-file system file-id)] + (cfv/validate-file-schema! file) + (println "OK")) + (catch Exception cause + (if-let [explain (-> cause ex-data ::sm/explain)] + (println (sm/humanize-explain explain)) + (ex/print-throwable cause)))))))) + (defn repair-file! "Repair the list of errors detected by validation." [file-id & {:keys [rollback?] :or {rollback? true} :as opts}] diff --git a/common/src/app/common/files/migrations.cljc b/common/src/app/common/files/migrations.cljc index 324254902d..8255fd18d5 100644 --- a/common/src/app/common/files/migrations.cljc +++ b/common/src/app/common/files/migrations.cljc @@ -856,7 +856,7 @@ (update-object [object] (if (cfh/text-shape? object) - (update object :content #(txt/transform-nodes identity update-text-node %)) + (update object :content #(txt/transform-nodes txt/is-content-node? update-text-node %)) object)) (update-container [container] @@ -1106,7 +1106,7 @@ ;; The text shape also can has fills on the text ;; fragments so we need to fix fills there (cond-> (cfh/text-shape? object) - (update :content (partial txt/transform-nodes identity fix-fills))))) + (update :content (partial txt/transform-nodes txt/is-content-node? fix-fills))))) (update-container [container] (d/update-when container :objects d/update-vals update-object))] @@ -1278,22 +1278,21 @@ (update :pages-index d/update-vals update-container) (d/update-when :components d/update-vals update-container)))) -(defmethod migrate-data "0002-normalize-bool-content" +(defmethod migrate-data "0002-normalize-bool-content-v2" [data _] (letfn [(update-object [object] - ;; NOTE: we still preserve the previous value for possible - ;; rollback, we still need to perform an other migration - ;; for properly delete the bool-content prop from shapes - ;; once the know the migration was OK - (if (and (cfh/bool-shape? object) - (not (contains? object :content))) - (if-let [content (:bool-content object)] - (assoc object :content content) - object) + (if (cfh/bool-shape? object) + (if (contains? object :content) + (dissoc object :bool-content) + (let [content (:bool-content object)] + (-> object + (assoc :content content) + (dissoc :bool-content)))) + (dissoc object :bool-content :bool-type))) (update-container [container] - (d/update-when container :objects update-vals update-object))] + (d/update-when container :objects d/update-vals update-object))] (-> data (update :pages-index d/update-vals update-container) @@ -1332,38 +1331,43 @@ object)) (update-container [container] - (d/update-when container :objects update-vals update-object))] + (d/update-when container :objects d/update-vals update-object))] (-> data (update :pages-index d/update-vals update-container) (d/update-when :components d/update-vals update-container)))) -(defmethod migrate-data "0004-clean-shadow-and-colors" +(defmethod migrate-data "0004-clean-shadow-color" [data _] - (letfn [(clean-shadow [shadow] - (update shadow :color (fn [color] - (let [ref-id (get color :id) - ref-file (get color :file-id)] - (-> (d/without-qualified color) - (select-keys [:opacity :color :gradient :image :ref-id :ref-file]) - (cond-> ref-id - (assoc :ref-id ref-id)) - (cond-> ref-file - (assoc :ref-file ref-file))))))) + (let [decode-color (sm/decoder types.color/schema:color sm/json-transformer) - (update-object [object] - (d/update-when object :shadow #(mapv clean-shadow %))) + clean-shadow-color + (fn [color] + (let [ref-id (get color :id) + ref-file (get color :file-id)] + (-> (d/without-qualified color) + (select-keys [:opacity :color :gradient :image :ref-id :ref-file]) + (cond-> ref-id + (assoc :ref-id ref-id)) + (cond-> ref-file + (assoc :ref-file ref-file)) + (decode-color)))) - (update-container [container] - (d/update-when container :objects d/update-vals update-object)) + clean-shadow + (fn [shadow] + (update shadow :color clean-shadow-color)) - (clean-library-color [color] - (dissoc color :file-id))] + update-object + (fn [object] + (d/update-when object :shadow #(mapv clean-shadow %))) + + update-container + (fn [container] + (d/update-when container :objects d/update-vals update-object))] (-> data (update :pages-index d/update-vals update-container) - (d/update-when :components d/update-vals update-container) - (d/update-when :colors d/update-vals clean-library-color)))) + (d/update-when :components d/update-vals update-container)))) (defmethod migrate-data "0005-deprecate-image-type" [data _] @@ -1403,7 +1407,7 @@ (update-object [object] (if (cfh/text-shape? object) - (update object :content (partial txt/transform-nodes identity fix-fills)) + (update object :content (partial txt/transform-nodes txt/is-content-node? fix-fills)) object)) (update-container [container] @@ -1481,7 +1485,7 @@ (update :pages-index d/update-vals update-container) (d/update-when :components d/update-vals update-container)))) -(defmethod migrate-data "0008-fix-library-colors-v3" +(defmethod migrate-data "0008-fix-library-colors-v4" [data _] (letfn [(clear-color-opacity [color] (if (and (contains? color :opacity) @@ -1492,7 +1496,8 @@ (clear-color [color] (-> color (select-keys types.color/library-color-attrs) - (clear-color-opacity)))] + (clear-color-opacity) + (d/without-nils)))] (d/update-when data :colors d/update-vals clear-color))) @@ -1576,13 +1581,13 @@ "legacy-66" "legacy-67" "0001-remove-tokens-from-groups" - "0002-normalize-bool-content" + "0002-normalize-bool-content-v2" "0002-clean-shape-interactions" "0003-fix-root-shape" "0003-convert-path-content" - "0004-clean-shadow-and-colors" + "0004-clean-shadow-color" "0005-deprecate-image-type" "0006-fix-old-texts-fills" "0007-clear-invalid-strokes-and-fills-v2" - "0008-fix-library-colors-v3" + "0008-fix-library-colors-v4" "0009-add-partial-text-touched-flags"])) diff --git a/common/src/app/common/text.cljc b/common/src/app/common/text.cljc index 84cc27b218..9e635f6a03 100644 --- a/common/src/app/common/text.cljc +++ b/common/src/app/common/text.cljc @@ -130,8 +130,8 @@ (defn is-text-node? [node] - (and (string? (:text node)) - (not= (:text node) ""))) + (and (nil? (:type node)) + (string? (:text node)))) (defn is-paragraph-set-node? [node]