⚡ Add several optimizations for fonts zip download

Mainly prevent hold the whole zip in memory and uses an
unified response type, leavin frontend fetching the blob
data from the assets/storage subsystem.

backend/src/app/http/sse.clj

@@ -53,6 +53,7 @@
      ::yres/status 200
      ::yres/body (yres/stream-body
                   (fn [_ output]
+
                     (let [channel  (sp/chan :buf buf :xf (keep encode))
                           listener (events/spawn-listener
                                     channel

backend/src/app/media.clj

@@ -54,7 +54,7 @@
    [:path ::fs/path]
    [:mtype {:optional true} ::sm/text]])
 
-(def ^:private check-input
+(def check-input
   (sm/check-fn schema:input))
 
 (defn validate-media-type!

backend/src/app/rpc.clj

@@ -73,9 +73,13 @@
                                      (if (nil? result)
                                        204
                                        200))
-                         headers (cond-> (::http/headers mdata {})
+
-                                   (yres/stream-body? result)
+                         headers (::http/headers mdata {})
+                         headers (cond-> headers
+                                   (and (yres/stream-body? result)
+                                        (not (contains? headers "content-type")))
                                    (assoc "content-type" "application/octet-stream"))]
+
                      {::yres/status  status
                       ::yres/headers headers
                       ::yres/body    result}))]

backend/src/app/rpc/commands/fonts.clj

@@ -16,6 +16,7 @@
    [app.db :as db]
    [app.db.sql :as-alias sql]
    [app.features.logical-deletion :as ldel]
+   [app.http :as-alias http]
    [app.loggers.audit :as-alias audit]
    [app.loggers.webhooks :as-alias webhooks]
    [app.media :as media]
@@ -32,7 +33,6 @@
    [app.util.services :as sv]
    [datoteka.io :as io])
   (:import
-   java.io.ByteArrayOutputStream
    java.io.InputStream
    java.io.OutputStream
    java.io.SequenceInputStream
@@ -303,90 +303,95 @@
 
 ;; --- DOWNLOAD FONT
 
+(defn- make-temporal-storage-object
+  [cfg profile-id content]
+  (let [storage (sto/resolve cfg)
+        content (media/check-input content)
+        hash    (sto/calculate-hash (:path content))
+        data    (-> (sto/content (:path content))
+                    (sto/wrap-with-hash hash))
+        mtype   (:mtype content "application/octet-stream")
+        content {::sto/content data
+                 ::sto/deduplicate? true
+                 ::sto/touched-at (ct/in-future {:minutes 30})
+                 :profile-id profile-id
+                 :content-type mtype
+                 :bucket "tempfile"}]
+
+    (sto/put-object! storage content)))
+
+(defn- make-variant-filename
+  [v mtype]
+  (str (:font-family v) "-" (:font-weight v)
+       (when-not (= "normal" (:font-style v)) (str "-" (:font-style v)))
+       (cmedia/mtype->extension mtype)))
+
 (def ^:private schema:download-font
   [:map {:title "download-font"}
    [:id ::sm/uuid]])
 
 (sv/defmethod ::download-font
-  {::doc/added "1.18"
+  "Download the font file. Returns a http redirect to the asset resource uri."
+  {::doc/added "2.15"
    ::sm/params schema:download-font}
   [{:keys [::sto/storage ::db/pool] :as cfg} {:keys [::rpc/profile-id id]}]
-  (dm/with-open [conn (db/open pool)]
+  (let [variant (db/get pool :team-font-variant {:id id})]
-    (let [variant (db/get conn :team-font-variant
+    (teams/check-read-permissions! pool profile-id (:team-id variant))
-                          {:id id
-                           :deleted-at nil})]
-      (when-not variant
-        (ex/raise :type :not-found
-                  :code :object-not-found))
 
-      (teams/check-read-permissions! conn profile-id (:team-id variant))
+    ;; Try to get the best available font format (prefer TTF for broader compatibility).
+    (let [media-id (or (:ttf-file-id variant)
+                       (:otf-file-id variant)
+                       (:woff2-file-id variant)
+                       (:woff1-file-id variant))
+          sobj     (sto/get-object storage media-id)
+          mtype    (-> sobj meta :content-type)]
 
-      ;; Try to get the best available font format (prefer TTF for broader compatibility).
+      {:id (:id sobj)
-      (let [file-id (or (:ttf-file-id variant)
+       :uri (files/resolve-public-uri (:id sobj))
-                        (:otf-file-id variant)
+       :name (make-variant-filename variant mtype)})))
-                        (:woff2-file-id variant)
-                        (:woff1-file-id variant))]
-        (when-not file-id
-          (ex/raise :type :not-found
-                    :code :font-file-not-found))
-
-        (let [font-obj (sto/get-object storage file-id)
-              font-bytes (sto/get-object-bytes storage font-obj)]
-          (when-not font-obj
-            (ex/raise :type :not-found
-                      :code :font-file-not-found))
-
-          ;; Return base64-encoded string and mime-type for transit serialization.
-          (let [data  (.encodeToString (java.util.Base64/getEncoder) font-bytes)
-                mtype (or (:content-type font-obj) (-> font-obj meta :content-type) "application/octet-stream")]
-            {:data data :mtype mtype}))))))
 
 (def ^:private schema:download-font-family
   [:map {:title "download-font-family"}
    [:font-id ::sm/uuid]])
 
 (sv/defmethod ::download-font-family
-  {::doc/added "1.18"
+  "Download the entire font family as a zip file. Returns the zip
+  bytes on the body, without encoding it on transit or json."
+  {::doc/added "2.15"
    ::sm/params schema:download-font-family}
   [{:keys [::sto/storage ::db/pool] :as cfg} {:keys [::rpc/profile-id font-id]}]
-  (dm/with-open [conn (db/open pool)]
+  (let [variants (db/query pool :team-font-variant
-    (let [variants (db/query conn :team-font-variant
+                           {:font-id font-id
-                             {:font-id font-id
+                            :deleted-at nil})]
-                              :deleted-at nil})]
-      (when-not (seq variants)
-        (ex/raise :type :not-found
-                  :code :object-not-found))
 
-      (teams/check-read-permissions! conn profile-id (:team-id (first variants)))
+    (when-not (seq variants)
+      (ex/raise :type :not-found
+                :code :object-not-found))
 
-      (let [entries
+    (teams/check-read-permissions! pool profile-id (:team-id (first variants)))
-            (->> variants
-                 (map (fn [v]
-                        (let [file-id (or (:ttf-file-id v)
-                                          (:otf-file-id v)
-                                          (:woff2-file-id v)
-                                          (:woff1-file-id v))]
-                          (when-not file-id
-                            (ex/raise :type :not-found :code :font-file-not-found))
 
-                          (let [sobj (sto/get-object storage file-id)
+    (let [tempfile (tmp/tempfile :suffix ".zip")
-                                bytes (sto/get-object-bytes storage sobj)
+          ffamily  (-> variants first :font-family)]
-                                mtype (or (:content-type sobj) (-> sobj meta :content-type) "application/octet-stream")
-                                ext (cmedia/mtype->extension mtype)
-                                name (str (:font-family v) "-" (:font-weight v)
-                                          (when-not (= "normal" (:font-style v)) (str "-" (:font-style v)))
-                                          (or ext ""))]
-                            {:name name :bytes bytes})))))]
 
-        ;; Build zip in memory.
+      (with-open [^OutputStream output (io/output-stream tempfile)
-        (let [baos (ByteArrayOutputStream.)
+                  ^OutputStream output (ZipOutputStream. output)]
-              zos  (ZipOutputStream. baos)]
+        (doseq [v variants]
-          (doseq [{:keys [name bytes]} entries]
+          (let [media-id (or (:ttf-file-id v)
-            (let [entry (ZipEntry. name)]
+                             (:otf-file-id v)
-              (.putNextEntry zos entry)
+                             (:woff2-file-id v)
-              (.write zos ^bytes bytes)
+                             (:woff1-file-id v))
-              (.closeEntry zos)))
+                sobj     (sto/get-object storage media-id)
-          (.close zos)
+                mtype    (-> sobj meta :content-type)
-          (let [zip-bytes (.toByteArray baos)
+                name     (make-variant-filename v mtype)]
-                data (.encodeToString (java.util.Base64/getEncoder) zip-bytes)]
+
-            {:data data :mtype "application/zip"}))))))
+            (with-open [input (sto/get-object-data storage sobj)]
+              (.putNextEntry ^ZipOutputStream output (ZipEntry. ^String name))
+              (io/copy input output :size (:size sobj))
+              (.closeEntry ^ZipOutputStream output)))))
+
+      (let [{:keys [id] :as sobj} (make-temporal-storage-object cfg profile-id
+                                                                {:mtype "application/zip"
+                                                                 :path tempfile})]
+        {:id id
+         :uri (files/resolve-public-uri id)
+         :name (str ffamily ".zip")}))))

frontend/src/app/main/ui/dashboard/fonts.cljs

@@ -7,6 +7,7 @@
 (ns app.main.ui.dashboard.fonts
   (:require-macros [app.main.style :as stl])
   (:require
+   [app.common.data :as d]
    [app.common.data.macros :as dm]
    [app.common.media :as cm]
    [app.common.uuid :as uuid]
@@ -22,10 +23,9 @@
    [app.main.ui.icons :as deprecated-icon]
    [app.main.ui.notifications.context-notification :refer [context-notification]]
    [app.util.dom :as dom]
+   [app.util.http :as http]
    [app.util.i18n :as i18n :refer [tr]]
    [app.util.keyboard :as kbd]
-   [app.util.timers :as tm]
-   [app.util.webapi :as wapi]
    [beicon.v2.core :as rx]
    [cuerdas.core :as str]
    [okulary.core :as l]
@@ -354,26 +354,18 @@
         (mf/use-fn
          (mf/deps variants)
          (fn [_event]
-           (let [variant (first variants)
+           (let [variant    (first variants)
                  variant-id (:id variant)
-                 multiple? (> (count variants) 1)
+                 multiple?  (> (count variants) 1)
-                 cmd (if multiple? :download-font-family :download-font)
+                 cmd        (if multiple? :download-font-family :download-font)
-                 params (if multiple? {:font-id font-id} {:id variant-id})]
+                 params     (if multiple? {:font-id font-id} {:id variant-id})]
              (->> (rp/cmd! cmd params)
-                  (rx/subs! (fn [font-data]
+                  (rx/mapcat (fn [{:keys [name uri]}]
-                              ;; font-data is base64-encoded or a map {:data :mtype}
+                               (->> (http/send! {:uri uri :method :get :response-type :blob})
-                              (let [b64 (if (string? font-data) font-data (:data font-data))
+                                    (rx/map :body)
-                                    default-mtype "application/octet-stream"
+                                    (rx/map (fn [blob] (d/vec2 name blob))))))
-                                    mtype (if (string? font-data) default-mtype (or (:mtype font-data) default-mtype))
+                  (rx/subs! (fn [[filename blob]]
-                                    binary-str (js/atob b64)
+                              (dom/trigger-download filename blob))
-                                    bytes (js/Uint8Array.
-                                           (for [i (range (.-length binary-str))]
-                                             (.charCodeAt binary-str i)))
-                                    blob (wapi/create-blob bytes mtype)
-                                    uri (wapi/create-uri blob)
-                                    name (:font-family font)]
-                                (dom/trigger-download-uri name mtype uri)
-                                (tm/schedule-on-idle #(wapi/revoke-uri uri))))
                             (fn [error]
                               (js/console.error "error downloading font" error)
                               (st/emit! (ntf/error (tr "errors.download-font")))))))))

frontend/src/app/util/dom.cljs

@@ -748,7 +748,11 @@
 
 (defn trigger-download
   [filename blob]
-  (trigger-download-uri filename (.-type ^js blob) (wapi/create-uri blob)))
+  (let [uri (wapi/create-uri blob)]
+    (try
+      (trigger-download-uri filename (.-type ^js blob) uri)
+      (finally
+        (wapi/revoke-uri uri)))))
 
 (defn event
   "Create an instance of DOM Event"

frontend/src/app/util/http.cljs

@@ -190,6 +190,11 @@
   [{:keys [status]}]
   (<= 400 status 499))
 
+(defn blob?
+  [^js v]
+  (when (some? v)
+    (instance? js/Blob v)))
+
 (defn as-promise
   [observable]
   (p/create