CalvinBackup/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2026-02-12 14:42:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

🎉 Add stacked exports.

Browse Source
This commit is contained in:
Andrey Antukh
2020-07-02 14:48:17 +02:00
committed by Hirunatan
parent a8d5cdc29f
commit 2fb4e72240
14 changed files with 549 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
exporter/package-lock.json generated
View File

@@ -261,6 +261,11 @@
"integrity": "sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=",
"dev": true
},
"bytes": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
"integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg=="
},
"cache-content-type": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/cache-content-type/-/cache-content-type-1.0.1.tgz",
@@ -344,8 +349,7 @@
"core-util-is": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
"dev": true
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
},
"create-ecdh": {
"version": "4.0.3",
@@ -697,11 +701,29 @@
}
}
},
"iconv-lite": {
"version": "0.4.24",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
"requires": {
"safer-buffer": ">= 2.1.2 < 3"
}
},
"ieee754": {
"version": "1.1.13",
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz",
"integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg=="
},
"immediate": {
"version": "3.0.6",
"resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz",
"integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps="
},
"inflation": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/inflation/-/inflation-2.0.0.tgz",
"integrity": "sha1-i0F+R8KPklpFEz2RTKH9OJEH8w8="
},
"inflight": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
@@ -724,8 +746,7 @@
"isarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
"dev": true
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
},
"isexe": {
"version": "2.0.0",
@@ -733,6 +754,41 @@
"integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=",
"dev": true
},
"jszip": {
"version": "3.5.0",
"resolved": "https://registry.npmjs.org/jszip/-/jszip-3.5.0.tgz",
"integrity": "sha512-WRtu7TPCmYePR1nazfrtuF216cIVon/3GWOvHS9QR5bIwSbnxtdpma6un3jyGGNhHsKCSzn5Ypk+EkDRvTGiFA==",
"requires": {
"lie": "~3.3.0",
"pako": "~1.0.2",
"readable-stream": "~2.3.6",
"set-immediate-shim": "~1.0.1"
},
"dependencies": {
"readable-stream": {
"version": "2.3.7",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
},
"string_decoder": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"requires": {
"safe-buffer": "~5.1.0"
}
}
}
},
"keygrip": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/keygrip/-/keygrip-1.1.0.tgz",
@@ -795,6 +851,14 @@
}
}
},
"lie": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz",
"integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==",
"requires": {
"immediate": "~3.0.5"
}
},
"md5.js": {
"version": "1.3.5",
"resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz",
@@ -993,8 +1057,7 @@
"pako": {
"version": "1.0.11",
"resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz",
"integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==",
"dev": true
"integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="
},
"parse-asn1": {
"version": "5.1.5",
@@ -1053,8 +1116,7 @@
"process-nextick-args": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==",
"dev": true
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"progress": {
"version": "2.0.3",
@@ -1190,6 +1252,17 @@
"safe-buffer": "^5.1.0"
}
},
"raw-body": {
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.1.tgz",
"integrity": "sha512-9WmIKF6mkvA0SLmA2Knm9+qj89e+j1zqgyn8aXGd7+nAduPoqgI9lO57SAZNn/Byzo5P7JhXTyg9PzaJbH73bA==",
"requires": {
"bytes": "3.1.0",
"http-errors": "1.7.3",
"iconv-lite": "0.4.24",
"unpipe": "1.0.0"
}
},
"readable-stream": {
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
@@ -1234,6 +1307,16 @@
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
},
"safer-buffer": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
},
"set-immediate-shim": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz",
"integrity": "sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E="
},
"setimmediate": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz",
@@ -1497,6 +1580,11 @@
"through": "^2.3.8"
}
},
"unpipe": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
"integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw="
},
"url": {
"version": "0.11.0",
"resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz",

3
exporter/package.json
View File

@@ -9,9 +9,12 @@
"author": "UXBOX LABS SL",
"license": "SEE LICENSE IN <LICENSE>",
"dependencies": {
"inflation": "^2.0.0",
"jszip": "^3.5.0",
"koa": "^2.13.0",
"puppeteer": "^4.0.1",
"puppeteer-cluster": "^0.21.0",
"raw-body": "^2.4.1",
"xregexp": "^4.3.0"
},
"devDependencies": {

3
exporter/shadow-cljs.edn
View File

@@ -4,7 +4,8 @@
[funcool/cuerdas "2020.03.26-3"]
[lambdaisland/glogi "1.0.63"]
[metosin/reitit-core "0.5.2"]
[com.cognitect/transit-cljs "0.8.264"]]
[com.cognitect/transit-cljs "0.8.264"]
[frankiesardo/linked "1.3.0"]]
:source-paths ["src" "../common"]
:nrepl {:port 3497}

33
exporter/src/app/browser.cljs
View File

@@ -15,11 +15,13 @@
(f page)))))
(defn emulate!
[page {:keys [viewport user-agent]
:or {user-agent USER-AGENT}}]
[page {:keys [viewport user-agent scale]
:or {user-agent USER-AGENT
scale 1}}]
(let [[width height] viewport]
(.emulate page #js {:viewport #js {:width width
:height height}
:height height
:deviceScaleFactor scale}
:userAgent user-agent})))
(defn navigate!
@@ -33,10 +35,20 @@
(.waitFor ^js page ms))
(defn screenshot
([page] (screenshot page nil))
([page {:keys [full-page?]
:or {full-page? true}}]
(.screenshot ^js page #js {:fullPage full-page? :omitBackground true})))
([frame] (screenshot frame nil))
([frame {:keys [full-page? omit-background?]
:or {full-page? false
omit-background? false}}]
(.screenshot ^js frame #js {:fullPage full-page?
:omitBackground omit-background?})))
(defn eval!
[frame f]
(.evaluate ^js frame f))
(defn select
[frame selector]
(.$ ^js frame selector))
(defn set-cookie!
[page {:keys [key value domain]}]
@@ -47,16 +59,15 @@
(defn start!
([] (start! nil))
([{:keys [concurrency concurrency-strategy]
:or {concurrency 2
concurrency-strategy :browser}}]
:or {concurrency 10
concurrency-strategy :incognito}}]
(let [ccst (case concurrency-strategy
:browser (.-CONCURRENCY_BROWSER ^js ppc/Cluster)
:incognito (.-CONCURRENCY_CONTEXT ^js ppc/Cluster)
:page (.-CONCURRENCY_PAGE ^js ppc/Cluster))
opts #js {:concurrency ccst
:maxConcurrency concurrency
:puppeteerOptions #js {:args #js ["--no-sandbox"
"--explicitly-allowed-ports=6000"]}}]
:puppeteerOptions #js {:args #js ["--no-sandbox"]}}]
(.launch ^js ppc/Cluster opts))))
(defn stop!

57
exporter/src/app/http.cljs
View File

@@ -3,13 +3,14 @@
[promesa.core :as p]
[lambdaisland.glogi :as log]
[app.browser :as bwr]
[app.http.screenshot :refer [bitmap-handler
page-handler]]
[app.http.bitmap-export :refer [bitmap-export-handler]]
[app.util.transit :as t]
[reitit.core :as r]
[cuerdas.core :as str]
["koa" :as koa]
["http" :as http])
["http" :as http]
["inflation" :as inflate]
["raw-body" :as raw-body])
(:import
goog.Uri))
@@ -26,11 +27,7 @@
[router ctx]
(let [uri (.parse Uri (unchecked-get ctx "originalUrl"))]
(when-let [match (r/match-by-path router (.getPath uri))]
(let [qparams (query-params uri)
params {:path (:path-params match) :query qparams}]
(assoc match
:params params
:query-params qparams)))))
(assoc match :query-params (query-params uri)))))
(defn- handle-error
[error request]
@@ -72,15 +69,33 @@
(transient {})
(js/Object.keys orig)))))
(def parse-body?
#{"POST" "PUT" "DELETE"})
(defn- parse-body
[ctx]
(let [headers (unchecked-get ctx "headers")
ctype (unchecked-get headers "content-type")]
(when (parse-body? (.-method ^js ctx))
(-> (inflate (.-req ^js ctx))
(raw-body #js {:limit "5mb" :encoding "utf8"})
(p/then (fn [data]
(cond-> data
(= ctype "application/transit+json")
(t/decode))))))))
(defn- wrap-handler
[f extra]
(fn [ctx]
(let [cookies (unchecked-get ctx "cookies")
headers (parse-headers ctx)
request (assoc extra
:ctx ctx
:headers headers
:cookies cookies)]
(p/let [cookies (unchecked-get ctx "cookies")
headers (parse-headers ctx)
body (parse-body ctx)
request (assoc extra
:method (str/lower (unchecked-get ctx "method"))
:body body
:ctx ctx
:headers headers
:cookies cookies)]
(-> (p/do! (f request))
(p/then (fn [rsp]
(when (map? rsp)
@@ -91,16 +106,20 @@
(def routes
[["/export"
["/bitmap" {:handler bitmap-handler}]
["/page" {:handler page-handler}]]])
["/bitmap" {:handler bitmap-export-handler}]]])
(defn- router-handler
[router]
(fn [{:keys [ctx] :as req}]
(fn [{:keys [ctx body] :as request}]
(let [route (match router ctx)
request (assoc req
params (merge {}
(:query-params route)
(:path-params route)
(when (map? body) body))
request (assoc request
:route route
:params (:params route))
:params params)
handler (get-in route [:data :handler])]
(if (and route handler)
(handler request)

120
exporter/src/app/http/bitmap_export.cljs Normal file
View File

@@ -0,0 +1,120 @@
(ns app.http.bitmap-export
(:require
[cuerdas.core :as str]
[app.browser :as bwr]
[app.config :as cfg]
[app.zipfile :as zip]
[lambdaisland.glogi :as log]
[cljs.spec.alpha :as s]
[promesa.core :as p]
[uxbox.common.exceptions :as exc :include-macros true]
[uxbox.common.data :as d]
[uxbox.common.pages :as cp]
[uxbox.common.spec :as us])
(:import
goog.Uri))
(defn- screenshot-object
[browser {:keys [page-id object-id token scale suffix]}]
(letfn [(handle [page]
(let [path (str "/render-object/" page-id "/" object-id)
uri (doto (Uri. (:public-uri cfg/config))
(.setPath "/")
(.setFragment path))
cookie {:domain (str (.getDomain uri)
":"
(.getPort uri))
:key "auth-token"
:value token}]
(log/info :uri (.toString uri))
(screenshot page (.toString uri) cookie)))
(screenshot [page uri cookie]
(p/do!
(bwr/emulate! page {:viewport [1920 1080]
:scale scale})
(bwr/set-cookie! page cookie)
(bwr/navigate! page uri)
(bwr/eval! page (js* "() => document.body.style.background = 'transparent'"))
(p/let [dom (bwr/select page "#screenshot")]
(bwr/screenshot dom {:omit-background? true
:type type}))))]
(bwr/exec! browser handle)))
(s/def ::name ::us/string)
(s/def ::page-id ::us/uuid)
(s/def ::object-id ::us/uuid)
(s/def ::scale ::us/number)
(s/def ::suffix ::us/string)
(s/def ::type ::us/keyword)
(s/def ::suffix string?)
(s/def ::scale number?)
(s/def ::export
(s/keys :req-un [::type ::suffix ::scale]))
(s/def ::exports (s/coll-of ::export :kind vector?))
(s/def ::bitmap-handler-params
(s/keys :req-un [::page-id ::object-id ::name ::exports]))
(declare handle-single-export)
(declare handle-multiple-export)
(defn bitmap-export-handler
[{:keys [params browser cookies] :as request}]
(let [{:keys [exports page-id object-id name]} (us/conform ::bitmap-handler-params params)
token (.get ^js cookies "auth-token")]
(case (count exports)
0 (exc/raise :type :validation :code :missing-exports)
1 (handle-single-export
request
(assoc (first exports)
:name name
:token token
:page-id page-id
:object-id object-id))
(handle-multiple-export
request
(->> (d/enumerate exports)
(map (fn [[index item]]
(assoc item
:name name
:index index
:token token
:page-id page-id
:object-id object-id))))))))
(defn perform-bitmap-export
[browser params]
(p/let [content (screenshot-object browser params)]
{:content content
:filename (str (str/slug (:name params))
(if (not (str/blank? (:suffix params "")))
(:suffix params "")
(let [index (:index params 0)]
(when (pos? index)
(str "-" (inc index)))))
".png")
:length (alength content)
:mime-type "image/png"}))
(defn handle-single-export
[{:keys [browser]} params]
(p/let [result (perform-bitmap-export browser params)]
{:status 200
:body (:content result)
:headers {"content-type" (:mime-type result)
"content-length" (:length result)}}))
(defn handle-multiple-export
[{:keys [browser]} exports]
(let [proms (map (partial perform-bitmap-export browser) exports)]
(-> (p/all proms)
(p/then (fn [results]
(reduce #(zip/add! %1 (:filename %2) (:content %2)) (zip/create) results)))
(p/then (fn [fzip]
{:status 200
:headers {"content-type" "application/zip"}
:body (.generateNodeStream ^js fzip)})))))

73
exporter/src/app/http/screenshot.cljs
View File

@@ -1,73 +0,0 @@
(ns app.http.screenshot
(:require
[app.browser :as bwr]
[app.config :as cfg]
[lambdaisland.glogi :as log]
[cljs.spec.alpha :as s]
[promesa.core :as p]
[uxbox.common.exceptions :as exc :include-macros true]
[uxbox.common.spec :as us])
(:import
goog.Uri))
(defn- load-and-screenshot
[page url cookie]
(p/do!
(bwr/emulate! page {:viewport [1920 1080]})
(bwr/set-cookie! page cookie)
(bwr/navigate! page url)
(bwr/sleep page 500)
(.evaluate page (js* "() => document.body.style.background = 'transparent'"))
;; (.screenshot ^js page #js {:omitBackground true :fullPage true})
(p/let [dom (.$ page "#screenshot")]
(.screenshot ^js dom #js {:omitBackground true}))))
(defn- take-screenshot
[browser {:keys [page-id object-id token]}]
(letfn [(on-browser [page]
(let [path (str "/render-object/" page-id "/" object-id)
uri (doto (Uri. (:public-uri cfg/config))
(.setPath "/")
(.setFragment path))
cookie {:domain (str (.getDomain uri)
":"
(.getPort uri))
:key "auth-token"
:value token}]
(log/info :uri (.toString uri))
(load-and-screenshot page (.toString uri) cookie)))]
(bwr/exec! browser on-browser)))
(s/def ::page-id ::us/uuid)
(s/def ::object-id ::us/uuid)
(s/def ::bitmap-handler-params
(s/keys :req-un [::page-id ::object-id]))
(defn bitmap-handler
[{:keys [params browser cookies] :as request}]
(let [params (us/conform ::bitmap-handler-params (:query params))
token (.get ^js cookies "auth-token")]
(-> (take-screenshot browser {:page-id (:page-id params)
:object-id (:object-id params)
:token token})
(p/then (fn [result]
{:status 200
:body result
:headers {"content-type" "image/png"
"content-length" (alength result)}})))))
(defn page-handler
[{:keys [params browser] :as request}]
(letfn [(screenshot [page uri]
(p/do!
(bwr/emulate! page {:viewport [1920 1080]})
(bwr/navigate! page uri)
(bwr/sleep page 500)
;; (.evaluate page (js* "() => document.body.style.background = 'transparent'"))
(.screenshot ^js page #js {:omitBackground false})))]
(p/let [uri (get-in params [:query :uri])
sht (bwr/exec! browser #(screenshot % uri))]
{:status 200
:body sht
:headers {"content-type" "image/png"
"content-length" (alength sht)}})))

13
exporter/src/app/zipfile.cljs Normal file
View File

@@ -0,0 +1,13 @@
(ns app.zipfile
(:require
["jszip" :as jszip]))
(defn create
[]
(new jszip))
(defn add!
[zfile name data]
(.file ^js zfile name data)
zfile)