diff --git a/backend/bin/start-prod b/backend/bin/start-prod index 3ea781c077..28613e2736 100755 --- a/backend/bin/start-prod +++ b/backend/bin/start-prod @@ -2,4 +2,4 @@ set -e -clojure ${CLOJURE_OPTIONS} -m uxbox.main +clojure -O:jmx-remote -A:dev -J-Xms100m -J-Xmx100m -J-XX:+AlwaysPreTouch -J-XX:+UseBiasedLocking -J-Duxbox.enable-asserts=false -J-Dclojure.compiler.direct-linking=true -J-Dclojure.server.repl='{:port 5555 :accept clojure.core.server/repl}' -m uxbox.main diff --git a/backend/deps.edn b/backend/deps.edn index ecb80c9087..9a43a21467 100644 --- a/backend/deps.edn +++ b/backend/deps.edn @@ -9,11 +9,15 @@ ;; Logging org.clojure/tools.logging {:mvn/version "1.1.0"} - org.apache.logging.log4j/log4j-api {:mvn/version "2.13.2"} - org.apache.logging.log4j/log4j-core {:mvn/version "2.13.2"} - org.apache.logging.log4j/log4j-web {:mvn/version "2.13.2"} - org.apache.logging.log4j/log4j-jul {:mvn/version "2.13.2"} - org.apache.logging.log4j/log4j-slf4j-impl {:mvn/version "2.13.2"} + org.apache.logging.log4j/log4j-api {:mvn/version "2.13.3"} + org.apache.logging.log4j/log4j-core {:mvn/version "2.13.3"} + org.apache.logging.log4j/log4j-web {:mvn/version "2.13.3"} + org.apache.logging.log4j/log4j-jul {:mvn/version "2.13.3"} + org.apache.logging.log4j/log4j-slf4j-impl {:mvn/version "2.13.3"} + + io.prometheus/simpleclient {:mvn/version "0.9.0"} + io.prometheus/simpleclient_hotspot {:mvn/version "0.9.0"} + io.prometheus/simpleclient_httpserver {:mvn/version "0.9.0"} expound/expound {:mvn/version "0.8.4"} instaparse/instaparse {:mvn/version "1.4.10"} @@ -26,7 +30,7 @@ seancorfield/next.jdbc {:mvn/version "1.0.424"} metosin/reitit-ring {:mvn/version "0.4.2"} org.postgresql/postgresql {:mvn/version "42.2.12"} - com.zaxxer/HikariCP {:mvn/version "3.4.3"} + com.zaxxer/HikariCP {:mvn/version "3.4.5"} funcool/datoteka {:mvn/version "1.2.0"} funcool/promesa {:mvn/version "5.1.0"} @@ -51,7 +55,7 @@ io.aviso/pretty {:mvn/version "0.1.37"} mount/mount {:mvn/version "0.1.16"} - environ/environ {:mvn/version "1.1.0"}} + environ/environ {:mvn/version "1.2.0"}} :paths ["src" "resources" "../common" "common"] :aliases {:dev diff --git a/backend/resources/log4j2.xml b/backend/resources/log4j2.xml index 2e684138e5..12cbb25f71 100644 --- a/backend/resources/log4j2.xml +++ b/backend/resources/log4j2.xml @@ -1,3 +1,4 @@ + diff --git a/backend/src/uxbox/db.clj b/backend/src/uxbox/db.clj index 538356d7fd..af5940fe62 100644 --- a/backend/src/uxbox/db.clj +++ b/backend/src/uxbox/db.clj @@ -6,6 +6,7 @@ (ns uxbox.db (:require + [clojure.data.json :as json] [clojure.string :as str] [clojure.tools.logging :as log] [lambdaisland.uri :refer [uri]] @@ -16,10 +17,13 @@ [next.jdbc.result-set :as jdbc-rs] [next.jdbc.sql :as jdbc-sql] [next.jdbc.sql.builder :as jdbc-bld] + [uxbox.metrics :as mtx] [uxbox.common.exceptions :as ex] [uxbox.config :as cfg] [uxbox.util.data :as data]) (:import + org.postgresql.util.PGobject + com.zaxxer.hikari.metrics.prometheus.PrometheusMetricsTrackerFactory com.zaxxer.hikari.HikariConfig com.zaxxer.hikari.HikariDataSource)) @@ -28,17 +32,20 @@ (let [dburi (:database-uri cfg) username (:database-username cfg) password (:database-password cfg) - config (HikariConfig.)] + config (HikariConfig.) + mfactory (PrometheusMetricsTrackerFactory. mtx/registry)] (doto config (.setJdbcUrl (str "jdbc:" dburi)) + (.setPoolName "main") (.setAutoCommit true) (.setReadOnly false) - (.setConnectionTimeout 30000) - (.setValidationTimeout 5000) - (.setIdleTimeout 600000) - (.setMaxLifetime 1800000) - (.setMinimumIdle 10) - (.setMaximumPoolSize 20)) + (.setConnectionTimeout 30000) ;; 30seg + (.setValidationTimeout 5000) ;; 5seg + (.setIdleTimeout 900000) ;; 15min + (.setMaxLifetime 900000) ;; 15min + (.setMinimumIdle 5) + (.setMaximumPoolSize 10) + (.setMetricsTrackerFactory mfactory)) (when username (.setUsername config username)) (when password (.setPassword config password)) config)) @@ -112,3 +119,24 @@ (get-by-params ds table {:id id} nil)) ([ds table id opts] (get-by-params ds table {:id id} opts))) + +(defn pgobject? + [v] + (instance? PGobject v)) + +(defn decode-pgobject + [^PGobject obj] + (let [typ (.getType obj) + val (.getValue obj)] + (if (or (= typ "json") + (= typ "jsonb")) + (json/read-str val) + val))) + +;; Instrumentation + +(mtx/instrument-with-counter! + {:var [#'jdbc/execute-one! + #'jdbc/execute!] + :id "database__query_counter" + :help "An absolute counter of database queries."}) diff --git a/backend/src/uxbox/emails.clj b/backend/src/uxbox/emails.clj index 7926ebb427..2f7aeac72a 100644 --- a/backend/src/uxbox/emails.clj +++ b/backend/src/uxbox/emails.clj @@ -41,9 +41,9 @@ :reply-to (:sendmail-reply-to cfg/config)} data (merge defaults context) email (email-factory data)] - (tasks/schedule! conn {:name "sendmail" - :delay 0 - :props email})))) + (tasks/submit! conn {:name "sendmail" + :delay 0 + :props email})))) ;; --- Emails diff --git a/backend/src/uxbox/http.clj b/backend/src/uxbox/http.clj index 1b5d6a551c..4ee8a44c06 100644 --- a/backend/src/uxbox/http.clj +++ b/backend/src/uxbox/http.clj @@ -17,12 +17,14 @@ [uxbox.http.middleware :as middleware] [uxbox.http.session :as session] [uxbox.http.ws :as ws] + [uxbox.metrics :as mtx] [uxbox.services.notifications :as usn])) (defn- create-router [] (rring/router - [["/api" {:middleware [[middleware/format-response-body] + [["/metrics" {:get mtx/dump}] + ["/api" {:middleware [[middleware/format-response-body] [middleware/errors errors/handle] [middleware/parse-request-body] [middleware/params] @@ -37,7 +39,6 @@ ["/logout" {:handler handlers/logout-handler :method :post}] - ["/w" {:middleware [session/auth]} ["/query/:type" {:get handlers/query-handler}] ["/mutation/:type" {:post handlers/mutation-handler}]]]])) @@ -46,8 +47,9 @@ :start (rring/ring-handler (create-router) (constantly {:status 404, :body ""}) - {:middleware [middleware/development-resources - middleware/development-cors]})) + {:middleware [[middleware/development-resources] + [middleware/development-cors] + [middleware/metrics]]})) (defn start-server [cfg app] diff --git a/backend/src/uxbox/http/errors.clj b/backend/src/uxbox/http/errors.clj index 23593eef4e..0ec60f4dfe 100644 --- a/backend/src/uxbox/http/errors.clj +++ b/backend/src/uxbox/http/errors.clj @@ -9,6 +9,7 @@ (:require [clojure.tools.logging :as log] [cuerdas.core :as str] + [uxbox.metrics :as mtx] [io.aviso.exception :as e])) (defmulti handle-exception diff --git a/backend/src/uxbox/http/middleware.clj b/backend/src/uxbox/http/middleware.clj index e16a6da143..4955cfa68b 100644 --- a/backend/src/uxbox/http/middleware.clj +++ b/backend/src/uxbox/http/middleware.clj @@ -15,6 +15,7 @@ [ring.middleware.multipart-params :refer [wrap-multipart-params]] [ring.middleware.params :refer [wrap-params]] [ring.middleware.resource :refer [wrap-resource]] + [uxbox.metrics :as mtx] [uxbox.common.exceptions :as ex] [uxbox.config :as cfg] [uxbox.util.transit :as t])) @@ -83,6 +84,12 @@ {:name ::errors :compile (constantly wrap-errors)}) +(def metrics + {:name ::metrics + :wrap (fn [handler] + (mtx/wrap-counter handler {:id "http__requests_counter" + :help "Absolute http requests counter."}))}) + (def cookies {:name ::cookies :compile (constantly wrap-cookies)}) diff --git a/backend/src/uxbox/media_loader.clj b/backend/src/uxbox/media_loader.clj index 7c8b522fdb..988bffb661 100644 --- a/backend/src/uxbox/media_loader.clj +++ b/backend/src/uxbox/media_loader.clj @@ -49,7 +49,8 @@ (s/def ::path ::us/string) (s/def ::regex #(instance? java.util.regex.Pattern %)) -(s/def ::colors (s/every ::us/color :kind set?)) +(s/def ::colors + (s/* (s/cat :name ::us/string :color ::us/color))) (s/def ::import-item-media (s/keys :req-un [::name ::path ::regex])) @@ -238,22 +239,23 @@ id)) (defn- create-color - [conn library-id content] + [conn library-id name content] (s/assert ::us/uuid library-id) (s/assert ::us/color content) (let [color-id (uuid/namespaced +colors-uuid-ns+ (str library-id content))] - (log/info "Creating color" content color-id) + (log/info "Creating color" color-id "-" name content) (colors/create-color conn {:id color-id :library-id library-id - :name content + :name name :content content}) color-id)) (defn- import-colors [conn library-id {:keys [colors] :as item}] - (us/verify ::import-item-color item) (db/delete! conn :color {:library-id library-id}) - (run! #(create-color conn library-id %) colors)) + (run! (fn [[name content]] + (create-color conn library-id name content)) + (partition-all 2 colors))) (defn- process-colors-library [conn {:keys [name id colors] :as item}] diff --git a/backend/src/uxbox/metrics.clj b/backend/src/uxbox/metrics.clj new file mode 100644 index 0000000000..01b2f73864 --- /dev/null +++ b/backend/src/uxbox/metrics.clj @@ -0,0 +1,181 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; This Source Code Form is "Incompatible With Secondary Licenses", as +;; defined by the Mozilla Public License, v. 2.0. +;; +;; Copyright (c) 2020 UXBOX Labs SL + +(ns uxbox.metrics + (:require + [clojure.tools.logging :as log] + [cuerdas.core :as str]) + (:import + io.prometheus.client.CollectorRegistry + io.prometheus.client.Counter + io.prometheus.client.Gauge + io.prometheus.client.Summary + io.prometheus.client.exporter.common.TextFormat + io.prometheus.client.hotspot.DefaultExports + java.io.StringWriter)) + +(defn- create-registry + [] + (let [registry (CollectorRegistry.)] + (DefaultExports/register registry) + registry)) + +(defonce registry (create-registry)) +(defonce cache (atom {})) + +(defmacro with-measure + [sym expr teardown] + `(let [~sym (System/nanoTime)] + (try + ~expr + (finally + (let [~sym (/ (- (System/nanoTime) ~sym) 1000000)] + ~teardown))))) + +(defn make-counter + [{:keys [id help] :as props}] + (let [instance (doto (Counter/build) + (.name id) + (.help help)) + instance (.register instance registry)] + (reify + clojure.lang.IDeref + (deref [_] instance) + + clojure.lang.IFn + (invoke [_ cmd] + (.inc ^Counter instance)) + + (invoke [_ cmd val] + (case cmd + :wrap (fn + ([a] + (.inc ^Counter instance) + (val a)) + ([a b] + (.inc ^Counter instance) + (val a b)) + ([a b c] + (.inc ^Counter instance) + (val a b c))) + + (throw (IllegalArgumentException. "invalid arguments"))))))) + +(defn counter + [{:keys [id] :as props}] + (or (get @cache id) + (let [v (make-counter props)] + (swap! cache assoc id v) + v))) + +(defn make-gauge + [{:keys [id help] :as props}] + (let [instance (doto (Gauge/build) + (.name id) + (.help help)) + instance (.register instance registry)] + (reify + clojure.lang.IDeref + (deref [_] instance) + + clojure.lang.IFn + (invoke [_ cmd] + (case cmd + :inc (.inc ^Gauge instance) + :dec (.dec ^Gauge instance)))))) + +(defn gauge + [{:keys [id] :as props}] + (or (get @cache id) + (let [v (make-gauge props)] + (swap! cache assoc id v) + v))) + +(defn make-summary + [{:keys [id help] :as props}] + (let [instance (doto (Summary/build) + (.name id) + (.help help) + (.quantile 0.5 0.05) + (.quantile 0.9 0.01) + (.quantile 0.99 0.001)) + instance (.register instance registry)] + (reify + clojure.lang.IDeref + (deref [_] instance) + + clojure.lang.IFn + (invoke [_ val] + (.observe ^Summary instance val)) + + (invoke [_ cmd val] + (case cmd + :wrap (fn + ([a] + (with-measure $$ + (val a) + (.observe ^Summary instance $$))) + ([a b] + (with-measure $$ + (val a b) + (.observe ^Summary instance $$))) + ([a b c] + (with-measure $$ + (val a b c) + (.observe ^Summary instance $$)))) + + (throw (IllegalArgumentException. "invalid arguments"))))))) + +(defn summary + [{:keys [id] :as props}] + (or (get @cache id) + (let [v (make-summary props)] + (swap! cache assoc id v) + v))) + +(defn wrap-summary + [f props] + (let [sm (summary props)] + (sm :wrap f))) + +(defn wrap-counter + [f props] + (let [cnt (counter props)] + (cnt :wrap f))) + +(defn instrument-with-counter! + [{:keys [var] :as props}] + (let [cnt (counter props) + vars (if (var? var) [var] var)] + (doseq [var vars] + (alter-var-root var (fn [root] + (let [mdata (meta root) + original (::counter-original mdata root)] + (with-meta + (cnt :wrap original) + (assoc mdata ::counter-original original)))))))) + +(defn instrument-with-summary! + [{:keys [var] :as props}] + (let [sm (summary props)] + (alter-var-root var (fn [root] + (let [mdata (meta root) + original (::summary-original mdata root)] + (with-meta + (sm :wrap original) + (assoc mdata ::summary-original original))))))) + +(defn dump + [& args] + (let [samples (.metricFamilySamples ^CollectorRegistry registry) + writer (StringWriter.)] + (TextFormat/write004 writer samples) + {:headers {"content-type" TextFormat/CONTENT_TYPE_004} + :body (.toString writer)})) + diff --git a/backend/src/uxbox/services/middleware.clj b/backend/src/uxbox/services/middleware.clj new file mode 100644 index 0000000000..24c129b34f --- /dev/null +++ b/backend/src/uxbox/services/middleware.clj @@ -0,0 +1,73 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; This Source Code Form is "Incompatible With Secondary Licenses", as +;; defined by the Mozilla Public License, v. 2.0. +;; +;; Copyright (c) 2020 UXBOX Labs SL + +(ns uxbox.services.middleware + "Common middleware for services." + (:require + [clojure.tools.logging :as log] + [clojure.spec.alpha :as s] + [cuerdas.core :as str] + [expound.alpha :as expound] + [uxbox.common.exceptions :as ex] + [uxbox.common.spec :as us] + [uxbox.metrics :as mtx])) + +(defn wrap-spec + [handler] + (let [mdata (meta handler) + spec (s/get-spec (:spec mdata))] + (if (nil? spec) + handler + (with-meta + (fn [params] + (let [result (us/conform spec params)] + (handler result))) + (assoc mdata ::wrap-spec true))))) + +(defn wrap-error + [handler] + (let [mdata (meta handler)] + (with-meta + (fn [params] + (try + (handler params) + (catch Throwable error + (ex/raise :type :service-error + :name (:spec mdata) + :cause error)))) + (assoc mdata ::wrap-error true)))) + +(defn- get-prefix + [nsname] + (let [[a b c] (str/split nsname ".")] + c)) + +(defn wrap-metrics + [handler] + (let [mdata (meta handler) + nsname (namespace (:spec mdata)) + smname (name (:spec mdata)) + prefix (get-prefix nsname) + + sname (str prefix "/" smname) + + props {:id (str/join "__" [prefix + (str/snake smname) + "response_time"]) + :help (str "Service timing measures for: " sname ".")}] + (with-meta + (mtx/wrap-summary handler props) + (assoc mdata ::wrap-metrics true)))) + +(defn wrap + [handler] + (-> handler + (wrap-spec) + (wrap-error) + (wrap-metrics))) diff --git a/backend/src/uxbox/services/mutations.clj b/backend/src/uxbox/services/mutations.clj index a4877fc072..8f12adb087 100644 --- a/backend/src/uxbox/services/mutations.clj +++ b/backend/src/uxbox/services/mutations.clj @@ -5,16 +5,16 @@ ;; This Source Code Form is "Incompatible With Secondary Licenses", as ;; defined by the Mozilla Public License, v. 2.0. ;; -;; Copyright (c) 2019-2020 Andrey Antukh +;; Copyright (c) 2020 UXBOX Labs SL (ns uxbox.services.mutations (:require + [uxbox.services.middleware :as middleware] [uxbox.util.dispatcher :as uds])) (uds/defservice handle :dispatch-by ::type - :wrap [uds/wrap-spec - uds/wrap-error]) + :wrap middleware/wrap) (defmacro defmutation [key & rest] diff --git a/backend/src/uxbox/services/mutations/colors.clj b/backend/src/uxbox/services/mutations/colors.clj index 394ecd5251..82428854ff 100644 --- a/backend/src/uxbox/services/mutations/colors.clj +++ b/backend/src/uxbox/services/mutations/colors.clj @@ -104,9 +104,9 @@ (teams/check-edition-permissions! conn profile-id (:team-id lib)) ;; Schedule object deletion - (tasks/schedule! conn {:name "delete-object" - :delay cfg/default-deletion-delay - :props {:id id :type :color-library}}) + (tasks/submit! conn {:name "delete-object" + :delay cfg/default-deletion-delay + :props {:id id :type :color-library}}) (db/update! conn :color-library {:deleted-at (dt/now)} @@ -188,9 +188,9 @@ (teams/check-edition-permissions! conn profile-id (:team-id clr)) ;; Schedule object deletion - (tasks/schedule! conn {:name "delete-object" - :delay cfg/default-deletion-delay - :props {:id id :type :color}}) + (tasks/submit! conn {:name "delete-object" + :delay cfg/default-deletion-delay + :props {:id id :type :color}}) (db/update! conn :color {:deleted-at (dt/now)} diff --git a/backend/src/uxbox/services/mutations/demo.clj b/backend/src/uxbox/services/mutations/demo.clj index ea0a5ae56f..aa319fa1f3 100644 --- a/backend/src/uxbox/services/mutations/demo.clj +++ b/backend/src/uxbox/services/mutations/demo.clj @@ -38,8 +38,8 @@ :password password}) ;; Schedule deletion of the demo profile - (tasks/schedule! conn {:name "delete-profile" - :delay cfg/default-deletion-delay - :props {:profile-id id}}) + (tasks/submit! conn {:name "delete-profile" + :delay cfg/default-deletion-delay + :props {:profile-id id}}) {:email email :password password}))) diff --git a/backend/src/uxbox/services/mutations/files.clj b/backend/src/uxbox/services/mutations/files.clj index 4dffcb94dd..c1723078af 100644 --- a/backend/src/uxbox/services/mutations/files.clj +++ b/backend/src/uxbox/services/mutations/files.clj @@ -113,9 +113,9 @@ (files/check-edition-permissions! conn profile-id id) ;; Schedule object deletion - (tasks/schedule! conn {:name "delete-object" - :delay cfg/default-deletion-delay - :props {:id id :type :file}}) + (tasks/submit! conn {:name "delete-object" + :delay cfg/default-deletion-delay + :props {:id id :type :file}}) (mark-file-deleted conn params))) diff --git a/backend/src/uxbox/services/mutations/icons.clj b/backend/src/uxbox/services/mutations/icons.clj index d8bad54a77..14e8c55f7e 100644 --- a/backend/src/uxbox/services/mutations/icons.clj +++ b/backend/src/uxbox/services/mutations/icons.clj @@ -111,9 +111,9 @@ (teams/check-edition-permissions! conn profile-id (:team-id lib)) ;; Schedule object deletion - (tasks/schedule! conn {:name "delete-object" - :delay cfg/default-deletion-delay - :props {:id id :type :icon-library}}) + (tasks/submit! conn {:name "delete-object" + :delay cfg/default-deletion-delay + :props {:id id :type :icon-library}}) (db/update! conn :icon-library {:deleted-at (dt/now)} @@ -196,9 +196,9 @@ (teams/check-edition-permissions! conn profile-id (:team-id icn)) ;; Schedule object deletion - (tasks/schedule! conn {:name "delete-object" - :delay cfg/default-deletion-delay - :props {:id id :type :icon}}) + (tasks/submit! conn {:name "delete-object" + :delay cfg/default-deletion-delay + :props {:id id :type :icon}}) (db/update! conn :icon {:deleted-at (dt/now)} diff --git a/backend/src/uxbox/services/mutations/images.clj b/backend/src/uxbox/services/mutations/images.clj index 905fc8a23d..69ee47e526 100644 --- a/backend/src/uxbox/services/mutations/images.clj +++ b/backend/src/uxbox/services/mutations/images.clj @@ -96,9 +96,9 @@ (teams/check-edition-permissions! conn profile-id (:team-id lib)) ;; Schedule object deletion - (tasks/schedule! conn {:name "delete-object" - :delay cfg/default-deletion-delay - :props {:id id :type :image-library}}) + (tasks/submit! conn {:name "delete-object" + :delay cfg/default-deletion-delay + :props {:id id :type :image-library}}) (db/update! conn :image-library {:deleted-at (dt/now)} @@ -226,9 +226,9 @@ (teams/check-edition-permissions! conn profile-id (:team-id img)) ;; Schedule object deletion - (tasks/schedule! conn {:name "delete-object" - :delay cfg/default-deletion-delay - :props {:id id :type :image}}) + (tasks/submit! conn {:name "delete-object" + :delay cfg/default-deletion-delay + :props {:id id :type :image}}) (db/update! conn :image {:deleted-at (dt/now)} diff --git a/backend/src/uxbox/services/mutations/pages.clj b/backend/src/uxbox/services/mutations/pages.clj index 8c570938e4..382c731c76 100644 --- a/backend/src/uxbox/services/mutations/pages.clj +++ b/backend/src/uxbox/services/mutations/pages.clj @@ -21,8 +21,10 @@ [uxbox.services.queries.files :as files] [uxbox.services.queries.pages :refer [decode-row]] [uxbox.tasks :as tasks] + [uxbox.redis :as redis] [uxbox.util.blob :as blob] - [uxbox.util.time :as dt])) + [uxbox.util.time :as dt] + [uxbox.util.transit :as t])) ;; --- Helpers & Specs @@ -148,9 +150,10 @@ (s/def ::changes (s/coll-of map? :kind vector?)) +(s/def ::session-id ::us/uuid) (s/def ::revn ::us/integer) (s/def ::update-page - (s/keys :req-un [::id ::profile-id ::revn ::changes])) + (s/keys :req-un [::id ::session-id ::profile-id ::revn ::changes])) (declare update-page) (declare retrieve-lagged-changes) @@ -172,7 +175,9 @@ :hint "The incoming revision number is greater that stored version." :context {:incoming-revn (:revn params) :stored-revn (:revn page)})) - (let [changes (:changes params) + (let [sid (:session-id params) + changes (->> (:changes params) + (mapv #(assoc % :session-id sid))) data (-> (:data page) (blob/decode) (cp/process-changes changes) @@ -183,7 +188,16 @@ :revn (inc (:revn page)) :changes (blob/encode changes)) - chng (insert-page-change! conn page)] + chng (insert-page-change! conn page) + msg {:type :page-change + :profile-id (:profile-id params) + :page-id (:id page) + :session-id sid + :revn (:revn page) + :changes changes}] + + @(redis/run! :publish {:channel (str (:file-id page)) + :message (t/encode-str msg)}) (db/update! conn :page {:revn (:revn page) @@ -192,13 +206,6 @@ (retrieve-lagged-changes conn chng params))) -;; (p/do! (ve/publish! uxbox.core/system topic -;; {:type :page-change -;; :profile-id (:profile-id params) -;; :page-id (:page-id s) -;; :revn (:revn s) -;; :changes changes}) - (defn- insert-page-change! [conn {:keys [revn data changes] :as page}] (let [id (uuid/next) @@ -242,9 +249,9 @@ (files/check-edition-permissions! conn profile-id (:file-id page)) ;; Schedule object deletion - (tasks/schedule! conn {:name "delete-object" - :delay cfg/default-deletion-delay - :props {:id id :type :page}}) + (tasks/submit! conn {:name "delete-object" + :delay cfg/default-deletion-delay + :props {:id id :type :page}}) (db/update! conn :page {:deleted-at (dt/now)} diff --git a/backend/src/uxbox/services/mutations/profile.clj b/backend/src/uxbox/services/mutations/profile.clj index e3e39cc3f4..e880bb77a0 100644 --- a/backend/src/uxbox/services/mutations/profile.clj +++ b/backend/src/uxbox/services/mutations/profile.clj @@ -155,8 +155,8 @@ ;; Schedule deletion of old photo (when (and (string? (:photo profile)) (not (str/blank? (:photo profile)))) - (tasks/schedule! conn {:name "remove-media" - :props {:path (:photo profile)}})) + (tasks/submit! conn {:name "remove-media" + :props {:path (:photo profile)}})) ;; Save new photo (update-profile-photo conn profile-id photo)))) @@ -363,9 +363,9 @@ (check-teams-ownership! conn profile-id) ;; Schedule a complete deletion of profile - (tasks/schedule! conn {:name "delete-profile" - :delay (dt/duration {:hours 48}) - :props {:profile-id profile-id}}) + (tasks/submit! conn {:name "delete-profile" + :delay (dt/duration {:hours 48}) + :props {:profile-id profile-id}}) (db/update! conn :profile {:deleted-at (dt/now)} diff --git a/backend/src/uxbox/services/mutations/projects.clj b/backend/src/uxbox/services/mutations/projects.clj index b23552aaa2..382acefb1d 100644 --- a/backend/src/uxbox/services/mutations/projects.clj +++ b/backend/src/uxbox/services/mutations/projects.clj @@ -124,9 +124,9 @@ (check-edition-permissions! conn profile-id id) ;; Schedule object deletion - (tasks/schedule! conn {:name "delete-object" - :delay cfg/default-deletion-delay - :props {:id id :type :project}}) + (tasks/submit! conn {:name "delete-object" + :delay cfg/default-deletion-delay + :props {:id id :type :project}}) (mark-project-deleted conn params))) diff --git a/backend/src/uxbox/services/notifications.clj b/backend/src/uxbox/services/notifications.clj index 27dcd80a9e..8b44557ecb 100644 --- a/backend/src/uxbox/services/notifications.clj +++ b/backend/src/uxbox/services/notifications.clj @@ -13,8 +13,9 @@ [ring.adapter.jetty9 :as jetty] [uxbox.common.exceptions :as ex] [uxbox.common.uuid :as uuid] - [uxbox.redis :as redis] [uxbox.db :as db] + [uxbox.redis :as redis] + [uxbox.metrics :as mtx] [uxbox.util.time :as dt] [uxbox.util.transit :as t])) @@ -193,11 +194,20 @@ (jetty/send! conn (t/encode-str val)) (recur))))) +(defonce metrics-active-connections + (mtx/gauge {:id "notificatons__active_connections" + :help "Active connections to the notifications service."})) + +(defonce metrics-message-counter + (mtx/counter {:id "notificatons__messages_counter" + :help "A total number of messages handled by the notifications service."})) + (defn websocket [{:keys [file-id] :as params}] (let [in (a/chan 32) out (a/chan 32)] {:on-connect (fn [conn] + (metrics-active-connections :inc) (let [xf (map t/decode-str) sub (redis/subscribe (str file-id) xf) ws (WebSocket. conn in out sub nil params)] @@ -207,21 +217,19 @@ (a/close! sub)))) :on-error (fn [conn e] - ;; (prn "websocket" :on-error e) (a/close! out) (a/close! in)) :on-close (fn [conn status-code reason] - ;; (prn "websocket" :on-close status-code reason) + (metrics-active-connections :dec) (a/close! out) (a/close! in)) :on-text (fn [ws message] + (metrics-message-counter :inc) (let [message (t/decode-str message)] - ;; (prn "websocket" :on-text message) (a/>!! in message))) - :on-bytes (fn [ws bytes offset len] - #_(prn "websocket" :on-bytes bytes))})) + :on-bytes (constantly nil)})) diff --git a/backend/src/uxbox/services/queries.clj b/backend/src/uxbox/services/queries.clj index 1a55a047ed..8608f55122 100644 --- a/backend/src/uxbox/services/queries.clj +++ b/backend/src/uxbox/services/queries.clj @@ -2,16 +2,19 @@ ;; License, v. 2.0. If a copy of the MPL was not distributed with this ;; file, You can obtain one at http://mozilla.org/MPL/2.0/. ;; -;; Copyright (c) 2019 Andrey Antukh +;; This Source Code Form is "Incompatible With Secondary Licenses", as +;; defined by the Mozilla Public License, v. 2.0. +;; +;; Copyright (c) 2020 UXBOX Labs SL (ns uxbox.services.queries (:require + [uxbox.services.middleware :as middleware] [uxbox.util.dispatcher :as uds])) (uds/defservice handle :dispatch-by ::type - :wrap [uds/wrap-spec - uds/wrap-error]) + :wrap middleware/wrap) (defmacro defquery [key & rest] diff --git a/backend/src/uxbox/services/queries/icons.clj b/backend/src/uxbox/services/queries/icons.clj index ae92594038..1ea653725d 100644 --- a/backend/src/uxbox/services/queries/icons.clj +++ b/backend/src/uxbox/services/queries/icons.clj @@ -10,8 +10,6 @@ (ns uxbox.services.queries.icons (:require [clojure.spec.alpha :as s] - [promesa.core :as p] - [promesa.exec :as px] [uxbox.common.exceptions :as ex] [uxbox.common.spec :as us] [uxbox.common.uuid :as uuid] diff --git a/backend/src/uxbox/services/queries/images.clj b/backend/src/uxbox/services/queries/images.clj index d790e9bc9d..7563138d97 100644 --- a/backend/src/uxbox/services/queries/images.clj +++ b/backend/src/uxbox/services/queries/images.clj @@ -10,13 +10,12 @@ (ns uxbox.services.queries.images (:require [clojure.spec.alpha :as s] - [promesa.core :as p] [uxbox.common.exceptions :as ex] [uxbox.common.spec :as us] [uxbox.db :as db] [uxbox.images :as images] - [uxbox.services.queries.teams :as teams] - [uxbox.services.queries :as sq])) + [uxbox.services.queries :as sq] + [uxbox.services.queries.teams :as teams])) (s/def ::id ::us/uuid) (s/def ::name ::us/string) diff --git a/backend/src/uxbox/tasks.clj b/backend/src/uxbox/tasks.clj index 4108a59d0a..4a320c5535 100644 --- a/backend/src/uxbox/tasks.clj +++ b/backend/src/uxbox/tasks.clj @@ -16,12 +16,23 @@ [uxbox.common.spec :as us] [uxbox.config :as cfg] [uxbox.db :as db] + [uxbox.metrics :as mtx] [uxbox.tasks.sendmail] + [uxbox.tasks.gc] [uxbox.tasks.remove-media] [uxbox.tasks.delete-profile] [uxbox.tasks.delete-object] [uxbox.tasks.impl :as impl] - [uxbox.util.time :as dt])) + [uxbox.util.time :as dt]) + (:import + java.util.concurrent.ScheduledExecutorService + java.util.concurrent.Executors)) + +;; --- Scheduler Executor Initialization + +(defstate scheduler + :start (Executors/newScheduledThreadPool (int 1)) + :stop (.shutdownNow ^ScheduledExecutorService scheduler)) ;; --- State initialization @@ -36,33 +47,30 @@ "remove-media" #'uxbox.tasks.remove-media/handler "sendmail" #'uxbox.tasks.sendmail/handler}) +(def ^:private schedule + [{:id "remove-deleted-media" + :cron (dt/cron "1 1 */1 * * ? *") + :fn #'uxbox.tasks.gc/remove-media}]) + (defstate worker - :start (impl/start-worker! {:tasks tasks}) + :start (impl/start-worker! {:tasks tasks + :xtor scheduler}) + :stop (impl/stop! worker)) + +(defstate scheduler-worker + :start (impl/start-scheduler-worker! {:schedule schedule + :xtor scheduler}) :stop (impl/stop! worker)) ;; --- Public API -(defn schedule! - ([opts] (schedule! db/pool opts)) +(defn submit! + ([opts] (submit! db/pool opts)) ([conn opts] (s/assert ::impl/task-options opts) - (impl/schedule! conn opts))) + (impl/submit! conn opts))) -;; (defstate scheduler -;; :start (impl/start-scheduler! tasks) -;; :stop (impl/stop! tasks-worker)) - -;; :start (as-> (impl/worker-verticle {:tasks tasks}) $$ -;; (vc/deploy! system $$ {:instances 1}) -;; (deref $$))) - -;; (def ^:private schedule -;; [{:id "every 1 hour" -;; :cron (dt/cron "1 1 */1 * * ? *") -;; :fn #'uxbox.tasks.gc/handler -;; :props {:foo 1}}]) - -;; (defstate scheduler -;; :start (as-> (impl/scheduler-verticle {:schedule schedule}) $$ -;; (vc/deploy! system $$ {:instances 1 :worker true}) -;; (deref $$))) +(mtx/instrument-with-counter! + {:var #'submit! + :id "tasks__submit_counter" + :help "Absolute task submit counter."}) diff --git a/backend/src/uxbox/tasks/delete_object.clj b/backend/src/uxbox/tasks/delete_object.clj index 652c3f916e..ea193d3d1c 100644 --- a/backend/src/uxbox/tasks/delete_object.clj +++ b/backend/src/uxbox/tasks/delete_object.clj @@ -15,7 +15,7 @@ [uxbox.common.exceptions :as ex] [uxbox.common.spec :as us] [uxbox.db :as db] - [uxbox.media :as media] + [uxbox.metrics :as mtx] [uxbox.util.storage :as ust])) (s/def ::type keyword?) @@ -36,6 +36,11 @@ (db/with-atomic [conn db/pool] (handle-deletion conn props))) +(mtx/instrument-with-summary! + {:var #'handler + :id "tasks__delete_object" + :help "Timing of remove-object task."}) + (defmethod handle-deletion :image [conn {:keys [id] :as props}] (let [sql "delete from image where id=? and deleted_at is not null"] diff --git a/backend/src/uxbox/tasks/delete_profile.clj b/backend/src/uxbox/tasks/delete_profile.clj index 74bb6339c9..ea23290358 100644 --- a/backend/src/uxbox/tasks/delete_profile.clj +++ b/backend/src/uxbox/tasks/delete_profile.clj @@ -15,7 +15,7 @@ [uxbox.common.exceptions :as ex] [uxbox.common.spec :as us] [uxbox.db :as db] - [uxbox.media :as media] + [uxbox.metrics :as mtx] [uxbox.util.storage :as ust])) (declare delete-profile-data) @@ -39,6 +39,11 @@ (log/warn "Profile " (:id profile) "does not match constraints for deletion"))))) +(mtx/instrument-with-summary! + {:var #'handler + :id "tasks__delete_profile" + :help "Timing of delete-profile task."}) + (defn- delete-profile-data [conn profile-id] (log/info "Proceding to delete all data related to profile" profile-id) diff --git a/backend/src/uxbox/tasks/gc.clj b/backend/src/uxbox/tasks/gc.clj index c4db5c9d95..690ef922d7 100644 --- a/backend/src/uxbox/tasks/gc.clj +++ b/backend/src/uxbox/tasks/gc.clj @@ -9,8 +9,8 @@ (ns uxbox.tasks.gc (:require - [clojure.tools.logging :as log] [clojure.spec.alpha :as s] + [clojure.tools.logging :as log] [cuerdas.core :as str] [postal.core :as postal] [promesa.core :as p] @@ -18,36 +18,47 @@ [uxbox.common.spec :as us] [uxbox.config :as cfg] [uxbox.db :as db] - [uxbox.util.blob :as blob])) + [uxbox.media :as media] + [uxbox.util.blob :as blob] + [uxbox.util.storage :as ust])) -;; TODO: delete media referenced in pendint_to_delete table +(def ^:private sql:delete-items + "with items_part as ( + select i.id + from pending_to_delete as i + order by i.created_at + limit ? + for update skip locked + ) + delete from pending_to_delete + where id in (select id from items_part) + returning *") -;; (def ^:private sql:delete-item -;; "with items_part as ( -;; select i.id -;; from pending_to_delete as i -;; order by i.created_at -;; limit 1 -;; for update skip locked -;; ) -;; delete from pending_to_delete -;; where id in (select id from items_part) -;; returning *") +(defn- impl-remove-media + [result] + (run! (fn [item] + (let [path1 (get item "path") + path2 (get item "thumb_path")] + (ust/delete! media/media-storage path1) + (ust/delete! media/media-storage path2))) + result)) -;; (defn- remove-items -;; [] -;; (vu/loop [] -;; (db/with-atomic [conn db/pool] -;; (-> (db/query-one conn sql:delete-item) -;; (p/then decode-row) -;; (p/then (vu/wrap-blocking remove-media)) -;; (p/then (fn [item] -;; (when (not (empty? items)) -;; (p/recur)))))))) +(defn- decode-row + [{:keys [data] :as row}] + (cond-> row + (db/pgobject? data) (assoc :data (db/decode-pgobject data)))) + +(defn- get-items + [conn] + (->> (db/exec! conn [sql:delete-items 10]) + (map decode-row) + (map :data))) + +(defn remove-media + [{:keys [props] :as task}] + (db/with-atomic [conn db/pool] + (loop [result (get-items conn)] + (when-not (empty? result) + (impl-remove-media result) + (recur (get-items conn)))))) -;; (defn- remove-media -;; [{:keys -;; (doseq [item files] -;; (ust/delete! media/media-storage (:path item)) -;; (ust/delete! media/media-storage (:thumb-path item))) -;; files) diff --git a/backend/src/uxbox/tasks/impl.clj b/backend/src/uxbox/tasks/impl.clj index 249a8bf640..121797e6f0 100644 --- a/backend/src/uxbox/tasks/impl.clj +++ b/backend/src/uxbox/tasks/impl.clj @@ -13,6 +13,7 @@ [clojure.core.async :as a] [clojure.spec.alpha :as s] [clojure.tools.logging :as log] + [promesa.exec :as px] [uxbox.common.spec :as us] [uxbox.common.uuid :as uuid] [uxbox.config :as cfg] @@ -20,14 +21,12 @@ [uxbox.util.blob :as blob] [uxbox.util.time :as dt]) (:import + java.util.concurrent.ScheduledExecutorService + java.util.concurrent.Executors java.time.Duration java.time.Instant java.util.Date)) -(defrecord Worker [stop] - java.lang.AutoCloseable - (close [_] (a/close! stop))) - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Tasks ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -37,7 +36,8 @@ (with-out-str (.printStackTrace err (java.io.PrintWriter. *out*)))) -(def ^:private sql:mark-as-retry +(def ^:private + sql:mark-as-retry "update task set scheduled_at = clock_timestamp() + '5 seconds'::interval, error = ?, @@ -45,48 +45,32 @@ retry_num = retry_num + 1 where id = ?") -(defn- reschedule +(defn- mark-as-retry [conn task error] (let [explain (ex-message error) sqlv [sql:mark-as-retry explain (:id task)]] (db/exec-one! conn sqlv) nil)) -(def ^:private sql:mark-as-failed - "update task - set scheduled_at = clock_timestamp() + '5 seconds'::interval, - error = ?, - status = 'failed' - where id = ?;") - (defn- mark-as-failed [conn task error] - (let [explain (ex-message error) - sqlv [sql:mark-as-failed explain (:id task)]] - (db/exec-one! conn sqlv) + (let [explain (ex-message error)] + (db/update! conn :task + {:error explain + :status "failed"} + {:id (:id task)}) nil)) -(def ^:private sql:mark-as-completed - "update task - set completed_at = clock_timestamp(), - status = 'completed' - where id = ?") - (defn- mark-as-completed [conn task] - (db/exec-one! conn [sql:mark-as-completed (:id task)]) + (db/update! conn :task + {:completed-at (dt/now) + :status "completed"} + {:id (:id task)}) nil) -(defn- handle-task - [tasks {:keys [name] :as item}] - (let [task-fn (get tasks name)] - (if task-fn - (task-fn item) - (do - (log/warn "no task handler found for" (pr-str name)) - nil)))) - -(def ^:private sql:select-next-task +(def ^:private + sql:select-next-task "select * from task as t where t.scheduled_at <= now() and t.queue = ? @@ -108,6 +92,15 @@ (with-out-str (.printStackTrace ^Throwable err (java.io.PrintWriter. *out*))))) +(defn- handle-task + [tasks {:keys [name] :as item}] + (let [task-fn (get tasks name)] + (if task-fn + (task-fn item) + (do + (log/warn "no task handler found for" (pr-str name)) + nil)))) + (defn- event-loop-fn [{:keys [tasks] :as options}] (let [queue (:queue options "default") @@ -125,156 +118,140 @@ (log-task-error item e) (if (>= (:retry-num item) max-retries) (mark-as-failed conn item e) - (reschedule conn item e))))))))) + (mark-as-retry conn item e))))))))) -(defn- start-worker-eventloop! - [options] - (let [stop (::stop options) - mbs (:max-batch-size options 10)] - (a/go-loop [] - (let [timeout (a/timeout 5000) - [val port] (a/alts! [stop timeout])] - (when (= port timeout) - (a/ mbs cnt)) - (recur (inc 1) - (event-loop-fn options)))))) - (recur)))))) - -(defn- duration->pginterval - [^Duration d] - (->> (/ (.toMillis d) 1000.0) - (format "%s seconds"))) - -(defn start-worker! - [options] - (let [stop (a/chan)] - (start-worker-eventloop! (assoc options ::stop stop)) - (->Worker stop))) - -(defn stop! - [worker] - (.close ^java.lang.AutoCloseable worker)) +(defn- execute-worker-task + [{:keys [::stop ::xtor poll-interval] + :or {poll-interval 5000} + :as opts}] + (try + (when-not @stop + (let [res (event-loop-fn opts)] + (if (= res ::handled) + (px/schedule! xtor 0 (partial execute-worker-task opts)) + (px/schedule! xtor poll-interval (partial execute-worker-task opts))))) + (catch Throwable e + (log/error "unexpected exception:" e) + (px/schedule! xtor poll-interval (partial execute-worker-task opts))))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Scheduled Tasks ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; (def ^:privatr sql:upsert-scheduled-task -;; "insert into scheduled_task (id, cron_expr) -;; values ($1, $2) -;; on conflict (id) -;; do update set cron_expr=$2") +(def ^:private + sql:upsert-scheduled-task + "insert into scheduled_task (id, cron_expr) + values (?, ?) + on conflict (id) + do update set cron_expr=?") -;; (defn- synchronize-schedule-item -;; [conn {:keys [id cron]}] -;; (-> (db/query-one conn [sql:upsert-scheduled-task id (str cron)]) -;; (p/then' (constantly nil)))) +(defn- synchronize-schedule-item + [conn {:keys [id cron] :as item}] + (let [cron (str cron)] + (db/exec-one! conn [sql:upsert-scheduled-task id cron cron]))) -;; (defn- synchronize-schedule -;; [schedule] -;; (db/with-atomic [conn db/pool] -;; (p/run! (partial synchronize-schedule-item conn) schedule))) +(defn- synchronize-schedule! + [schedule] + (db/with-atomic [conn db/pool] + (run! (partial synchronize-schedule-item conn) schedule))) -;; (def ^:private sql:lock-scheduled-task -;; "select id from scheduled_task where id=$1 for update skip locked") +(def ^:private sql:lock-scheduled-task + "select id from scheduled_task where id=? for update skip locked") -;; (declare schedule-task) +(declare schedule-task!) -;; (defn- log-scheduled-task-error -;; [item err] -;; (log/error "Unhandled exception on scheduled task '" (:id item) "' \n" -;; (with-out-str -;; (.printStackTrace ^Throwable err (java.io.PrintWriter. *out*))))) +(defn- log-scheduled-task-error + [item err] + (log/error "Unhandled exception on scheduled task '" (:id item) "' \n" + (with-out-str + (.printStackTrace ^Throwable err (java.io.PrintWriter. *out*))))) -;; (defn- execute-scheduled-task -;; [{:keys [id cron] :as stask}] -;; (db/with-atomic [conn db/pool] -;; ;; First we try to lock the task in the database, if locking us -;; ;; successful, then we execute the scheduled task; if locking is -;; ;; not possible (because other instance is already locked id) we -;; ;; just skip it and schedule to be executed in the next slot. -;; (-> (db/query-one conn [sql:lock-scheduled-task id]) -;; (p/then (fn [result] -;; (when result -;; (-> (p/do! ((:fn stask) stask)) -;; (p/catch (fn [e] -;; (log-scheduled-task-error stask e) -;; nil)))))) -;; (p/finally (fn [v e] -;; (-> (vu/current-context) -;; (schedule-task stask))))))) -;; (defn ms-until-valid -;; [cron] -;; (s/assert dt/cron? cron) -;; (let [^Instant now (dt/now) -;; ^Instant next (dt/next-valid-instant-from cron now) -;; ^Duration duration (Duration/between now next)] -;; (.toMillis duration))) +(defn- execute-scheduled-task + [{:keys [id cron ::xtor] :as task}] + (try + (db/with-atomic [conn db/pool] + ;; First we try to lock the task in the database, if locking is + ;; successful, then we execute the scheduled task; if locking is + ;; not possible (because other instance is already locked id) we + ;; just skip it and schedule to be executed in the next slot. + (when (db/exec-one! conn [sql:lock-scheduled-task id]) + (log/info "Executing scheduled task" id) + ((:fn task) task))) -;; (defn- schedule-task -;; [ctx {:keys [cron] :as stask}] -;; (let [ms (ms-until-valid cron)] -;; (vt/schedule! ctx (assoc stask -;; :ctx ctx -;; ::vt/once true -;; ::vt/delay ms -;; ::vt/fn execute-scheduled-task)))) + (catch Throwable e + (log-scheduled-task-error task e)) + (finally + (schedule-task! xtor task)))) -;; (defn- on-scheduler-start -;; [ctx {:keys [schedule] :as options}] -;; (-> (synchronize-schedule schedule) -;; (p/then' (fn [_] -;; (run! #(schedule-task ctx %) schedule))))) +(defn ms-until-valid + [cron] + (s/assert dt/cron? cron) + (let [^Instant now (dt/now) + ^Instant next (dt/next-valid-instant-from cron now)] + (inst-ms (dt/duration-between now next)))) + +(defn- schedule-task! + [xtor {:keys [cron] :as task}] + (let [ms (ms-until-valid cron) + task (assoc task ::xtor xtor)] + (px/schedule! xtor ms (partial execute-scheduled-task task)))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Public API ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; --- Worker Verticle +(s/def ::id string?) +(s/def ::name string?) +(s/def ::cron dt/cron?) +(s/def ::fn (s/or :var var? :fn fn?)) +(s/def ::props (s/nilable map?)) +(s/def ::xtor #(instance? ScheduledExecutorService %)) -;; (s/def ::callable (s/or :fn fn? :var var?)) -;; (s/def ::max-batch-size ::us/integer) -;; (s/def ::max-retries ::us/integer) -;; (s/def ::tasks (s/map-of string? ::callable)) +(s/def ::scheduled-task + (s/keys :req-un [::id ::cron ::fn] + :opt-un [::props])) -;; (s/def ::worker-verticle-options -;; (s/keys :req-un [::tasks] -;; :opt-un [::queue ::max-batch-size])) +(s/def ::tasks (s/map-of string? ::fn)) +(s/def ::schedule (s/coll-of ::scheduled-task)) -;; (defn worker-verticle -;; [options] -;; (s/assert ::worker-verticle-options options) -;; (let [on-start #(on-worker-start % options)] -;; (vc/verticle {:on-start on-start}))) +(defn start-scheduler-worker! + [{:keys [schedule xtor] :as opts}] + (us/assert ::xtor xtor) + (us/assert ::schedule schedule) + (let [stop (atom false)] + (synchronize-schedule! schedule) + (run! (partial schedule-task! xtor) schedule) + (reify + java.lang.AutoCloseable + (close [_] + (reset! stop true))))) -;; --- Scheduler Verticle +(defn start-worker! + [{:keys [tasks xtor poll-interval] + :or {poll-interval 5000} + :as opts}] + (us/assert ::tasks tasks) + (us/assert ::xtor xtor) + (us/assert number? poll-interval) + (let [stop (atom false) + opts (assoc opts + ::xtor xtor + ::stop stop)] + (px/schedule! xtor poll-interval (partial execute-worker-task opts)) + (reify + java.lang.AutoCloseable + (close [_] + (reset! stop true))))) -;; (s/def ::id string?) -;; (s/def ::cron dt/cron?) -;; (s/def ::fn ::callable) -;; (s/def ::props (s/nilable map?)) +(defn stop! + [worker] + (.close ^java.lang.AutoCloseable worker)) -;; (s/def ::scheduled-task -;; (s/keys :req-un [::id ::cron ::fn] -;; :opt-un [::props])) -;; (s/def ::schedule (s/coll-of ::scheduled-task)) -;; (s/def ::scheduler-verticle-options -;; (s/keys :opt-un [::schedule])) -;; (defn scheduler-verticle -;; [options] -;; (s/assert ::scheduler-verticle-options options) -;; (let [on-start #(on-scheduler-start % options)] -;; (vc/verticle {:on-start on-start}))) -;; --- Schedule API +;; --- Submit API (s/def ::name ::us/string) (s/def ::delay @@ -290,7 +267,12 @@ values (?, ?, ?, ?, clock_timestamp()+cast(?::text as interval)) returning id") -(defn schedule! +(defn- duration->pginterval + [^Duration d] + (->> (/ (.toMillis d) 1000.0) + (format "%s seconds"))) + +(defn submit! [conn {:keys [name delay props queue key] :or {delay 0 props {} queue "default"} :as options}] @@ -299,9 +281,7 @@ pginterval (duration->pginterval duration) props (blob/encode props) id (uuid/next)] - (log/info "Schedule task" name - ;; "with props" (pr-str props) - "to be executed in" (str duration)) + (log/info "Submit task" name "to be executed in" (str duration)) (db/exec-one! conn [sql:insert-new-task id name props queue pginterval]) id)) diff --git a/backend/src/uxbox/tasks/remove_media.clj b/backend/src/uxbox/tasks/remove_media.clj index 3553e95073..0b438fdb8c 100644 --- a/backend/src/uxbox/tasks/remove_media.clj +++ b/backend/src/uxbox/tasks/remove_media.clj @@ -15,6 +15,7 @@ [uxbox.common.exceptions :as ex] [uxbox.common.spec :as us] [uxbox.media :as media] + [uxbox.metrics :as mtx] [uxbox.util.storage :as ust])) (s/def ::path ::us/not-empty-string) @@ -28,3 +29,7 @@ (ust/delete! media/media-storage (:path props)) (log/debug "Media " (:path props) " removed."))) +(mtx/instrument-with-summary! + {:var #'handler + :id "tasks__remove_media" + :help "Timing of remove-media task."}) diff --git a/backend/src/uxbox/tasks/sendmail.clj b/backend/src/uxbox/tasks/sendmail.clj index 2fbc9119da..63fcdf94bc 100644 --- a/backend/src/uxbox/tasks/sendmail.clj +++ b/backend/src/uxbox/tasks/sendmail.clj @@ -15,6 +15,7 @@ [uxbox.common.data :as d] [uxbox.common.exceptions :as ex] [uxbox.config :as cfg] + [uxbox.metrics :as mtx] [uxbox.util.http :as http])) (defmulti sendmail (fn [config email] (:sendmail-backend config))) @@ -94,3 +95,7 @@ [{:keys [props] :as task}] (sendmail cfg/config props)) +(mtx/instrument-with-summary! + {:var #'handler + :id "tasks__sendmail" + :help "Timing of sendmail task."}) diff --git a/backend/src/uxbox/util/dispatcher.clj b/backend/src/uxbox/util/dispatcher.clj index a187a67338..b41c1df7bf 100644 --- a/backend/src/uxbox/util/dispatcher.clj +++ b/backend/src/uxbox/util/dispatcher.clj @@ -20,22 +20,18 @@ (definterface IDispatcher (^void add [key f])) -(defn- wrap-handler - [items handler] - (reduce #(%2 %1) handler items)) - -(deftype Dispatcher [reg attr wrap-fns] +(deftype Dispatcher [reg attr wrap] IDispatcher (add [this key f] - (let [f (wrap-handler wrap-fns f)] - (.put ^Map reg key f) - this)) + (.put ^Map reg key (wrap f)) + this) + clojure.lang.IDeref (deref [_] {:registry reg :attr attr - :wrap-fns wrap-fns}) + :wrap wrap}) clojure.lang.IFn (invoke [_ params] @@ -100,36 +96,3 @@ `(do (s/assert dispatcher? ~sym) (add-method ~sym ~key ~f ~meta)))) - -(defn wrap-spec - [handler] - (let [mdata (meta handler) - spec (s/get-spec (:spec mdata))] - (if (nil? spec) - handler - (with-meta - (fn [params] - (let [result (s/conform spec params)] - (if (not= result ::s/invalid) - (handler result) - (let [data (s/explain-data spec params)] - (ex/raise :type :validation - :code :spec-validation - :explain (with-out-str - (expound/printer data)) - :data (::s/problems data)))))) - (assoc mdata ::wrap-spec true))))) - -(defn wrap-error - [handler] - (let [mdata (meta handler)] - (with-meta - (fn [params] - (try - (handler params) - (catch Throwable error - (ex/raise :type :service-error - :name (:spec mdata) - :cause error)))) - (assoc mdata ::wrap-error true)))) - diff --git a/backend/tests/user.clj b/backend/tests/user.clj index 9d112e12a8..1c0e9bae0b 100644 --- a/backend/tests/user.clj +++ b/backend/tests/user.clj @@ -19,11 +19,9 @@ [clojure.repl :refer :all] [criterium.core :refer [quick-bench bench with-progress-reporting]] [clj-kondo.core :as kondo] - [promesa.core :as p] - [promesa.exec :as px] [uxbox.migrations] [uxbox.db :as db] - ;; [uxbox.redis :as rd] + [uxbox.metrics :as mtx] [uxbox.util.storage :as st] [uxbox.util.time :as tm] [uxbox.util.blob :as blob] diff --git a/docker/devenv/docker-compose.yaml b/docker/devenv/docker-compose.yaml index e773d4bc48..cfbcc2985a 100644 --- a/docker/devenv/docker-compose.yaml +++ b/docker/devenv/docker-compose.yaml @@ -38,7 +38,6 @@ services: - 9090:9090 environment: - - CLOJURE_OPTS=-J-XX:-OmitStackTraceInFastThrow - UXBOX_DATABASE_URI=postgresql://postgres/uxbox - UXBOX_DATABASE_USERNAME=uxbox - UXBOX_DATABASE_PASSWORD=uxbox diff --git a/frontend/src/uxbox/main/data/workspace.cljs b/frontend/src/uxbox/main/data/workspace.cljs index 07ba516277..7f73465771 100644 --- a/frontend/src/uxbox/main/data/workspace.cljs +++ b/frontend/src/uxbox/main/data/workspace.cljs @@ -197,7 +197,6 @@ (defn initialize-viewport [{:keys [width height] :as size}] - (js/console.log "initialize-viewport" size) (ptk/reify ::initialize-viewport ptk/UpdateEvent (update [_ state] @@ -208,11 +207,7 @@ (update :vbox (fn [vbox] (if (nil? vbox) (assoc size :x 0 :y 0) - vbox))))))) - - ptk/WatchEvent - (watch [_ state stream] - #_(rx/of zoom-to-fit-all)))) + vbox))))))))) (defn update-viewport-position [{:keys [x y] :or {x identity y identity}}] @@ -852,7 +847,8 @@ shapes (map lookup selected) shape? #(not= (:type %) :frame)] - (rx/of (delete-shapes selected)))))) + (rx/of (delete-shapes selected) + deselect-all))))) ;; --- Rename Shape diff --git a/frontend/src/uxbox/main/data/workspace/notifications.cljs b/frontend/src/uxbox/main/data/workspace/notifications.cljs index c30aea24f6..2da564e32c 100644 --- a/frontend/src/uxbox/main/data/workspace/notifications.cljs +++ b/frontend/src/uxbox/main/data/workspace/notifications.cljs @@ -18,6 +18,8 @@ [uxbox.main.repo :as rp] [uxbox.main.store :as st] [uxbox.main.streams :as ms] + [uxbox.main.data.workspace.common :as dwc] + [uxbox.main.data.workspace.persistence :as dwp] [uxbox.util.avatars :as avatars] [uxbox.util.geom.point :as gpt] [uxbox.util.time :as dt] @@ -75,7 +77,6 @@ (ptk/reify ::send-keepalive ptk/EffectEvent (effect [_ state stream] - (prn "send-keepalive" file-id) (when-let [ws (get-in state [:ws file-id])] (ws/-send ws (t/encode {:type :keepalive})))))) @@ -165,13 +166,11 @@ (ws/-send ws (t/encode msg)))))) (defn handle-page-change - [{:keys [profile-id page-id revn operations] :as msg}] + [msg] (ptk/reify ::handle-page-change ptk/WatchEvent (watch [_ state stream] - #_(let [page-id' (get-in state [:workspace-page :id])] - (when (= page-id page-id') - (rx/of (shapes-changes-commited msg))))))) - + (rx/of (dwp/shapes-changes-persisted msg) + (dwc/update-page-indices (:page-id msg)))))) diff --git a/frontend/src/uxbox/main/data/workspace/persistence.cljs b/frontend/src/uxbox/main/data/workspace/persistence.cljs index 0c957bb712..29afd2700a 100644 --- a/frontend/src/uxbox/main/data/workspace/persistence.cljs +++ b/frontend/src/uxbox/main/data/workspace/persistence.cljs @@ -42,7 +42,7 @@ (let [stoper (rx/filter #(= ::finalize %) stream) notifier (->> stream (rx/filter (ptk/type? ::dwc/commit-changes)) - (rx/debounce 2000) + (rx/debounce 200) (rx/merge stoper))] (rx/merge (->> stream @@ -64,15 +64,13 @@ (ptk/reify ::persist-changes ptk/WatchEvent (watch [_ state stream] - (let [session-id (:session-id state) - page (get-in state [:workspace-pages page-id]) - changes (->> changes - (mapcat identity) - (map #(assoc % :session-id session-id)) - (vec)) - params {:id (:id page) - :revn (:revn page) - :changes changes}] + (let [sid (:session-id state) + page (get-in state [:workspace-pages page-id]) + changes (into [] (mapcat identity) changes) + params {:id (:id page) + :revn (:revn page) + :session-id sid + :changes changes}] (->> (rp/mutation :update-page params) (rx/map shapes-changes-persisted)))))) diff --git a/frontend/src/uxbox/main/ui/workspace/shapes/text.cljs b/frontend/src/uxbox/main/ui/workspace/shapes/text.cljs index fdb04b344e..3a41a73cf4 100644 --- a/frontend/src/uxbox/main/ui/workspace/shapes/text.cljs +++ b/frontend/src/uxbox/main/ui/workspace/shapes/text.cljs @@ -260,15 +260,16 @@ (fn [event] (mf/set-ref-val! selecting-ref false)) - on-keyup + on-key-up (fn [event] + (dom/stop-propagation event) (when (= (.-keyCode event) 27) ; ESC (on-close))) on-mount (fn [] (let [lkey1 (events/listen js/document EventType.CLICK on-click) - lkey2 (events/listen js/document EventType.KEYUP on-keyup)] + lkey2 (events/listen js/document EventType.KEYUP on-key-up)] (st/emit! (dwt/assign-editor id editor)) #(do (st/emit! (dwt/assign-editor id nil)) diff --git a/frontend/src/uxbox/main/ui/workspace/viewport.cljs b/frontend/src/uxbox/main/ui/workspace/viewport.cljs index 9632ec599d..4baa5c6fc1 100644 --- a/frontend/src/uxbox/main/ui/workspace/viewport.cljs +++ b/frontend/src/uxbox/main/ui/workspace/viewport.cljs @@ -33,6 +33,7 @@ [uxbox.main.ui.workspace.snap-feedback :refer [snap-feedback]] [uxbox.util.math :as mth] [uxbox.util.dom :as dom] + [uxbox.util.object :as obj] [uxbox.util.geom.point :as gpt] [uxbox.util.perf :as perf] [uxbox.common.uuid :as uuid]) @@ -162,10 +163,7 @@ (and (not edition) (= 2 (.-which event))) - (handle-viewport-positioning viewport-ref) - - :else - (js/console.log "on-mouse-down" event))))) + (handle-viewport-positioning viewport-ref))))) on-context-menu (mf/use-callback @@ -234,10 +232,13 @@ shift? (kbd/shift? event) opts {:key key :shift? shift? - :ctrl? ctrl?}] + :ctrl? ctrl?} + target (dom/get-target event)] + (when-not (.-repeat bevent) (st/emit! (ms/->KeyboardEvent :down key ctrl? shift?)) - (when (kbd/space? event) + (when (and (kbd/space? event) + (not= "rich-text" (obj/get target "className"))) (handle-viewport-positioning viewport-ref)))))) on-key-up diff --git a/sample_media/config.edn b/sample_media/config.edn index 83f01ec4eb..082c93749b 100644 --- a/sample_media/config.edn +++ b/sample_media/config.edn @@ -47,568 +47,467 @@ ] :colors - [{:name "UXBOX" - :id #uuid "00000000-0000-0000-0000-000000000001" - :colors #{"#78dbbe" - "#b6dd75" - "#a599c6" - "#e6a16f" - "#de4762" - "#59b9e2" - "#ffffff" - "#000000" - "#90969d" - "#D3D3D3" - "#C0C0C0" - "#A9A9A9" - "#DCDCDC" - "#808080" - "#696969"}} + [{:name "Flat design" + :id #uuid "00000000-0000-0000-0000-00000000001" + :colors ["turquoise-50" "#e8f8f5" + "turquoise-100" "#d1f2eb" + "turquoise-200" "#a3e4d7" + "turquoise-300" "#76d7c4" + "turquoise-400" "#48c9b0" + "turquoise-500" "#1abc9c" + "turquoise-600" "#17a589" + "turquoise-700" "#148f77" + "turquoise-800" "#117864" + "turquoise-900" "#0e6251" + "green-sea-50" "#e8f6f3" + "green-sea-100" "#d0ece7" + "green-sea-200" "#a2d9ce" + "green-sea-300" "#73c6b6" + "green-sea-400" "#45b39d" + "green-sea-500" "#16a085" + "green-sea-600" "#138d75" + "green-sea-700" "#117a65" + "green-sea-800" "#0e6655" + "green-sea-900" "#0b5345" + "emerald-50" "#eafaf1" + "emerald-100" "#d5f5e3" + "emerald-200" "#abebc6" + "emerald-300" "#82e0aa" + "emerald-400" "#58d68d" + "emerald-500" "#2ecc71" + "emerald-600" "#28b463" + "emerald-700" "#239b56" + "emerald-800" "#1d8348" + "emerald-900" "#186a3b" + "nephritis-50" "#e9f7ef" + "nephritis-100" "#d4efdf" + "nephritis-200" "#a9dfbf" + "nephritis-300" "#7dcea0" + "nephritis-400" "#52be80" + "nephritis-500" "#27ae60" + "nephritis-600" "#229954" + "nephritis-700" "#1e8449" + "nephritis-800" "#196f3d" + "nephritis-900" "#145a32" + "peter-river-50" "#ebf5fb" + "peter-river-100" "#d6eaf8" + "peter-river-200" "#aed6f1" + "peter-river-300" "#85c1e9" + "peter-river-400" "#5dade2" + "peter-river-500" "#3498db" + "peter-river-600" "#2e86c1" + "peter-river-700" "#2874a6" + "peter-river-800" "#21618c" + "peter-river-900" "#1b4f72" + "belize-hole-50" "#eaf2f8" + "belize-hole-100" "#d4e6f1" + "belize-hole-200" "#a9cce3" + "belize-hole-300" "#7fb3d5" + "belize-hole-400" "#5499c7" + "belize-hole-500" "#2980b9" + "belize-hole-600" "#2471a3" + "belize-hole-700" "#1f618d" + "belize-hole-800" "#1a5276" + "belize-hole-900" "#154360" + "amethyst-50" "#f5eef8" + "amethyst-100" "#ebdef0" + "amethyst-200" "#d7bde2" + "amethyst-300" "#c39bd3" + "amethyst-400" "#af7ac5" + "amethyst-500" "#9b59b6" + "amethyst-600" "#884ea0" + "amethyst-700" "#76448a" + "amethyst-800" "#633974" + "amethyst-900" "#512e5f" + "wisteria-50" "#f4ecf7" + "wisteria-100" "#e8daef" + "wisteria-200" "#d2b4de" + "wisteria-300" "#bb8fce" + "wisteria-400" "#a569bd" + "wisteria-500" "#8e44ad" + "wisteria-600" "#7d3c98" + "wisteria-700" "#6c3483" + "wisteria-800" "#5b2c6f" + "wisteria-900" "#4a235a" + "wet-asphalt-50" "#ebedef" + "wet-asphalt-100" "#d6dbdf" + "wet-asphalt-200" "#aeb6bf" + "wet-asphalt-300" "#85929e" + "wet-asphalt-400" "#5d6d7e" + "wet-asphalt-500" "#34495e" + "wet-asphalt-600" "#2e4053" + "wet-asphalt-700" "#283747" + "wet-asphalt-800" "#212f3c" + "wet-asphalt-900" "#1b2631" + "midnight-blue-50" "#eaecee" + "midnight-blue-100" "#d5d8dc" + "midnight-blue-200" "#abb2b9" + "midnight-blue-300" "#808b96" + "midnight-blue-400" "#566573" + "midnight-blue-500" "#2c3e50" + "midnight-blue-600" "#273746" + "midnight-blue-700" "#212f3d" + "midnight-blue-800" "#1c2833" + "midnight-blue-900" "#17202a" + "sunflower-50" "#fef9e7" + "sunflower-100" "#fcf3cf" + "sunflower-200" "#f9e79f" + "sunflower-300" "#f7dc6f" + "sunflower-400" "#f4d03f" + "sunflower-500" "#f1c40f" + "sunflower-600" "#d4ac0d" + "sunflower-700" "#b7950b" + "sunflower-800" "#9a7d0a" + "sunflower-900" "#7d6608" + "orange-50" "#fef5e7" + "orange-100" "#fdebd0" + "orange-200" "#fad7a0" + "orange-300" "#f8c471" + "orange-400" "#f5b041" + "orange-500" "#f39c12" + "orange-600" "#d68910" + "orange-700" "#b9770e" + "orange-800" "#9c640c" + "orange-900" "#7e5109" + "carrot-50" "#fdf2e9" + "carrot-100" "#fae5d3" + "carrot-200" "#f5cba7" + "carrot-300" "#f0b27a" + "carrot-400" "#eb984e" + "carrot-500" "#e67e22" + "carrot-600" "#ca6f1e" + "carrot-700" "#af601a" + "carrot-800" "#935116" + "carrot-900" "#784212" + "pumpkin-50" "#fbeee6" + "pumpkin-100" "#f6ddcc" + "pumpkin-200" "#edbb99" + "pumpkin-300" "#e59866" + "pumpkin-400" "#dc7633" + "pumpkin-500" "#d35400" + "pumpkin-600" "#ba4a00" + "pumpkin-700" "#a04000" + "pumpkin-800" "#873600" + "pumpkin-900" "#6e2c00" + "alizarin-50" "#fdedec" + "alizarin-100" "#fadbd8" + "alizarin-200" "#f5b7b1" + "alizarin-300" "#f1948a" + "alizarin-400" "#ec7063" + "alizarin-500" "#e74c3c" + "alizarin-600" "#cb4335" + "alizarin-700" "#b03a2e" + "alizarin-800" "#943126" + "alizarin-900" "#78281f" + "pomegranate-50" "#f9ebea" + "pomegranate-100" "#f2d7d5" + "pomegranate-200" "#e6b0aa" + "pomegranate-300" "#d98880" + "pomegranate-400" "#cd6155" + "pomegranate-500" "#c0392b" + "pomegranate-600" "#a93226" + "pomegranate-700" "#922b21" + "pomegranate-800" "#7b241c" + "pomegranate-900" "#641e16" + "clouds-50" "#fdfefe" + "clouds-100" "#fbfcfc" + "clouds-200" "#f7f9f9" + "clouds-300" "#f4f6f7" + "clouds-400" "#f0f3f4" + "clouds-500" "#ecf0f1" + "clouds-600" "#d0d3d4" + "clouds-700" "#b3b6b7" + "clouds-800" "#979a9a" + "clouds-900" "#7b7d7d" + "silver-50" "#f8f9f9" + "silver-100" "#f2f3f4" + "silver-200" "#e5e7e9" + "silver-300" "#d7dbdd" + "silver-400" "#cacfd2" + "silver-500" "#bdc3c7" + "silver-600" "#a6acaf" + "silver-700" "#909497" + "silver-800" "#797d7f" + "silver-900" "#626567" + "concrete-50" "#f4f6f6" + "concrete-100" "#eaeded" + "concrete-200" "#d5dbdb" + "concrete-300" "#bfc9ca" + "concrete-400" "#aab7b8" + "concrete-500" "#95a5a6" + "concrete-600" "#839192" + "concrete-700" "#717d7e" + "concrete-800" "#5f6a6a" + "concrete-900" "#4d5656" + "asbestos-50" "#f2f4f4" + "asbestos-100" "#e5e8e8" + "asbestos-200" "#ccd1d1" + "asbestos-300" "#b2babb" + "asbestos-400" "#99a3a4" + "asbestos-500" "#7f8c8d" + "asbestos-600" "#707b7c" + "asbestos-700" "#616a6b" + "asbestos-800" "#515a5a" + "asbestos-900" "#424949"]} - {:name "UXBOX (Light)" - :id #uuid "00000000-0000-0000-0000-000000000002" - :colors #{"#e9eaeb" - "#a6abb1" - "#90969d" - "#d7d9dc" - "#757a7f" - "#565a5e"}} - {:name "UXBOX (Dark)" - :id #uuid "00000000-0000-0000-0000-000000000003" - :colors #{"#2C2C2C" - "#3d3f40" - "#181818" - "#a9adaf" - "#808386" - "#4a4e52" - "#e0e6e9" - "#8d9496" - "#4e4f50" - "#878c8e"}} + {:name "Material design" + :id #uuid "00000000-0000-0000-0000-000000000020" + :colors ["red-50" "#ffebee" + "red-100" "#ffcdd2" + "red-200" "#ef9a9a" + "red-300" "#e57373" + "red-400" "#ef5350" + "red-500" "#f44336" + "red-600" "#e53935" + "red-700" "#d32f2f" + "red-800" "#c62828" + "red-900" "#b71c1c" + "red-a100" "#ff8a80" + "red-a200" "#ff5252" + "red-a400" "#ff1744" + "red-a700" "#d50000" + "pink-50" "#fce4ec" + "pink-100" "#f8bbd0" + "pink-200" "#f48fb1" + "pink-300" "#f06292" + "pink-400" "#ec407a" + "pink-500" "#e91e63" + "pink-600" "#d81b60" + "pink-700" "#c2185b" + "pink-800" "#ad1457" + "pink-900" "#880e4f" + "pink-a100" "#ff80ab" + "pink-a200" "#ff4081" + "pink-a400" "#f50057" + "pink-a700" "#c51162" + "purple-50" "#f3e5f5" + "purple-100" "#e1bee7" + "purple-200" "#ce93d8" + "purple-300" "#ba68c8" + "purple-400" "#ab47bc" + "purple-500" "#9c27b0" + "purple-600" "#8e24aa" + "purple-700" "#7b1fa2" + "purple-800" "#6a1b9a" + "purple-900" "#4a148c" + "purple-a100" "#ea80fc" + "purple-a200" "#e040fb" + "purple-a400" "#d500f9" + "purple-a700" "#aa00ff" + "deep-purple-50" "#ede7f6" + "deep-purple-100" "#d1c4e9" + "deep-purple-200" "#b39ddb" + "deep-purple-300" "#9575cd" + "deep-purple-400" "#7e57c2" + "deep-purple-500" "#673ab7" + "deep-purple-600" "#5e35b1" + "deep-purple-700" "#512da8" + "deep-purple-800" "#4527a0" + "deep-purple-900" "#311b92" + "deep-purple-a100" "#b388ff" + "deep-purple-a200" "#7c4dff" + "deep-purple-a400" "#651fff" + "deep-purple-a700" "#6200ea" + "indigo-50" "#e8eaf6" + "indigo-100" "#c5cae9" + "indigo-200" "#9fa8da" + "indigo-300" "#7986cb" + "indigo-400" "#5c6bc0" + "indigo-500" "#3f51b5" + "indigo-600" "#3949ab" + "indigo-700" "#303f9f" + "indigo-800" "#283593" + "indigo-900" "#1a237e" + "indigo-a100" "#8c9eff" + "indigo-a200" "#536dfe" + "indigo-a400" "#3d5afe" + "indigo-a700" "#304ffe" + "blue-50" "#e3f2fd" + "blue-100" "#bbdefb" + "blue-200" "#90caf9" + "blue-300" "#64b5f6" + "blue-400" "#42a5f5" + "blue-500" "#2196f3" + "blue-600" "#1e88e5" + "blue-700" "#1976d2" + "blue-800" "#1565c0" + "blue-900" "#0d47a1" + "blue-a100" "#82b1ff" + "blue-a200" "#448aff" + "blue-a400" "#2979ff" + "blue-a700" "#2962ff" + "light-blue-50" "#e1f5fe" + "light-blue-100" "#b3e5fc" + "light-blue-200" "#81d4fa" + "light-blue-300" "#4fc3f7" + "light-blue-400" "#29b6f6" + "light-blue-500" "#03a9f4" + "light-blue-600" "#039be5" + "light-blue-700" "#0288d1" + "light-blue-800" "#0277bd" + "light-blue-900" "#01579b" + "light-blue-a100" "#80d8ff" + "light-blue-a200" "#40c4ff" + "light-blue-a400" "#00b0ff" + "light-blue-a700" "#0091ea" + "cyan-50" "#e0f7fa" + "cyan-100" "#b2ebf2" + "cyan-200" "#80deea" + "cyan-300" "#4dd0e1" + "cyan-400" "#26c6da" + "cyan-500" "#00bcd4" + "cyan-600" "#00acc1" + "cyan-700" "#0097a7" + "cyan-800" "#00838f" + "cyan-900" "#006064" + "cyan-a100" "#84ffff" + "cyan-a200" "#18ffff" + "cyan-a400" "#00e5ff" + "cyan-a700" "#00b8d4" + "teal-50" "#e0f2f1" + "teal-100" "#b2dfdb" + "teal-200" "#80cbc4" + "teal-300" "#4db6ac" + "teal-400" "#26a69a" + "teal-500" "#009688" + "teal-600" "#00897b" + "teal-700" "#00796b" + "teal-800" "#00695c" + "teal-900" "#004d40" + "teal-a100" "#a7ffeb" + "teal-a200" "#64ffda" + "teal-a400" "#1de9b6" + "teal-a700" "#00bfa5" + "green-50" "#e8f5e9" + "green-100" "#c8e6c9" + "green-200" "#a5d6a7" + "green-300" "#81c784" + "green-400" "#66bb6a" + "green-500" "#4caf50" + "green-600" "#43a047" + "green-700" "#388e3c" + "green-800" "#2e7d32" + "green-900" "#1b5e20" + "green-a100" "#b9f6ca" + "green-a200" "#69f0ae" + "green-a400" "#00e676" + "green-a700" "#00c853" + "light-green-50" "#f1f8e9" + "light-green-100" "#dcedc8" + "light-green-200" "#c5e1a5" + "light-green-300" "#aed581" + "light-green-400" "#9ccc65" + "light-green-500" "#8bc34a" + "light-green-600" "#7cb342" + "light-green-700" "#689f38" + "light-green-800" "#558b2f" + "light-green-900" "#33691e" + "light-green-a100" "#ccff90" + "light-green-a200" "#b2ff59" + "light-green-a400" "#76ff03" + "light-green-a700" "#64dd17" + "lime-50" "#f9fbe7" + "lime-100" "#f0f4c3" + "lime-200" "#e6ee9c" + "lime-300" "#dce775" + "lime-400" "#d4e157" + "lime-500" "#cddc39" + "lime-600" "#c0ca33" + "lime-700" "#afb42b" + "lime-800" "#9e9d24" + "lime-900" "#827717" + "lime-a100" "#f4ff81" + "lime-a200" "#eeff41" + "lime-a400" "#c6ff00" + "lime-a700" "#aeea00" + "yellow-50" "#fffde7" + "yellow-100" "#fff9c4" + "yellow-200" "#fff59d" + "yellow-300" "#fff176" + "yellow-400" "#ffee58" + "yellow-500" "#ffeb3b" + "yellow-600" "#fdd835" + "yellow-700" "#fbc02d" + "yellow-800" "#f9a825" + "yellow-900" "#f57f17" + "yellow-a100" "#ffff8d" + "yellow-a200" "#ffff00" + "yellow-a400" "#ffea00" + "yellow-a700" "#ffd600" + "amber-50" "#fff8e1" + "amber-100" "#ffecb3" + "amber-200" "#ffe082" + "amber-300" "#ffd54f" + "amber-400" "#ffca28" + "amber-500" "#ffc107" + "amber-600" "#ffb300" + "amber-700" "#ffa000" + "amber-800" "#ff8f00" + "amber-900" "#ff6f00" + "amber-a100" "#ffe57f" + "amber-a200" "#ffd740" + "amber-a400" "#ffc400" + "amber-a700" "#ffab00" + "orange-50" "#fff3e0" + "orange-100" "#ffe0b2" + "orange-200" "#ffcc80" + "orange-300" "#ffb74d" + "orange-400" "#ffa726" + "orange-500" "#ff9800" + "orange-600" "#fb8c00" + "orange-700" "#f57c00" + "orange-800" "#ef6c00" + "orange-900" "#e65100" + "orange-a100" "#ffd180" + "orange-a200" "#ffab40" + "orange-a400" "#ff9100" + "orange-a700" "#ff6d00" + "deep-orange-50" "#fbe9e7" + "deep-orange-100" "#ffccbc" + "deep-orange-200" "#ffab91" + "deep-orange-300" "#ff8a65" + "deep-orange-400" "#ff7043" + "deep-orange-500" "#ff5722" + "deep-orange-600" "#f4511e" + "deep-orange-700" "#e64a19" + "deep-orange-800" "#d84315" + "deep-orange-900" "#bf360c" + "deep-orange-a100" "#ff9e80" + "deep-orange-a200" "#ff6e40" + "deep-orange-a400" "#ff3d00" + "deep-orange-a700" "#dd2c00" + "brown-50" "#efebe9" + "brown-100" "#d7ccc8" + "brown-200" "#bcaaa4" + "brown-300" "#a1887f" + "brown-400" "#8d6e63" + "brown-500" "#795548" + "brown-600" "#6d4c41" + "brown-700" "#5d4037" + "brown-800" "#4e342e" + "brown-900" "#3e2723" + "grey-50" "#fafafa" + "grey-100" "#f5f5f5" + "grey-200" "#eeeeee" + "grey-300" "#e0e0e0" + "grey-400" "#bdbdbd" + "grey-500" "#9e9e9e" + "grey-600" "#757575" + "grey-700" "#616161" + "grey-800" "#424242" + "grey-900" "#212121" + "blue-grey-50" "#eceff1" + "blue-grey-100" "#cfd8dc" + "blue-grey-200" "#b0bec5" + "blue-grey-300" "#90a4ae" + "blue-grey-400" "#78909c" + "blue-grey-500" "#607d8b" + "blue-grey-600" "#546e7a" + "blue-grey-700" "#455a64" + "blue-grey-800" "#37474f" + "blue-grey-900" "#263238" + "white" "#ffffff" + "black" "#000000"]} + ]} - {:name "UXBOX (Blues)" - :id #uuid "00000000-0000-0000-0000-000000000004" - :colors #{"#F0F8FF" - "#E6E6FA" - "#B0E0E6" - "#ADD8E6" - "#87CEFA" - "#87CEEB" - "#00BFFF" - "#B0C4DE" - "#1E90FF" - "#6495ED" - "#4682B4" - "#5F9EA0" - "#7B68EE" - "#6A5ACD" - "#483D8B" - "#4169E1" - "#0000FF" - "#0000CD" - "#00008B" - "#000080" - "#191970" - "#8A2BE2" - "#4B0082"}} - - ;; https://github.com/twbs/bootstrap - {:name "Bootstrap" - :id #uuid "00000000-0000-0000-0000-000000000005" - :colors #{"#ffffff" - "#f8f9fa" - "#e9ecef" - "#dee2e6" - "#ced4da" - "#adb5bd" - "#6c757d" - "#495057" - "#343a40" - "#212529" - "#000000" - "#007bff" - "#6610f2" - "#6f42c1" - "#e83e8c" - "#dc3545" - "#fd7e14" - "#ffc107" - "#28a745" - "#20c997" - "#17a2b8"}} - - ;; https://material.io/design/color/the-color-system.html#tools-for-picking-colors - {:name "Material Design (Red)" - :id #uuid "00000000-0000-0000-0000-000000000006" - :colors #{"#ffebee" - "#ffcdd2" - "#ef9a9a" - "#e57373" - "#ef5350" - "#f44336" - "#e53935" - "#d32f2f" - "#c62828" - "#b71c1c" - "#ff8a80" - "#ff5252" - "#ff1744" - "#d50000"}} - - ;; https://material.io/design/color/the-color-system.html#tools-for-picking-colors - {:name "Material Design (Pink)" - :id #uuid "00000000-0000-0000-0000-000000000007" - :colors #{"#fce4ec" - "#f8bbd0" - "#f48fb1" - "#f06292" - "#ec407a" - "#e91e63" - "#d81b60" - "#c2185b" - "#ad1457" - "#880e4f" - "#ff80ab" - "#ff4081" - "#f50057" - "#c51162"}} - - ;; https://material.io/design/color/the-color-system.html#tools-for-picking-colors - {:name "Material Design (Purple)" - :id #uuid "00000000-0000-0000-0000-000000000008" - :colors #{"#f3e5f5" - "#e1bee7" - "#ce93d8" - "#ba68c8" - "#ab47bc" - "#9c27b0" - "#8e24aa" - "#7b1fa2" - "#6a1b9a" - "#4a148c" - "#ea80fc" - "#e040fb" - "#d500f9" - "#aa00ff"}} - - ;; https://material.io/design/color/the-color-system.html#tools-for-picking-colors - {:name "Material Design (Deep Purple)" - :id #uuid "00000000-0000-0000-0000-000000000009" - :colors #{"#ede7f6" - "#d1c4e9" - "#b39ddb" - "#9575cd" - "#7e57c2" - "#673ab7" - "#5e35b1" - "#512da8" - "#4527a0" - "#311b92" - "#b388ff" - "#7c4dff" - "#651fff" - "#6200ea"}} - - ;; https://material.io/design/color/the-color-system.html#tools-for-picking-colors - {:name "Material Design (Indigo)" - :id #uuid "00000000-0000-0000-0000-000000000010" - :colors #{"#e8eaf6" - "#c5cae9" - "#9fa8da" - "#7986cb" - "#5c6bc0" - "#3f51b5" - "#3949ab" - "#303f9f" - "#283593" - "#1a237e" - "#8c9eff" - "#536dfe" - "#3d5afe" - "#304ffe"}} - - ;; https://material.io/design/color/the-color-system.html#tools-for-picking-colors - {:name "Material Design (Blue)" - :id #uuid "00000000-0000-0000-0000-000000000011" - :colors #{"#e3f2fd" - "#bbdefb" - "#90caf9" - "#64b5f6" - "#42a5f5" - "#2196f3" - "#1e88e5" - "#1976d2" - "#1565c0" - "#0d47a1" - "#82b1ff" - "#448aff" - "#2979ff" - "#2962ff"}} - - ;; https://material.io/design/color/the-color-system.html#tools-for-picking-colors - {:name "Material Design (Light Blue)" - :id #uuid "00000000-0000-0000-0000-000000000012" - :colors #{"#e1f5fe" - "#b3e5fc" - "#81d4fa" - "#4fc3f7" - "#29b6f6" - "#03a9f4" - "#039be5" - "#0288d1" - "#0277bd" - "#01579b" - "#80d8ff" - "#40c4ff" - "#00b0ff" - "#0091ea"}} - - ;; https://material.io/design/color/the-color-system.html#tools-for-picking-colors - {:name "Material Design (Cyan)" - :id #uuid "00000000-0000-0000-0000-000000000013" - :colors #{"#e0f7fa" - "#b2ebf2" - "#80deea" - "#4dd0e1" - "#26c6da" - "#00bcd4" - "#00acc1" - "#0097a7" - "#00838f" - "#006064" - "#84ffff" - "#18ffff" - "#00e5ff" - "#00b8d4"}} - - ;; https://material.io/design/color/the-color-system.html#tools-for-picking-colors - {:name "Material Design (Teal)" - :id #uuid "00000000-0000-0000-0000-000000000014" - :colors #{"#e0f2f1" - "#b2dfdb" - "#80cbc4" - "#4db6ac" - "#26a69a" - "#009688" - "#00897b" - "#00796b" - "#00695c" - "#004d40" - "#a7ffeb" - "#64ffda" - "#1de9b6" - "#00bfa5"}} - - ;; https://material.io/design/color/the-color-system.html#tools-for-picking-colors - {:name "Material Design (Green)" - :id #uuid "00000000-0000-0000-0000-000000000015" - :colors #{"#e8f5e9" - "#c8e6c9" - "#a5d6a7" - "#81c784" - "#66bb6a" - "#4caf50" - "#43a047" - "#388e3c" - "#2e7d32" - "#1b5e20" - "#b9f6ca" - "#69f0ae" - "#00e676" - "#00c853"}} - - ;; https://material.io/design/color/the-color-system.html#tools-for-picking-colors - {:name "Material Design (Light Green)" - :id #uuid "00000000-0000-0000-0000-000000000016" - :colors #{"#f1f8e9" - "#dcedc8" - "#c5e1a5" - "#aed581" - "#9ccc65" - "#8bc34a" - "#7cb342" - "#689f38" - "#558b2f" - "#33691e" - "#ccff90" - "#b2ff59" - "#76ff03" - "#64dd17"}} - - ;; https://material.io/design/color/the-color-system.html#tools-for-picking-colors - {:name "Material Design (Lime)" - :id #uuid "00000000-0000-0000-0000-000000000017" - :colors #{"#f9fbe7" - "#f0f4c3" - "#e6ee9c" - "#dce775" - "#d4e157" - "#cddc39" - "#c0ca33" - "#afb42b" - "#9e9d24" - "#827717" - "#f4ff81" - "#eeff41" - "#c6ff00" - "#aeea00"}} - - ;; https://material.io/design/color/the-color-system.html#tools-for-picking-colors - {:name "Material Design (Yellow)" - :id #uuid "00000000-0000-0000-0000-000000000018" - :colors #{"#fffde7" - "#fff9c4" - "#fff59d" - "#fff176" - "#ffee58" - "#ffeb3b" - "#fdd835" - "#fbc02d" - "#f9a825" - "#f57f17" - "#ffff8d" - "#ffff00" - "#ffea00" - "#ffd600"}} - - ;; https://material.io/design/color/the-color-system.html#tools-for-picking-colors - {:name "Material Design (Amber)" - :id #uuid "00000000-0000-0000-0000-000000000019" - :colors #{"#fff8e1" - "#ffecb3" - "#ffe082" - "#ffd54f" - "#ffca28" - "#ffc107" - "#ffb300" - "#ffa000" - "#ff8f00" - "#ff6f00" - "#ffe57f" - "#ffd740" - "#ffc400" - "#ffab00"}} - - ;; https://material.io/design/color/the-color-system.html#tools-for-picking-colors - {:name "Material Design (Orange)" - :id #uuid "00000000-0000-0000-0000-000000000020" - :colors #{"#fff3e0" - "#ffe0b2" - "#ffcc80" - "#ffb74d" - "#ffa726" - "#ff9800" - "#fb8c00" - "#f57c00" - "#ef6c00" - "#e65100" - "#ffd180" - "#ffab40" - "#ff9100" - "#ff6d00"}} - - ;; https://material.io/design/color/the-color-system.html#tools-for-picking-colors - {:name "Material Design (Deep Orange)" - :id #uuid "00000000-0000-0000-0000-000000000021" - :colors #{"#fbe9e7" - "#ffccbc" - "#ffab91" - "#ff8a65" - "#ff7043" - "#ff5722" - "#f4511e" - "#e64a19" - "#d84315" - "#bf360c" - "#ff9e80" - "#ff6e40" - "#ff3d00" - "#dd2c00"}} - - ;; https://material.io/design/color/the-color-system.html#tools-for-picking-colors - {:name "Material Design (Brown)" - :id #uuid "00000000-0000-0000-0000-000000000022" - :colors #{"#efebe9" - "#d7ccc8" - "#bcaaa4" - "#a1887f" - "#8d6e63" - "#795548" - "#6d4c41" - "#5d4037" - "#4e342e" - "#3e2723"}} - - ;; https://material.io/design/color/the-color-system.html#tools-for-picking-colors - {:name "Material Design (Gray)" - :id #uuid "00000000-0000-0000-0000-000000000023" - :colors #{"#fafafa" - "#f5f5f5" - "#eeeeee" - "#e0e0e0" - "#bdbdbd" - "#9e9e9e" - "#757575" - "#616161" - "#424242" - "#212121"}} - - ;; https://material.io/design/color/the-color-system.html#tools-for-picking-colors - {:name "Material Design (Blue Gray)" - :id #uuid "00000000-0000-0000-0000-000000000024" - :colors #{"#eceff1" - "#cfd8dc" - "#b0bec5" - "#90a4ae" - "#78909c" - "#607d8b" - "#546e7a" - "#455a64" - "#37474f" - "#263238"}} - - ;; https://github.com/redpik/social-media-colors - {:name "Social Media (FACEBOOK)" - :id #uuid "00000000-0000-0000-0000-000000000025" - :colors #{"#3B5998" - "#D8DFEA" - "#F03D25"}} - - ;; https://github.com/redpik/social-media-colors - {:name "Social Media (TWITTER)" - :id #uuid "00000000-0000-0000-0000-000000000026" - :colors #{"#55ACEE"}} - - ;; https://github.com/redpik/social-media-colors - {:name "Social Media (WHATSAPP)" - :id #uuid "00000000-0000-0000-0000-000000000027" - :colors #{"#00e676" - "#1ebea5"}} - - ;; https://github.com/redpik/social-media-colors - {:name "Social Media (LINKEDIN)" - :id #uuid "00000000-0000-0000-0000-000000000028" - :colors #{"#0976B4"}} - - ;; https://github.com/redpik/social-media-colors - {:name "Social Media (PINTEREST)" - :id #uuid "00000000-0000-0000-0000-000000000029" - :colors #{"#cd1d1f"}} - - ;; https://github.com/redpik/social-media-colors - {:name "Social Media (FOURSQUARE)" - :id #uuid "00000000-0000-0000-0000-000000000030" - :colors #{"#0072b1" - "#0cbadf" - "#8fd400" - "#ff7900"}} - - ;; https://github.com/redpik/social-media-colors - {:name "Social Media (YOUTUBE)" - :id #uuid "00000000-0000-0000-0000-000000000031" - :colors #{"#CC181E" - "#110f10"}} - - ;; https://github.com/redpik/social-media-colors - {:name "Social Media (INSTAGRAM)" - :id #uuid "00000000-0000-0000-0000-000000000032" - :colors #{"#3F729B"}} - - ;; https://github.com/redpik/social-media-colors - {:name "Social Media (FLICKR)" - :id #uuid "00000000-0000-0000-0000-000000000033" - :colors #{"#0063dd" - "#ff0085"}} - - ;; https://github.com/redpik/social-media-colors - {:name "Social Media (VIMEO)" - :id #uuid "00000000-0000-0000-0000-000000000034" - :colors #{"#1ab7ea" - "#162221"}} - - ;; https://github.com/redpik/social-media-colors - {:name "Social Media (TUMBLR)" - :id #uuid "00000000-0000-0000-0000-000000000035" - :colors #{"#35465d"}} - - ;; https://github.com/redpik/social-media-colors - {:name "Social Media (STUMBLEUPON)" - :id #uuid "00000000-0000-0000-0000-000000000036" - :colors #{"#EB4924" - "#333333"}} - - ;; https://github.com/redpik/social-media-colors - {:name "Social Media (MYSPACE)" - :id #uuid "00000000-0000-0000-0000-000000000037" - :colors #{"#008DDE" - "#313131" - "#1D1D1D"}} - - ;; https://github.com/redpik/social-media-colors - {:name "Social Media (DRIBBBLE)" - :id #uuid "00000000-0000-0000-0000-000000000038" - :colors #{"#ea4c89" - "#444444" - "#8aba56"}} - - ;; https://github.com/redpik/social-media-colors - {:name "Social Media (DAILYMOTION)" - :id #uuid "00000000-0000-0000-0000-000000000039" - :colors #{"#0079b8" - "#fed417"}} - - ;; https://github.com/redpik/social-media-colors - {:name "Social Media (DELICIOUS)" - :id #uuid "00000000-0000-0000-0000-000000000040" - :colors #{"#3274D0" - "#D3D2D2" - "#222222" - "#0B79E5"}} - - ;; https://github.com/redpik/social-media-colors - {:name "Social Media (SOUNDCLOUD)" - :id #uuid "00000000-0000-0000-0000-000000000041" - :colors #{"#FF3D00"}} - - ;; https://github.com/redpik/social-media-colors - {:name "Social Media (GITHUB)" - :id #uuid "00000000-0000-0000-0000-000000000042" - :colors #{"#171515" - "#4183c4"}} - - ;; https://github.com/redpik/social-media-colors - {:name "Social Media (QUORA)" - :id #uuid "00000000-0000-0000-0000-000000000043" - :colors #{"#BC2016"}} - - ;; https://github.com/redpik/social-media-colors - {:name "Social Media (BEHANCE)" - :id #uuid "00000000-0000-0000-0000-000000000044" - :colors #{"#1769FF" - "#242424"}} - - ;; https://github.com/redpik/social-media-colors - {:name "Social Media (VINE)" - :id #uuid "00000000-0000-0000-0000-000000000045" - :colors #{"#00B489"}} - - ;; https://github.com/redpik/social-media-colors - {:name "Social Media (VKONTAKTE)" - :id #uuid "00000000-0000-0000-0000-000000000046" - :colors #{"#587fa4" - "#e9edf1"}} - - ;; https://github.com/redpik/social-media-colors - {:name "Social Media (REDDIT)" - :id #uuid "00000000-0000-0000-0000-000000000047" - :colors #{"#F64720" - "#CEE3F8"}} - - ;; https://github.com/redpik/social-media-colors - {:name "Social Media (DROPBOX)" - :id #uuid "00000000-0000-0000-0000-000000000048" - :colors #{"#007ee5" - "#7B8994" - "#47525D" - "#3D464D"}}]}