diff --git a/README.md b/README.md index 1cd4d69..d7820b0 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,13 @@ Speed up your template development with our template workbench tool: - Docker and Docker Compose - Git - Make (recommended, the development workflow is built around make) +- Enough memory for the first build. The first start compiles the whole backend + dependency graph and installs the frontend, which is memory heavy. On a machine + with limited memory these two steps running at once can be killed by the OOM + killer, showing up as `signal: killed` on the backend and `vite: not found` on the + frontend. On such machines start with `make up-low-mem`, which builds the backend + before starting the frontend so the heavy steps do not overlap. Otherwise use the + normal `make up`, which builds everything in parallel for the fastest startup. ### Quick Start diff --git a/backend/database/factory.go b/backend/database/factory.go index 3c574d2..8639566 100644 --- a/backend/database/factory.go +++ b/backend/database/factory.go @@ -2,6 +2,8 @@ package database import ( "fmt" + "os" + "path/filepath" "strings" "github.com/phishingclub/phishingclub/config" @@ -17,6 +19,14 @@ func FromConfig(conf config.Config) (*gorm.DB, error) { switch conf.Database().Engine { case config.DefaultAdministrationUseSqlite: var err error + // ensure the directory holding the sqlite file exists, sqlite creates + // the database file but not its parent directory, so a fresh checkout + // where the data directory is absent fails with "unable to open database file" + if dir := sqliteDir(conf.Database().DSN); dir != "" { + if err := os.MkdirAll(dir, 0o750); err != nil { + return nil, errs.Wrap(err) + } + } // determine the correct separator for additional parameters // use & if user already has query params, otherwise use ? separator := "?" @@ -49,3 +59,22 @@ func FromConfig(conf config.Config) (*gorm.DB, error) { } return db, nil } + +// sqliteDir returns the directory that must exist for the sqlite DSN. +// it returns an empty string for in memory databases or when the path has no +// directory component, in which case no directory needs to be created. +func sqliteDir(dsn string) string { + path := strings.TrimPrefix(dsn, "file:") + // drop any query parameters + if i := strings.Index(path, "?"); i != -1 { + path = path[:i] + } + if path == "" || strings.Contains(path, ":memory:") { + return "" + } + dir := filepath.Dir(path) + if dir == "." || dir == "/" { + return "" + } + return dir +} diff --git a/docker-compose.low-mem.yml b/docker-compose.low-mem.yml new file mode 100644 index 0000000..6eae13a --- /dev/null +++ b/docker-compose.low-mem.yml @@ -0,0 +1,26 @@ +# Overrides for machines with limited memory. +# Use together with the base file, or run `make up-low-mem`: +# docker compose -f docker-compose.yml -f docker-compose.low-mem.yml up -d +# +# The first start runs two memory heavy steps, the backend Go build and the frontend +# install. Running them at the same time can be killed by the OOM killer (shows up as +# "signal: killed" on the backend and "vite: not found" on the frontend). These +# overrides serialize them: the frontend waits until the backend build has produced +# its binary, and the backend build parallelism is capped to lower its peak memory. +# The base file is unchanged, so the normal `make up` keeps full build parallelism. +services: + backend: + environment: + - GOMAXPROCS=2 + # healthy once air has produced the compiled binary, meaning the Go build finished + healthcheck: + test: ["CMD-SHELL", "test -x /app/.dev-air/platform"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 600s + + frontend: + depends_on: + backend: + condition: service_healthy diff --git a/docker-compose.yml b/docker-compose.yml index 676f42d..8f1e3b3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,7 @@ services: dockerfile: ./Dockerfile # the acme certs are deleted because the acme server does not store the account # and mouting them again would cause it to error because the account is not found - command: /bin/bash -c 'rm -rf /app/.dev/certs/acme/* && air -c /app/.air.docker.toml' + command: /bin/bash -c 'mkdir -p /app/.dev/certs/acme && rm -rf /app/.dev/certs/acme/* && air -c /app/.air.docker.toml' volumes: - ./backend:/app environment: diff --git a/frontend/dev.sh b/frontend/dev.sh index 86bfd26..7bc3430 100755 --- a/frontend/dev.sh +++ b/frontend/dev.sh @@ -1,6 +1,9 @@ #/bin/bash -# if node_modules folder does not exist, install dependencies -if [ ! -d "node_modules" ]; then +# install dependencies when the vite binary is missing. +# checking for the binary instead of the node_modules directory means a partial +# install (for example one killed by the OOM killer) is retried instead of being +# skipped forever, which would leave the dev server crash looping on "vite: not found" +if [ ! -x "node_modules/.bin/vite" ]; then npm install fi while true; do \ diff --git a/makefile b/makefile index b0a6683..c6719ff 100644 --- a/makefile +++ b/makefile @@ -1,4 +1,4 @@ -.PHONY: build down up fix-tls backend-purge backend-down purge logs backend-password dbgate-down dbgate-up geoip-fetch govulncheck +.PHONY: build down up up-low-mem fix-tls backend-purge backend-down purge logs backend-password dbgate-down dbgate-up geoip-fetch govulncheck up: sudo docker compose up -d backend frontend api-test-server pebble dbgate mailer dozzle stats dns test mitmproxy; \ sudo docker compose logs -f --tail 1000 backend frontend; @@ -6,6 +6,11 @@ down: -sudo docker compose down --remove-orphans up-build: sudo docker compose up --build --force +# same as up but for machines with limited memory, the frontend waits for the backend +# build to finish so the two heavy first build steps do not run at the same time +up-low-mem: + sudo docker compose -f docker-compose.yml -f docker-compose.low-mem.yml up -d backend frontend api-test-server pebble dbgate mailer dozzle stats dns test mitmproxy; \ + sudo docker compose -f docker-compose.yml -f docker-compose.low-mem.yml logs -f --tail 1000 backend frontend; up-reset: down purge up restart: down up prune: