update build for lower mem

Signed-off-by: Ronni Skansing <rskansing@gmail.com>

README.md

@@ -101,6 +101,13 @@ Speed up your template development with our template workbench tool:
 - Docker and Docker Compose
 - Git
 - Make (recommended, the development workflow is built around make)
+- Enough memory for the first build. The first start compiles the whole backend
+  dependency graph and installs the frontend, which is memory heavy. On a machine
+  with limited memory these two steps running at once can be killed by the OOM
+  killer, showing up as `signal: killed` on the backend and `vite: not found` on the
+  frontend. On such machines start with `make up-low-mem`, which builds the backend
+  before starting the frontend so the heavy steps do not overlap. Otherwise use the
+  normal `make up`, which builds everything in parallel for the fastest startup.
 
 ### Quick Start
 

backend/database/factory.go

@@ -2,6 +2,8 @@ package database
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/phishingclub/phishingclub/config"
@@ -17,6 +19,14 @@ func FromConfig(conf config.Config) (*gorm.DB, error) {
 	switch conf.Database().Engine {
 	case config.DefaultAdministrationUseSqlite:
 		var err error
+		// ensure the directory holding the sqlite file exists, sqlite creates
+		// the database file but not its parent directory, so a fresh checkout
+		// where the data directory is absent fails with "unable to open database file"
+		if dir := sqliteDir(conf.Database().DSN); dir != "" {
+			if err := os.MkdirAll(dir, 0o750); err != nil {
+				return nil, errs.Wrap(err)
+			}
+		}
 		// determine the correct separator for additional parameters
 		// use & if user already has query params, otherwise use ?
 		separator := "?"
@@ -49,3 +59,22 @@ func FromConfig(conf config.Config) (*gorm.DB, error) {
 	}
 	return db, nil
 }
+
+// sqliteDir returns the directory that must exist for the sqlite DSN.
+// it returns an empty string for in memory databases or when the path has no
+// directory component, in which case no directory needs to be created.
+func sqliteDir(dsn string) string {
+	path := strings.TrimPrefix(dsn, "file:")
+	// drop any query parameters
+	if i := strings.Index(path, "?"); i != -1 {
+		path = path[:i]
+	}
+	if path == "" || strings.Contains(path, ":memory:") {
+		return ""
+	}
+	dir := filepath.Dir(path)
+	if dir == "." || dir == "/" {
+		return ""
+	}
+	return dir
+}

docker-compose.low-mem.yml

@@ -0,0 +1,26 @@
+# Overrides for machines with limited memory.
+# Use together with the base file, or run `make up-low-mem`:
+#   docker compose -f docker-compose.yml -f docker-compose.low-mem.yml up -d
+#
+# The first start runs two memory heavy steps, the backend Go build and the frontend
+# install. Running them at the same time can be killed by the OOM killer (shows up as
+# "signal: killed" on the backend and "vite: not found" on the frontend). These
+# overrides serialize them: the frontend waits until the backend build has produced
+# its binary, and the backend build parallelism is capped to lower its peak memory.
+# The base file is unchanged, so the normal `make up` keeps full build parallelism.
+services:
+  backend:
+    environment:
+      - GOMAXPROCS=2
+    # healthy once air has produced the compiled binary, meaning the Go build finished
+    healthcheck:
+      test: ["CMD-SHELL", "test -x /app/.dev-air/platform"]
+      interval: 5s
+      timeout: 3s
+      retries: 3
+      start_period: 600s
+
+  frontend:
+    depends_on:
+      backend:
+        condition: service_healthy

docker-compose.yml

@@ -13,7 +13,7 @@ services:
       dockerfile: ./Dockerfile
     # the acme certs are deleted because the acme server does not store the account
     # and mouting them again would cause it to error because the account is not found
-    command: /bin/bash -c 'rm -rf /app/.dev/certs/acme/* && air -c /app/.air.docker.toml'
+    command: /bin/bash -c 'mkdir -p /app/.dev/certs/acme && rm -rf /app/.dev/certs/acme/* && air -c /app/.air.docker.toml'
     volumes:
       - ./backend:/app
     environment:

frontend/dev.sh

@@ -1,6 +1,9 @@
 #/bin/bash
-# if node_modules folder does not exist, install dependencies
-if [ ! -d "node_modules" ]; then
+# install dependencies when the vite binary is missing.
+# checking for the binary instead of the node_modules directory means a partial
+# install (for example one killed by the OOM killer) is retried instead of being
+# skipped forever, which would leave the dev server crash looping on "vite: not found"
+if [ ! -x "node_modules/.bin/vite" ]; then
 	npm install
 fi
 while true; do \

makefile

@@ -1,4 +1,4 @@
-.PHONY: build down up fix-tls backend-purge backend-down purge logs backend-password dbgate-down dbgate-up geoip-fetch govulncheck
+.PHONY: build down up up-low-mem fix-tls backend-purge backend-down purge logs backend-password dbgate-down dbgate-up geoip-fetch govulncheck
 up:
 	sudo docker compose up -d backend frontend api-test-server pebble dbgate mailer dozzle stats dns test mitmproxy; \
 	sudo docker compose logs -f --tail 1000 backend frontend;
@@ -6,6 +6,11 @@ down:
 	-sudo docker compose down --remove-orphans
 up-build:
 	sudo docker compose up --build --force
+# same as up but for machines with limited memory, the frontend waits for the backend
+# build to finish so the two heavy first build steps do not run at the same time
+up-low-mem:
+	sudo docker compose -f docker-compose.yml -f docker-compose.low-mem.yml up -d backend frontend api-test-server pebble dbgate mailer dozzle stats dns test mitmproxy; \
+	sudo docker compose -f docker-compose.yml -f docker-compose.low-mem.yml logs -f --tail 1000 backend frontend;
 up-reset: down purge up
 restart: down up
 prune: