From 9c069522b0df758611ae153abeee820b430e49a0 Mon Sep 17 00:00:00 2001 From: Chris King Date: Wed, 8 Jan 2025 00:08:54 -0800 Subject: [PATCH] Initial commit of docker compose project generator Creates project directory structure in /docker Creates docker-compose.yml with sensible defaults Creates borgmatic config in /etc/borgmatic.d Adds proxy info to caddyfile and reloads caddy Config file generation done via jsonnet and yq Borgmatic configured to backup to borgbase repo --- .gitignore | 1 + borgmatic/borg-config.jsonnet | 77 ++++++ borgmatic/post-backup-docker-compose-up.sh | 16 ++ borgmatic/pre-backup-docker-compose-down.sh | 15 + create-all-borg-configs.sh | 8 + docker-compose.jsonnet | 33 +++ docker-gen.sh | 286 ++++++++++++++++++++ 7 files changed, 436 insertions(+) create mode 100644 .gitignore create mode 100644 borgmatic/borg-config.jsonnet create mode 100755 borgmatic/post-backup-docker-compose-up.sh create mode 100755 borgmatic/pre-backup-docker-compose-down.sh create mode 100755 create-all-borg-configs.sh create mode 100644 docker-compose.jsonnet create mode 100755 docker-gen.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0f18981 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +secrets/ \ No newline at end of file diff --git a/borgmatic/borg-config.jsonnet b/borgmatic/borg-config.jsonnet new file mode 100644 index 0000000..b227f24 --- /dev/null +++ b/borgmatic/borg-config.jsonnet @@ -0,0 +1,77 @@ +function( + project, + encryption_passphrase, + parent_dir='/docker', + repo_path='ssh://s1dw3340@s1dw3340.repo.borgbase.com/./repo', + repo_label='winterfell-docker on BorgBase', + exclude_patterns=[ + // Generic caches & temp + '**/tmp/', + '**/temp/', + '**/.cache/', + '**/cache/', + + // Potential ephemeral DB areas (adjust if you do raw file backups) + '**/pg_stat_tmp/', + '**/pg_replslot/', + + // Node-based ephemeral + '**/node_modules/', + + // System or FS-specific + '**/lost+found/', + '**/*.lock', + ], + one_file_system=true, + compression='auto,zstd', + archive_name_format='docker-{now:%Y-%m-%d-%H%M%S}', + prefix_project_name_archive=true, + retries=0, + retry_wait=2, + keep_daily=3, + keep_weekly=4, + keep_monthly=12, + checks=[ + { + "name": "repository", + "frequency": "4 weeks" + }, + { + "name": "archives", + "frequency": "8 weeks" + } + ], + check_last=3, + before_backup='/code/scripts/docker-gen/borgmatic/pre-backup-docker-compose-down.sh {{project}}', + after_backup='/code/scripts/docker-gen/borgmatic/post-backup-docker-compose-up.sh {{project}}' +) + +{ + 'source_directories': [ + std.rstripChars(parent_dir, '/') + '/' + std.stripChars(project, ' ') + ], + 'repositories': [ + { + 'path': repo_path, + 'label': repo_label + } + ], + 'exclude_patterns': exclude_patterns, + 'one_file_system': one_file_system, + 'compression': compression, + 'encryption_passphrase': encryption_passphrase, + 'archive_name_format': if prefix_project_name_archive then project + '-' + archive_name_format else archive_name_format, + 'retries': retries, + 'retry_wait': retry_wait, + 'keep_daily': keep_daily, + 'keep_weekly': keep_weekly, + 'keep_monthly': keep_monthly, + 'checks': checks, + 'check_last': check_last, + 'before_backup': [ + std.strReplace(before_backup, '{{project}}', project) + ], + 'after_backup': [ + std.strReplace(after_backup, '{{project}}', project) + ] +} \ No newline at end of file diff --git a/borgmatic/post-backup-docker-compose-up.sh b/borgmatic/post-backup-docker-compose-up.sh new file mode 100755 index 0000000..5a2f663 --- /dev/null +++ b/borgmatic/post-backup-docker-compose-up.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +if [ $# -eq 0 ]; then + echo "Usage: $0 [folder2] ..." + exit 1 +fi + +for folder in "$@"; do + cd "/docker/$folder" || { + echo "Directory /docker/$folder not found!" + continue + } + + #echo "would docker compose up -d in /docker/$folder" + docker compose up -d +done diff --git a/borgmatic/pre-backup-docker-compose-down.sh b/borgmatic/pre-backup-docker-compose-down.sh new file mode 100755 index 0000000..f017eab --- /dev/null +++ b/borgmatic/pre-backup-docker-compose-down.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +if [ $# -eq 0 ]; then + echo "Usage: $0 [folder2] ..." + exit 1 +fi + +for folder in "$@"; do + cd "/docker/$folder" || { + echo "Directory /docker/$folder not found!" + continue + } + + docker compose down --remove-orphans +done diff --git a/create-all-borg-configs.sh b/create-all-borg-configs.sh new file mode 100755 index 0000000..8a50fd7 --- /dev/null +++ b/create-all-borg-configs.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +for dir in /docker/*; do + if [ -d "$dir" ] && [ "$(basename "$dir")" != "scripts" ]; then + base_name=$(basename "$dir") + /code/scripts/docker-gen/docker-gen.sh "$base_name" --only-borgmatic + fi +done diff --git a/docker-compose.jsonnet b/docker-compose.jsonnet new file mode 100644 index 0000000..ab76b73 --- /dev/null +++ b/docker-compose.jsonnet @@ -0,0 +1,33 @@ +function( + project, + image, + web_port="0000:0000", + ports=[], + tag="latest", + restart="unless-stopped" +) + +{ + "name": project, + "services": { + "app": { + "image": image + ':' + tag, + "restart": restart, + "ports": std.uniq(std.sort(std.flattenArrays([ports, [web_port]]))), + "environment": { + "DOCKER_TEMPLATE_CREATED": true + }, + "volumes": [ + './data:/data' + ], + "secrets": [ + 'asecret' + ] + } + }, + "secrets": { + "asecret": { + "file": './secrets/ASECRET' + } + } +} \ No newline at end of file diff --git a/docker-gen.sh b/docker-gen.sh new file mode 100755 index 0000000..decff01 --- /dev/null +++ b/docker-gen.sh @@ -0,0 +1,286 @@ +#!/usr/bin/env bash + +# Exit immediately if a command exits with a non-zero status. +set -e + +############################################################################### +# Usage: +# create-docker-compose.sh [] +# [--only-borgmatic] +# [--no-borgmatic] +# [--parent_directory PARENT_DIRECTORY] +# [--encryption_passphrase PASSPHRASE] +# [--tag TAG_NAME] +# [--web-port WEB_PORT] +# +# Examples: +# 1) Create Docker project + Borgmatic (default): +# ./create-docker-compose.sh my_project my_image +# +# 2) Create Docker project only (skip Borgmatic): +# ./create-docker-compose.sh my_project my_image --no-borgmatic +# +# 3) Create Borgmatic config only (skip Docker project & image param): +# ./create-docker-compose.sh my_project --only-borgmatic +# +# 4) Use a custom encryption passphrase: +# ./create-docker-compose.sh my_project my_image --encryption_passphrase "MyPass123" +# +# 5) Custom parent directory + custom passphrase: +# ./create-docker-compose.sh my_project my_image \ +# --parent_directory /custom/path \ +# --encryption_passphrase "SuperSecret!" +# +# 6) Specify a tag for the Docker image (default: "latest"): +# ./create-docker-compose.sh my_project my_image --tag dev +# +# 7) Specify a port mapping for a web UI (e.g., "8000:8080"): +# ./create-docker-compose.sh my_project my_image --web-port "8000:8080" +# This will pass "web_port" to the Jsonnet template and add a line to +# /etc/caddy/Caddyfile, then reload Caddy. +# +# Description: +# 1. By default (no --only-borgmatic), creates a Docker Compose project +# structure in /, generating: +# docker-compose.yml (from /code/scripts/docker-gen/docker-compose.jsonnet) +# config/ +# data/ +# owned by the user running this script. +# +# 2. By default, generates a Borgmatic config file in /etc/borgmatic.d/.yaml +# (owned by root), unless --no-borgmatic is specified. +# +# 3. If --only-borgmatic is used, becomes optional, and the script +# skips creating the Docker project folder entirely (only generating the +# Borgmatic config). +# +# 4. The default Borgmatic encryption passphrase is read from: +# /code/scripts/docker-gen/secrets/BORGMATIC_ENCRYPTION_PASSPHRASE +# If missing, the script falls back to "abcd1234". +# Can be overridden via --encryption_passphrase "". +# +# 5. --tag can override the default "latest" Docker tag in the Compose Jsonnet. +# +# 6. --web-port can pass an additional TLA param "web_port" to docker-compose.jsonnet, +# and if provided, a line is appended to /etc/caddy/Caddyfile in the format: +# import ttt-app +# followed by a reload of the Caddy service. The is the left side +# of the "host:container" mapping (e.g. "8000" if --web-port "8000:8080"). +# +############################################################################### + +############################################################################### +# 1) Initialize Variables +############################################################################### + +PROJECT_NAME="" +IMAGE="" +PARENT_DIR="/docker" # Default if not overridden +SKIP_BORGMATIC=false +ONLY_BORGMATIC=false +TAG="latest" # Default Docker image tag if not overridden +WEB_PORT="" # Optional Docker port, e.g. "8000:8001" + +# Read the default encryption passphrase from file, or fallback +if [ -f "/code/scripts/docker-gen/secrets/BORGMATIC_ENCRYPTION_PASSPHRASE" ]; then + DEFAULT_ENCRYPTION_PASSPHRASE="$(cat /code/scripts/docker-gen/secrets/BORGMATIC_ENCRYPTION_PASSPHRASE)" +else + echo "Warning: /code/scripts/docker-gen/secrets/BORGMATIC_ENCRYPTION_PASSPHRASE not found. Using 'abcd1234'." + DEFAULT_ENCRYPTION_PASSPHRASE="abcd1234" +fi +ENCRYPTION_PASSPHRASE="$DEFAULT_ENCRYPTION_PASSPHRASE" + +############################################################################### +# 2) Parse Command-line Arguments +############################################################################### +while [[ $# -gt 0 ]]; do + case "$1" in + --only-borgmatic) + ONLY_BORGMATIC=true + shift + ;; + --no-borgmatic) + SKIP_BORGMATIC=true + shift + ;; + --parent_directory) + PARENT_DIR="$2" + shift 2 + ;; + --encryption_passphrase) + ENCRYPTION_PASSPHRASE="$2" + shift 2 + ;; + --tag) + TAG="$2" + shift 2 + ;; + --web-port) + WEB_PORT="$2" + shift 2 + ;; + -h|--help) + echo "Usage: $0 [] [--only-borgmatic] [--no-borgmatic] [--parent_directory PARENT_DIRECTORY] [--encryption_passphrase PASSPHRASE] [--tag TAG] [--web-port WEB_PORT]" + exit 0 + ;; + *) + # If we haven't set the project name, do that first + if [ -z "$PROJECT_NAME" ]; then + PROJECT_NAME="$1" + # If we haven't set the image AND we're not in only-borgmatic mode, set it + elif [ -z "$IMAGE" ] && [ "$ONLY_BORGMATIC" = false ]; then + IMAGE="$1" + else + echo "Unknown parameter: $1" + echo "Usage: $0 [] [--only-borgmatic] [--no-borgmatic] [--parent_directory PARENT_DIRECTORY] [--encryption_passphrase PASSPHRASE] [--tag TAG] [--web-port WEB_PORT]" + exit 1 + fi + shift + ;; + esac +done + +############################################################################### +# 3) Validate Required Parameters +############################################################################### +if [ -z "$PROJECT_NAME" ]; then + echo "Error: Missing project name." + echo "Usage: $0 [] ..." + exit 1 +fi + +# If we're NOT in only-borgmatic mode, we require the image parameter +if [ "$ONLY_BORGMATIC" = false ] && [ -z "$IMAGE" ]; then + echo "Error: Missing image name. Provide or use --only-borgmatic." + exit 1 +fi + +# Check for conflicting flags +if [ "$ONLY_BORGMATIC" = true ] && [ "$SKIP_BORGMATIC" = true ]; then + echo "Error: --only-borgmatic and --no-borgmatic are mutually exclusive." + exit 1 +fi + +# Ensure no whitespace in project name +if [[ "$PROJECT_NAME" =~ [[:space:]] ]]; then + echo "Error: Project name cannot contain whitespace." + exit 1 +fi + +# Ensure the project name has only allowed chars +if [[ ! "$PROJECT_NAME" =~ ^[A-Za-z0-9._-]+$ ]]; then + echo "Error: Project name contains illegal characters." + echo "Allowed: alphanumeric, underscores (_), dashes (-), and dots (.)" + exit 1 +fi + +############################################################################### +# 4) Prepare Directory Paths +############################################################################### +# Remove trailing slash from the parent directory path if any +PARENT_DIR="${PARENT_DIR%/}" + +# Construct the project path +PROJECT_PATH="$PARENT_DIR/$PROJECT_NAME" + +############################################################################### +# 5) Optionally Create the Docker Project Structure +############################################################################### +if [ "$ONLY_BORGMATIC" = false ]; then + if [ -d "$PROJECT_PATH" ]; then + echo "Error: Project directory '$PROJECT_PATH' already exists." + exit 1 + fi + + echo "Creating Docker Compose project folder at: $PROJECT_PATH" + mkdir -p "$PROJECT_PATH/config" "$PROJECT_PATH/data" "$PROJECT_PATH/secrets" + + # Build jsonnet command for docker-compose + JSONNET_CMD=( + jsonnet + --tla-str "project=$PROJECT_NAME" + --tla-str "image=$IMAGE" + --tla-str "tag=$TAG" + ) + + # If a web port was specified, add it to the Jsonnet command + if [ -n "$WEB_PORT" ]; then + JSONNET_CMD+=( --tla-str "web_port=$WEB_PORT" ) + fi + + # Run jsonnet + yq to create docker-compose.yml + "${JSONNET_CMD[@]}" /code/scripts/docker-gen/docker-compose.jsonnet \ + | yq e '... style="" | with(.services.app.ports[] ; . style="double")' - \ + > "$PROJECT_PATH/docker-compose.yml" + + # Change ownership to the user running the script + chown -R "$(id -u):$(id -g)" "$PROJECT_PATH" + + # If a web port is provided, parse the host port and update Caddy + if [ -n "$WEB_PORT" ]; then + # Extract the host port (before the colon, e.g. "8000" if "8000:8080") + HOST_PORT="${WEB_PORT%%:*}" + + # Append line to Caddyfile: import ttt-app + echo "import ttt-app $PROJECT_NAME $HOST_PORT" | sudo tee -a /etc/caddy/Caddyfile > /dev/null + + # Reload Caddy + sudo systemctl reload caddy.service + echo "Updated /etc/caddy/Caddyfile and reloaded Caddy with 'import ttt-app $PROJECT_NAME $HOST_PORT'." + fi + +else + echo "Skipping Docker project folder creation (--only-borgmatic)." +fi + +############################################################################### +# 6) Generate the Borgmatic Configuration (unless --no-borgmatic) +############################################################################### +if [ "$SKIP_BORGMATIC" = false ]; then + echo "Generating Borgmatic configuration for project: '$PROJECT_NAME'..." + + # Ensure /etc/borgmatic.d exists and is owned by root + sudo mkdir -p /etc/borgmatic.d + sudo chown root:root /etc/borgmatic.d + + # Generate the config using jsonnet & yq, then tee it into place as root + jsonnet \ + --tla-str project="$PROJECT_NAME" \ + --tla-str encryption_passphrase="$ENCRYPTION_PASSPHRASE" \ + /code/scripts/docker-gen/borgmatic/borg-config.jsonnet \ + | yq -p json -o yaml - \ + | sudo tee "/etc/borgmatic.d/${PROJECT_NAME}.yaml" >/dev/null + + # Ensure the config file is owned by root + sudo chown root:root "/etc/borgmatic.d/${PROJECT_NAME}.yaml" + + echo "Borgmatic config created at: /etc/borgmatic.d/${PROJECT_NAME}.yaml (owned by root)" +else + echo "Skipping Borgmatic configuration (--no-borgmatic)." +fi + +############################################################################### +# 7) Final Output +############################################################################### +echo +if [ "$ONLY_BORGMATIC" = false ]; then + echo "Docker Compose project directory (if created) is at:" + echo " $PROJECT_PATH" + echo "Owned by user: $(id -un), group: $(id -gn)" + echo + echo "Directory structure:" + if [ -d "$PROJECT_PATH" ]; then + if command -v tree >/dev/null 2>&1; then + tree "$PROJECT_PATH" + else + ls -R "$PROJECT_PATH" + fi + else + echo " [Skipped creation of Docker project folder]" + fi +else + echo "Docker project folder creation was skipped (--only-borgmatic)." +fi + +echo +echo "Script execution completed."