From 9aae367408703775d6c5b1264ae43743db6fd5ed Mon Sep 17 00:00:00 2001 From: Obsidian-MBPM4 Date: Mon, 20 Apr 2026 12:33:43 +0200 Subject: [PATCH] vault backup: 2026-04-20 12:33:43 Affected files: .obsidian/workspace.json 2 Personal/Home Lab/Backup System - Kopia Server Setup.md --- .obsidian/workspace.json | 10 +- .../Backup System - Kopia Server Setup.md | 947 ++++++++++++++++++ 2 files changed, 952 insertions(+), 5 deletions(-) create mode 100644 2 Personal/Home Lab/Backup System - Kopia Server Setup.md diff --git a/.obsidian/workspace.json b/.obsidian/workspace.json index 1965faa..d76e69f 100644 --- a/.obsidian/workspace.json +++ b/.obsidian/workspace.json @@ -167,12 +167,12 @@ "state": { "type": "markdown", "state": { - "file": "2 Personal/1 Skills/Obisdian/Obsidian Setup.md", + "file": "2 Personal/Home Lab/Backup System - Kopia Server Setup.md", "mode": "source", "source": false }, "icon": "lucide-file", - "title": "Obsidian Setup" + "title": "Backup System - Kopia Server Setup" } } ], @@ -481,7 +481,7 @@ ], "direction": "vertical", "x": 0, - "y": 44, + "y": 42, "width": 900, "height": 777, "maximize": false, @@ -491,10 +491,11 @@ }, "active": "fac43a56fe618e9d", "lastOpenFiles": [ + "2 Personal/1 Skills/Obisdian/Obsidian Setup.md", + "2 Personal/Home Lab/Backup System - Kopia Server Setup.md", "2 Personal/Projects/AlpineView/Thoughts.md", "2 Personal/Projects/AlpineView", "2 Personal/Home Lab/Syncthing.md", - "2 Personal/1 Skills/Obisdian/Obsidian Setup.md", "2 Personal/Home Lab/NextiShareBot.md", "5 Media/8 Courses/Design Patterns by Construx.md", "2 Personal/Lists/Business Ideas.md", @@ -518,7 +519,6 @@ "Dashboard Canvas.canvas", "Dashboard.md", "8 Places/BusinessesDrawing 2023-10-12 16.01.52.excalidraw.md", - "2 Personal/Wohnen/Rezepte/Ideen für Vorratskammer.md", "Attachments/ESPSomfyRTS 2026-03-17T16_05_06.backup", "Attachments/Pasted image 20260121121234.png", "Attachments/ESPSomfyRTS 2026-01-18T16_26_16.backup", diff --git a/2 Personal/Home Lab/Backup System - Kopia Server Setup.md b/2 Personal/Home Lab/Backup System - Kopia Server Setup.md new file mode 100644 index 0000000..a785139 --- /dev/null +++ b/2 Personal/Home Lab/Backup System - Kopia Server Setup.md @@ -0,0 +1,947 @@ +--- +title: Backup System - Kopia Server Setup +created_date: 2026-04-20 +updated_date: 2026-04-20 +aliases: +tags: +--- +# Backup System - Kopia Server Setup + +## Overview + +This document describes the setup we built for a self-hosted Kopia backup server in the homelab. + +### Final architecture + +- **Hypervisor:** Proxmox +- **Backup server runtime:** Debian VM on Proxmox (IP:192.168.1.54 - IP managed by pi-hole) +- **Backup repository storage:** Synology NAS via NFS +- **Backup service:** Kopia Repository Server +- **Remote/private access:** intended via Tailscale or LAN +- **First client:** MacBook +- **TLS:** self-generated Kopia TLS certificate + +### Why this architecture was chosen + +We first tried to run Kopia in an LXC container. That led to multiple issues: + +- NFS mount permission issues with an **unprivileged LXC** +- AppArmor / systemd problems with newer Debian / systemd in LXC +- black Proxmox console / awkward LXC behavior + +Because of that, we switched to a **VM**, which is the cleaner and more robust setup for this use case. + +--- + +## Final design + +### Components + +- **Debian VM** runs Kopia Repository Server +- **Synology NAS** stores the actual backup repository blobs +- **Kopia clients** connect to the Repository Server over HTTPS +- **MacBook** connects as `claudio@macbook-main` + +### Repository model + +Important distinction: + +- The **repository** itself is stored on the NAS filesystem +- The **Repository Server** is the HTTP/HTTPS layer in front of it +- Users connect to the **server**, not directly to the NAS share + +This is better than mounting the share directly on each laptop because: + +- clients do not depend on a mounted NAS path +- you get per-user accounts on the server +- easier multi-user setup +- easier remote use over VPN/Tailscale + +--- + +## Synology setup + +### Shared folder + +A dedicated shared folder was used on Synology: + +- `kopia-repository` + +### NFS export + +The NFS export path used was: + +```bash +192.168.1.34:/volume1/kopia-repository +``` + +### Synology NFS settings + +Recommended / used settings: + +- **Privilege:** Read/Write +- **Squash:** No mapping +- **Security:** sys +- **Allow connections from non-privileged ports:** enabled if needed + +Notes: + +- `Squash: No mapping` is fine for a dedicated Kopia repository share. +- In the LXC attempt, permissions still failed because of UID/GID mapping issues with unprivileged containers. + +--- + +## Failed LXC attempt and why we abandoned it + +We first tried to run Kopia in a Proxmox LXC. + +### What we tried + +- Created an LXC +- Tried mounting NFS inside the LXC +- Then switched to mounting NFS on the Proxmox host and bind-mounting it into the LXC +- Added `/dev/net/tun` for Tailscale / ZeroTier +- Enabled `nesting=1` because of Debian/systemd issues + +### Problems encountered + +#### 1. Wrong mount point created via Proxmox UI + +We accidentally created an extra LXC disk instead of a bind mount. + +We had this wrong line: + +```ini +mp0: local-lvm:vm-102-disk-1,mp=/mnt/pve/kopia-repo,size=8G +``` + +This was **not** a bind mount from the host. + +The correct bind mount syntax was: + +```ini +mp0: /mnt/pve/kopia-repo,mp=/srv/kopia-repo +``` + +#### 2. Console issues / black screen + +The LXC booted, but the Proxmox console was black. + +`pct enter` worked, but `pct console` did not work reliably. + +#### 3. systemd / AppArmor issues + +We saw warnings such as: + +- `Systemd 257 detected. You may need to enable nesting.` +- AppArmor denials related to `userns_create`, `mount`, `journald`, `networkd`, etc. + +This was partially fixed with: + +```bash +pct set 102 --features nesting=1 +pct restart 102 +``` + +#### 4. NFS permission issues in unprivileged LXC + +Even when the share was mounted correctly, writes failed inside the container: + +```bash +touch /srv/kopia-repo/testfile +# Permission denied +``` + +This happened because unprivileged container root is UID-mapped and Synology/NFS did not allow writes the way we wanted. + +### Conclusion + +The LXC route was abandoned because it caused unnecessary complexity for a simple service. + +We switched to a **Debian VM**, which is simpler and more maintainable. + +--- + +## VM setup + +### Assumptions + +- Debian VM on Proxmox +- VM has network access to the Synology NAS +- NFS share exists on Synology +- Kopia repository will live on the NAS + +### Base packages installed + +On the VM, we installed the following: + +```bash +apt update +apt install -y nfs-common curl ca-certificates gnupg nano +``` + +--- + +## NFS mount inside the VM + +### Mountpoint creation + +```bash +mkdir -p /srv/kopia-repo +mkdir -p /var/lib/kopia +chmod 700 /var/lib/kopia +``` + +### Manual test mount + +```bash +mount -t nfs 192.168.1.34:/volume1/kopia-repository /srv/kopia-repo +``` + +### Validation commands + +```bash +df -h /srv/kopia-repo +touch /srv/kopia-repo/testfile +ls -l /srv/kopia-repo/testfile +rm /srv/kopia-repo/testfile +``` + +At this stage, the VM approach worked properly. + +### Persistent mount in `/etc/fstab` + +We added: + +```fstab +192.168.1.34:/volume1/kopia-repository /srv/kopia-repo nfs defaults,_netdev 0 0 +``` + +Then tested with: + +```bash +umount /srv/kopia-repo +mount -a +df -h /srv/kopia-repo +``` + +--- + +## Kopia installation on the VM + +### APT repo setup + +We used the official Kopia apt repository. + +Commands used: + +```bash +install -d -m 0755 /etc/apt/keyrings +curl -s https://kopia.io/signing-key | gpg --dearmor -o /etc/apt/keyrings/kopia-keyring.gpg +echo "deb [signed-by=/etc/apt/keyrings/kopia-keyring.gpg] http://packages.kopia.io/apt/ stable main" > /etc/apt/sources.list.d/kopia.list +apt update +apt install -y kopia +``` + +### Verify installation + +```bash +kopia --version +``` + +--- + +## Kopia repository creation + +### Repository directory + +```bash +mkdir -p /srv/kopia-repo/repository +``` + +### Create repository + +We created a filesystem repository on the NFS-backed path: + +```bash +kopia repository create filesystem --path=/srv/kopia-repo/repository +``` + +During this step, a **repository password** was chosen. + +This password is critical. It encrypts the repository contents. + +--- + +## Environment variables + +We used an environment file with exported variables for passwords. + +Important shell note: + +- `"$VAR"` expands the variable +- `'$VAR'` does **not** expand the variable + +So this is **wrong**: + +```bash +--server-password='$KOPIA_SRV_PW' +``` + +And this is **correct**: + +```bash +--server-password="$KOPIA_SRV_PW" +``` + +### Variables used + +Example variables: + +```bash +export KOPIA_REPO_PW="..." +export KOPIA_SRV_CTRL_PW="..." +export KOPIA_SRV_PW="..." +``` + +And for manual startup we exported: + +```bash +export KOPIA_PASSWORD="$KOPIA_REPO_PW" +``` + +--- + +## First manual Kopia server start attempts + +### Initial attempt without TLS + +We first tried something like: + +```bash +kopia server start \ + --address=0.0.0.0:51515 \ + --server-control-username=server-control \ + --server-control-password="$KOPIA_SRV_CTRL_PW" \ + --server-username=kopia \ + --server-password="$KOPIA_SRV_PW" +``` + +This failed with the message: + +- `TLS not configured. To start server without encryption pass --insecure` + +So the server did **not** actually listen on the port. + +### Working manual TLS start + +The working startup command was: + +```bash +kopia server start \ + --tls-generate-cert \ + --tls-cert-file ~/my.cert \ + --tls-key-file ~/my.key \ + --address 0.0.0.0:51515 \ + --server-control-username control +``` + +Notes: + +- This generated a self-signed TLS cert and key. +- After generation, future starts must **not** use `--tls-generate-cert` again. + +--- + +## Adding the first Kopia user + +We created the first Mac user with: + +```bash +kopia server user add claudio@macbook-main +``` + +This prompts for a password for that user. + +This user/password is what the Mac client uses to connect. + +--- + +## Server refresh issue and root cause + +We saw this error: + +```bash +kopia server refresh ... +400 Bad Request: not connected +``` + +And from the Mac / KopiaUI we saw: + +- `not connected to a direct repository` + +### Root cause + +The Repository Server was starting, but the process running under systemd was **not connected to the repository**. + +This happened because: + +- the repository had been created and connected as user **`cef`** +- but the systemd service was running as **root** by default +- root did not have the Kopia repository config/session + +### Fix + +We changed the systemd service to run as **user `cef`**. + +That solved the issue. + +--- + +## TLS certificate handling + +### Move cert and key to stable location + +We moved the generated files out of the home directory: + +```bash +sudo mkdir -p /etc/kopia +sudo mv ~/my.cert /etc/kopia/server.cert +sudo mv ~/my.key /etc/kopia/server.key +sudo chmod 600 /etc/kopia/server.key +sudo chmod 644 /etc/kopia/server.cert +``` + +### Important permission fix + +Because the service runs as user `cef`, that user needed access to the cert and key: + +```bash +sudo chown cef:cef /etc/kopia/server.cert /etc/kopia/server.key +sudo chmod 600 /etc/kopia/server.key +sudo chmod 644 /etc/kopia/server.cert +``` + +--- + +## Environment file for systemd + +We created: + +```bash +sudo nano /etc/kopia-server.env +``` + +Contents: + +```bash +KOPIA_PASSWORD=YOUR_REPOSITORY_PASSWORD +KOPIA_SRV_CTRL_PW=YOUR_SERVER_CONTROL_PASSWORD +KOPIA_SRV_PW=YOUR_WEB_UI_PASSWORD +``` + +Permissions: + +```bash +sudo chown root:cef /etc/kopia-server.env +sudo chmod 640 /etc/kopia-server.env +``` + +--- + +## Final systemd service + +We created: + +```bash +sudo nano /etc/systemd/system/kopia-server.service +``` + +Final service: + +```ini +[Unit] +Description=Kopia Repository Server +After=network-online.target remote-fs.target +Wants=network-online.target +Requires=remote-fs.target + +[Service] +Type=simple +User=cef +Group=cef +EnvironmentFile=/etc/kopia-server.env +ExecStart=/usr/bin/kopia server start \ + --tls-cert-file=/etc/kopia/server.cert \ + --tls-key-file=/etc/kopia/server.key \ + --address=0.0.0.0:51515 \ + --server-control-username=control \ + --server-control-password=${KOPIA_SRV_CTRL_PW} \ + --server-username=kopia \ + --server-password=${KOPIA_SRV_PW} +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target +``` + +### Enable and start + +```bash +sudo systemctl daemon-reload +sudo systemctl enable --now kopia-server +sudo systemctl status kopia-server --no-pager +``` + +### Check if listening + +```bash +ss -ltnp | grep 51515 +``` + +--- + +## Certificate fingerprint + +Because the TLS certificate is self-generated, clients must trust it using the SHA256 fingerprint. + +### Get fingerprint + +On the VM: + +```bash +openssl x509 -in /etc/kopia/server.cert -noout -fingerprint -sha256 | sed 's/://g' | cut -f 2 -d = +``` + +Save the resulting fingerprint. + +--- + +## Refreshing server credentials + +Once the service was working, refresh needed to use: + +- HTTPS +- control username/password +- server certificate fingerprint + +Command: + +```bash +kopia server refresh \ + --address=https://127.0.0.1:51515 \ + --server-control-username=control \ + --server-control-password="$KOPIA_SRV_CTRL_PW" \ + --server-cert-fingerprint=YOUR_FINGERPRINT +``` + +If you get `not connected`, the server process is not connected to the repository context. Check that the service runs as the same user that has a valid Kopia repository config. + +--- + +## MacBook setup + +### Install Kopia on macOS + +We used Homebrew: + +```bash +brew install kopia +brew install kopiaui +``` + +### Connect the Mac to the Repository Server + +We used the CLI first, because it is more reliable for initial connection than KopiaUI. + +Command: + +```bash +kopia repository connect server \ + --url=https://YOUR_VM_IP:51515 \ + --server-cert-fingerprint=YOUR_CERT_FINGERPRINT \ + --override-username=claudio \ + --override-hostname=macbook-main +``` + +The login password here is the password for the Kopia server user: + +- `claudio@macbook-main` + +### Verify connection + +```bash +kopia repository status +``` + +--- + +## MacBook first test backup + +### Create a test folder + +```bash +mkdir -p ~/kopia-test +echo "hello kopia" > ~/kopia-test/file1.txt +date > ~/kopia-test/file2.txt +``` + +### Create first snapshot + +```bash +kopia snapshot create ~/kopia-test +``` + +### List snapshots + +```bash +kopia snapshot list +``` + +### Test restore + +```bash +mkdir -p ~/kopia-restore-test +kopia restore latest ~/kopia-restore-test +ls -la ~/kopia-restore-test +``` + +This validates the full chain: + +- VM +- NFS mount +- repository +- Kopia Repository Server +- Mac client connection +- backup +- restore + +--- + +## Automatic backup on the Mac + +### Basic idea + +The recommended model is: + +1. connect the Mac to the repository once +2. define backup roots +3. use `kopia snapshot create --all` on a schedule + +### Suggested first backup roots + +Start simple. For example: + +- `~/Documents` +- `~/Desktop` +- local project folders + +Do **not** immediately back up all of `~/Library`. + +### Initial snapshots for real backup roots + +Example: + +```bash +kopia snapshot create ~/Documents +kopia snapshot create ~/Desktop +``` + +### Set retention / scheduling policy + +Example: + +```bash +kopia policy set ~/Documents \ + --snapshot-interval=12h \ + --keep-latest=14 \ + --keep-daily=14 \ + --keep-weekly=8 \ + --keep-monthly=12 + +kopia policy set ~/Desktop \ + --snapshot-interval=12h \ + --keep-latest=14 \ + --keep-daily=14 \ + --keep-weekly=8 \ + --keep-monthly=12 +``` + +### Robust automatic execution with launchd + +Create: + +```text +~/Library/LaunchAgents/com.claudio.kopia-backup.plist +``` + +Contents: + +```xml + + + + + Label + com.claudio.kopia-backup + + ProgramArguments + + /opt/homebrew/bin/kopia + snapshot + create + --all + --no-progress + + + StartInterval + 21600 + + RunAtLoad + + + StandardOutPath + /tmp/kopia-backup.out + + StandardErrorPath + /tmp/kopia-backup.err + + +``` + +Load it: + +```bash +launchctl unload ~/Library/LaunchAgents/com.claudio.kopia-backup.plist 2>/dev/null || true +launchctl load ~/Library/LaunchAgents/com.claudio.kopia-backup.plist +launchctl list | grep kopia +``` + +This runs every 6 hours (`21600` seconds). + +### Verify automatic backup + +```bash +kopia snapshot list +``` + +And occasionally: + +```bash +kopia restore latest ~/kopia-restore-test-2 +``` + +--- + +## How to add another user, for example your spouse + +The model is: + +- create a new user on the Kopia server +- connect that user’s machine with a fixed username/hostname identity +- create backup sources from that machine + +### Example: spouse on Windows + +Suppose you want: + +- username: `partner` +- hostname: `windows-laptop` + +### 1. Create the user on the Kopia server VM + +On the VM: + +```bash +kopia server user add partner@windows-laptop +``` + +Set a password when prompted. + +### 2. Refresh credentials if needed + +```bash +kopia server refresh \ + --address=https://127.0.0.1:51515 \ + --server-control-username=control \ + --server-control-password="$KOPIA_SRV_CTRL_PW" \ + --server-cert-fingerprint=YOUR_FINGERPRINT +``` + +### 3. Install Kopia on the spouse’s machine + +For Windows, install Kopia / KopiaUI from the official installer. + +### 4. Connect that machine to the server + +From CLI, the equivalent pattern is: + +```bash +kopia repository connect server \ + --url=https://YOUR_VM_IP:51515 \ + --server-cert-fingerprint=YOUR_CERT_FINGERPRINT \ + --override-username=partner \ + --override-hostname=windows-laptop +``` + +Then enter the password for: + +- `partner@windows-laptop` + +### 5. Create first backup sources on that machine + +For example on Windows: + +- Documents +- Desktop +- Pictures + +### 6. Run first backup and test restore + +Do exactly the same test pattern as on the Mac: + +- create first snapshot +- list snapshots +- restore a test folder + +### Naming convention recommendation + +Use stable names so future maintenance is simple. + +Examples: + +- `claudio@macbook-main` +- `claudio@windows-main` +- `partner@windows-laptop` +- `partner@android-photos` if you ever use a separate flow later + +--- + +## Maintenance and troubleshooting + +### Check service status + +```bash +sudo systemctl status kopia-server --no-pager +``` + +### View Kopia server logs + +```bash +sudo journalctl -u kopia-server -n 100 --no-pager +``` + +### Check if port is listening + +```bash +ss -ltnp | grep 51515 +``` + +### Check repository status as service user + +```bash +kopia repository status +``` + +If this fails under the service user context, the Repository Server will not work correctly. + +### Test local HTTPS endpoint + +```bash +curl -k https://127.0.0.1:51515/ +``` + +### Get certificate fingerprint again + +```bash +openssl x509 -in /etc/kopia/server.cert -noout -fingerprint -sha256 | sed 's/://g' | cut -f 2 -d = +``` + +### Common failure: wrong shell quoting + +Wrong: + +```bash +--server-password='$KOPIA_SRV_PW' +``` + +Correct: + +```bash +--server-password="$KOPIA_SRV_PW" +``` + +### Common failure: server not connected + +Symptoms: + +- `400 Bad Request: not connected` +- Mac / UI error: `not connected to a direct repository` + +Fix: + +- make sure systemd service runs as the same user that created / connected the repository +- in this setup that was `cef` + +### Common failure: browser works but Mac client fails + +This usually means: + +- HTTPS listener is fine +- web auth is fine +- but the server has no repository connection or client trust settings are wrong + +Check: + +- `kopia repository status` +- systemd service user +- cert fingerprint on client + +--- + +## Important files in this setup + +### On the VM + +- Repository storage: + - `/srv/kopia-repo/repository` +- NFS mountpoint: + - `/srv/kopia-repo` +- TLS cert: + - `/etc/kopia/server.cert` +- TLS key: + - `/etc/kopia/server.key` +- systemd env file: + - `/etc/kopia-server.env` +- systemd service: + - `/etc/systemd/system/kopia-server.service` + +### On the Mac + +- launchd job: + - `~/Library/LaunchAgents/com.claudio.kopia-backup.plist` + +--- + +## What not to forget later + +- Do not rerun `--tls-generate-cert` on every start. +- Keep the repository password safe. +- Keep the server control password safe. +- Keep the Web UI password safe. +- Keep the certificate fingerprint documented. +- Test restores periodically, not just backups. +- Do not assume browser access means repository connectivity is correct. + +--- + +## Suggested next steps + +1. Finalize the Mac backup roots +2. Test automatic backups from the Mac +3. Add spouse’s machine as a second user +4. Test restore from spouse’s machine too +5. Later consider offsite replication of the Kopia repository +6. Keep Time Machine in parallel on the Mac if you want easier full-machine restore +