From 2d3d9217e88ca5fac3f7c14b86ae8e775b6e6bf8 Mon Sep 17 00:00:00 2001 From: Nick Stokoe Date: Tue, 4 Apr 2023 11:52:08 +0100 Subject: [PATCH] snackpot.yml etc. - add borgmatic backup container --- snackpot.yml | 26 ++++++++ templates/appserver.service.j2 | 14 +++++ templates/bin/backup | 2 +- templates/borg.service.j2 | 2 +- templates/docker-compose/borgmatic/Dockerfile | 2 +- templates/docker-compose/docker-compose.yml | 63 +++++++++++++++++++ .../volumes/borgmatic-config/config.yaml | 25 +++++--- .../volumes/borgmatic-config/msmtp.env.j2 | 1 + 8 files changed, 122 insertions(+), 13 deletions(-) create mode 100644 templates/appserver.service.j2 create mode 100644 templates/docker-compose/volumes/borgmatic-config/msmtp.env.j2 diff --git a/snackpot.yml b/snackpot.yml index 0fbc430..a6ff70c 100644 --- a/snackpot.yml +++ b/snackpot.yml @@ -11,6 +11,12 @@ jellyfin_hostname: jf.noodlefactory.co.uk letsencrypt_email: webmaster@noodlefactory.co.uk docker_compose_base_dir: /opt/docker-compose + docker_compose_cmd: docker-compose + borg_passphrase: "{{lookup('passwordstore', 'servers/snackpot/borg.passphrase')}}" + smtp_password: "{{lookup('passwordstore', 'servers/snackpot/smtp.password')}}" + borg_ssh_key: "{{lookup('passwordstore', 'servers/snackpot/borg.id_rsa')}}" + borg_ssh_key_pub: "{{lookup('passwordstore', 'servers/snackpot/borg.id_rsa.pub')}}" + borg_repo_key: "{{lookup('passwordstore', 'servers/snackpot/borg_repo.key')}}" firewall_ports: - "22" - "80" @@ -136,5 +142,25 @@ with_filetree: templates/bin when: item.state == "file" and not item.path.endswith("~") tags: docker-config + + - name: install appserver and borg backup services + template: + dest: "/etc/systemd/system/{{ item }}" + src: "{{ item }}.j2" + owner: root + group: root + mode: 0550 + with_items: + - appserver.service + - borg.service + - borg.timer + tags: docker-config + + - name: enable backup service + service: + name: borg + state: started + enabled: yes + # config nextcloud # hide pg password diff --git a/templates/appserver.service.j2 b/templates/appserver.service.j2 new file mode 100644 index 0000000..f9610c4 --- /dev/null +++ b/templates/appserver.service.j2 @@ -0,0 +1,14 @@ +[Unit] +Description=appserver +Requires=docker.service +After=docker.service + +[Service] +Type=oneshot +WorkingDirectory={{docker_compose_base_dir}} +ExecStart={{docker_compose_cmd}} up -d --remove-orphans main-services +ExecStop={{docker_compose_cmd}} down +RemainAfterExit=true + +[Install] +WantedBy=multi-user.target diff --git a/templates/bin/backup b/templates/bin/backup index cba9701..1c2c6f2 100755 --- a/templates/bin/backup +++ b/templates/bin/backup @@ -9,6 +9,6 @@ DC_DIR=/opt/docker-compose/ cd $DC_DIR docker-compose down -docker run --name borgmatic -T --rm borgmatic /backup.sh +docker-compose run --name borgmatic -T --rm borgmatic /backup.sh docker-compose up -d main-services diff --git a/templates/borg.service.j2 b/templates/borg.service.j2 index 3159178..cb5c5c0 100644 --- a/templates/borg.service.j2 +++ b/templates/borg.service.j2 @@ -3,4 +3,4 @@ Description=Borg backups [Service] Type=oneshot -ExecStart=/opt/docker/bin/backup +ExecStart=/opt/docker-compose/bin/backup diff --git a/templates/docker-compose/borgmatic/Dockerfile b/templates/docker-compose/borgmatic/Dockerfile index 6e80b08..5af8ab1 100644 --- a/templates/docker-compose/borgmatic/Dockerfile +++ b/templates/docker-compose/borgmatic/Dockerfile @@ -1,4 +1,4 @@ FROM b3vis/borgmatic:latest-msmtp -COPY backup.sh /backup.sh +COPY --chmod=755 backup.sh /backup.sh diff --git a/templates/docker-compose/docker-compose.yml b/templates/docker-compose/docker-compose.yml index 47f2939..f874bf3 100644 --- a/templates/docker-compose/docker-compose.yml +++ b/templates/docker-compose/docker-compose.yml @@ -17,6 +17,7 @@ volumes: minidlna_state: minidlna_data: mopidy_data: + borgmatic-cache: networks: # This is for proxied containers @@ -95,6 +96,7 @@ services: - web.env depends_on: - nextcloud + - letsencrypt-companion networks: - proxy-tier - default @@ -193,6 +195,67 @@ services: ipv4_address: 192.168.0.243 restart: always + # a dummy container to start the main services as deps + # This allows the borgmatic image to be excluded when run as: + # docker-compose up main-services + main-services: + image: alpine:latest # a small dumy image + command: sh -c "sleep infinity" + depends_on: + - nextcloud + - nextcloud_cron + - web + - jellyfin + - minidlna + - upmpdcli + + borgmatic: + build: ./borgmatic + restart: 'no' # This container is only run when required + depends_on: # These containers need to be up for dumps + - postgres + networks: + # Networks for DB access for backups + - default + volumes: + # Backup mount + - /mnt/c/backup/nick:/mnt/borg-repository + # Volumes to back up + - certs:/mnt/source/certs:ro + - nextcloud_data:/mnt/source/nextcloud_data:ro + - vhost.d:/mnt/source/vhost.d:ro + - html:/mnt/source/html:ro + - jellyfin_config:/mnt/source/jellyfin_config:ro + - minidlna_state:/mnt/source/minidlna_state:ro + - minidlna_data:/mnt/source/minidlna_data:ro + # System volumes + - /etc/timezone:/etc/timezone:ro # timezone + - /etc/localtime:/etc/localtime:ro # localtime + - borgmatic-cache:/root/.cache/borg # non-volatile borg chunk cache + # Config volumes + - ./volumes/borgmatic-config:/etc/borgmatic.d/:ro # config.yaml, crontab.txt, mstmp.env + - ./volumes/borg-config:/root/.config/borg/ # borg encryption keys, other config written here + - ./volumes/borg-ssh-config:/root/.ssh/ # ssh keys; sshd writes knownhosts etc here + + environment: + POSTGRES_USER: nextcloud + POSTGRES_DB: nextcloud + POSTGRES_HOST: postgres + BORG_ARCHIVE: nick + MAIL_RELAY_HOST: mail.noodlefactory.co.uk + MAIL_PORT: 25 + MAIL_AUTH_METHOD: login + MAIL_STARTTLS: 'on' + MAIL_USER: nc.noodlefactory.co.uk + MAIL_FROM: borgmatic@snackpot.noodlefactory.co.uk + MAIL_TO: nick@noodlefactory.co.uk + MAIL_SUBJECT: Borgmatic Backup + # MAIL_PASSWORD is set via volumes/borgmatic-config/msmtp.env, created via ansible + env_file: + - ./borgmatic.env + # FIXME create backup service + + # Next three services adapted from # https://github.com/deisi/audiostation/blob/master/docker-compose.yml # and https://github.com/IVData/dockerfiles/blob/master/mopidy-multiroom/docker-compose.yml diff --git a/templates/docker-compose/volumes/borgmatic-config/config.yaml b/templates/docker-compose/volumes/borgmatic-config/config.yaml index ddfda05..f99037c 100644 --- a/templates/docker-compose/volumes/borgmatic-config/config.yaml +++ b/templates/docker-compose/volumes/borgmatic-config/config.yaml @@ -16,7 +16,7 @@ location: # is used, then add local repository paths in the systemd # service file to the ReadWritePaths list. repositories: - - /mnt/c/backup/nick + - /mnt/borg-repository # - ssh://${BORG_REPO_USER}@${BORG_REPO_HOST}:${BORG_REPO_PORT}/./${BORG_ARCHIVE} # Working directory for the "borg create" command. Tildes are @@ -103,6 +103,7 @@ location: - '*#' - '.cache' - 'cache' + - 'files_trashbin' # Read exclude patterns from one or more separate named files, # one pattern per line. See the output of "borg help patterns" @@ -451,18 +452,20 @@ hooks: # https://www.postgresql.org/docs/current/app-pgdump.html and # https://www.postgresql.org/docs/current/libpq-ssl.html for # details. - # postgresql_databases: + postgresql_databases: # Database name (required if using this hook). Or # "all" to dump all databases on the host. Note # that using this database hook implicitly enables # both read_special and one_file_system (see # above) to support dump and restore streaming. # - name: users + - name: ${POSTGRES_DB} # Database hostname to connect to. Defaults to # connecting via local Unix socket. # hostname: database.example.org - + hostname: ${POSTGRES_HOST} + # Port to connect to. Defaults to 5432. # port: 5433 @@ -471,14 +474,16 @@ hooks: # You probably want to specify the "postgres" # superuser here when the database name is "all". # username: dbuser - + username: ${POSTGRES_USER} + # Password with which to connect to the database. # Omitting a password will only work if PostgreSQL # is configured to trust the configured username # without a password or you create a ~/.pgpass # file. # password: trustsome1 - + password: ${POSTGRES_PASSWORD} + # Database dump output format. One of "plain", # "custom", "directory", or "tar". Defaults to # "custom" (unlike raw pg_dump). See pg_dump @@ -518,30 +523,30 @@ hooks: # mysqldump/mysql commands (from either MySQL or MariaDB). See # https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html or # https://mariadb.com/kb/en/library/mysqldump/ for details. - mysql_databases: + # mysql_databases: # Database name (required if using this hook). Or # "all" to dump all databases on the host. Note # that using this database hook implicitly enables # both read_special and one_file_system (see # above) to support dump and restore streaming. - - name: ${POSTGRES_DB} + # - name: ${POSTGRES_DB} # Database hostname to connect to. Defaults to # connecting via local Unix socket. - hostname: ${POSTGRES_HOST} + # hostname: ${POSTGRES_HOST} # Port to connect to. Defaults to 3306. # port: 3307 # Username with which to connect to the database. # Defaults to the username of the current user. - username: ${POSTGRES_USER} + # username: ${POSTGRES_USER} # Password with which to connect to the database. # Omitting a password will only work if MySQL is # configured to trust the configured username # without a password. - password: ${POSTGRES_PASSWORD} + # password: ${POSTGRES_PASSWORD} # Additional mysql options to pass directly to # the mysql command that lists available diff --git a/templates/docker-compose/volumes/borgmatic-config/msmtp.env.j2 b/templates/docker-compose/volumes/borgmatic-config/msmtp.env.j2 new file mode 100644 index 0000000..e686676 --- /dev/null +++ b/templates/docker-compose/volumes/borgmatic-config/msmtp.env.j2 @@ -0,0 +1 @@ +MAIL_PASSWORD={{ smtp_password }}