]> WPIA git - infra.git/blob - backup
chg: puppetlabs-postgresql requires stdlib < 5.0.0
[infra.git] / backup
1 #!/bin/bash
2 printf "Backup service ready\n"
3 postgres_cluster=9.6/main
4 if ! which chunked >/dev/null; then
5     printf "Installing chunked en-/decoder\n" >&2
6     # Install signing key for debian repo 'deb2.dogcraft.de' where 'chunked' is hosted. Downloadable from "http://deb.dogcraft.de/signer.gpg".
7     apt-key add - >&2 <<EOF
8 -----BEGIN PGP PUBLIC KEY BLOCK-----
9 Version: GnuPG v1
10
11 mQINBFRfgHwBEADKRVJpTEVbB6W37ZnIrh0xDRcsTqNgOIJTi/ZsloxN0c9/G8BU
12 knEHMpT++qqG3A6pFfg5rci4mEIirt3CoEJw40asyGVSTCl93PNPyuN4ArYxsAL3
13 y9lTBKy2UdSorbmrDNUgc+jkevlQ+xcXBypOOY0zDST5402Wfiyk+VVkBDzlLo8L
14 q1VsrVVZGqgIfC5D+p2SeRfVxjzDRuDBDS+ifZaxR57bx2bJrrA92C6r4Qo3i7CQ
15 IM955G44BU2k/HSaEbl7woXoxb9DTRzvmJ1/m0HT9cl3ak7Zl/UXhJlQd992/e98
16 gkX6S1UtFsIp+fmfWtGyeySRH0av3s2i8gcGwUncyyvQ3XIJJcUisaDxrlK4K6ZH
17 3XSpApgRoUWp0yDUxELme+rXxd6S32DHxHGbjgD8Crus99GHa9OaBSjAJ2vDp1+d
18 wF6Ol/luaZIhGopI4dOhrscZBl8PE8jDsJbOMMpj+KgWD64nRzmnBGYrvhsBaeUh
19 EtvnGCiPiK1ojO49ovxVaxkniIyXyZrej8wAHFBEoC7+KaXR0xml5HPonPOIPYSl
20 UHmOPYB+3EehJHDt1p2lAUkjFUNppUyLaArX0PZV3I8mgm5PFwbXgooqWwj1C9kW
21 cTU1b81KzEKHxf1CoG/rxRTu6qBzTn5yxs03k8uq/Yn534H5GGKwKm1KswARAQAB
22 tChEb2djcmFmdCBEZWIgUHVibGlzaGVyIDxkZWJAZG9nY3JhZnQuZGU+iQI+BBMB
23 AgAoBQJUX4B8AhsDBQkHhM4ABgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRCm
24 Mbavn9PflNi6EACw9TlR0LL/3RJ9QM0TUfL0Pn5l+NuANL0HUmMepjbEFSxLnmlA
25 ypPP0zhi07Ig6Pxkh0Ni9PYBdxkbtFT5ts0SBBcCYvdE8ZqtU7Yx9YtICrHfcx7Y
26 yUsvtvubFfvAT3QfXA1C3PHf4ocyQgW6TdqV5jxmbNi0orhggyBZgmVvvheeCMe2
27 4dIHqA2Ny3ODSON6pNF4q8QXL1dLU4AvYcWX+79ROMnyv/woMiNdNYFB+9ylSN9U
28 sHLu9deXpaxJR25mpk8NuWgyGQUYwtyqySO2ZAFqlC2pdpZ3hRhxi/biPG0XbtF3
29 MJQznfNxmf83FfS3u9fHLrZuCkrwbSBXeniKPmlF1aG7ZpGRIRuDUtNhEkgc2tmi
30 4tIG/brg0B9rpm2XEccDPxEO14MminXho6IImwpbhiKqnBfOKFv8OX0zzjQkUaFn
31 AFuytwyaNn8dBv5WeNNC+8KwBKoDaPUg4JcgzUyXATz0+SIuMBRqWN1q/rz4GCCz
32 Lf/TqEl41h8p1weM33md0p9Fa/xQRe3X9ChVbGyGylaIhniol0IcgROa3t04cO9x
33 xdh/wvgmh7wodCE3mK/G3jcTnQcBh6T6qr/z9PqToZJoRFqjPLv78UmnTJ6Za/ZX
34 uXeRJJf1C3Nswjfwexo2CeWbbQ3X+PGLcDgUmTG7yxDjKL/8rJ0DWWQ8fg==
35 =C8K/
36 -----END PGP PUBLIC KEY BLOCK-----
37 EOF
38     cp -v modules/lxc/files/dogcraft.list /etc/apt/sources.list.d/ >&2
39     apt-get update >&2
40     apt-get install -y chunked >&2
41 fi
42
43 while :; do
44     read -r command || break
45     if [[ $command == base ]]; then
46         sudo lxc-attach -n postgres-primary -- su -c 'pg_basebackup -Ft -z -D - -P' postgres | chunked
47         printf "base backup done\n"
48     elif [[ $command == incremental ]]; then
49         read -e req_name
50         read -e req_hash
51         if ! grep -qi -- '^[A-Z0-9]\{24\}$' <<< $req_name; then
52             printf "Error: invalid WAL-name.\n" >&2
53             printf "Error\n"
54         fi
55         real_hash="$(sha256sum "/data/postgres/data/archive/$req_name" | cut -d" " -f1)"
56         if [[ $req_hash != "-" ]] && [[ $real_hash != $req_hash ]]; then
57             printf "Error: hash mismatch on expected %s != provided %s\n" "$real_hash" "$req_hash" >&2
58             printf "Error\n"
59             exit 1
60         fi
61         printf "Ready\n"
62
63         files=( /data/postgres/data/archive/* )
64         for i in "${files[@]}"; do
65             name="$(basename $i)"
66             if ( [[ "$name" != "$req_name".* ]] && [[ $name > $req_name ]] ) || ( ( [[ "$name" == "$req_name".* ]] || [[ $req_name == $name ]] ) && [[ $req_hash == "-" ]] ); then
67                 printf "%s\n" "$name"
68             fi
69         done | tar cz -C /data/postgres/data/archive -T - | chunked
70         printf "incremental backup done, confirm cleanup!\n"
71         read -e confirmation
72         if [[ $confirmation == "y" ]]; then
73             printf "Cleaning up archive\n" >&2
74             lxc-attach -n postgres-primary -- pg_archivecleanup /var/lib/postgresql/archive/ "$req_name"
75         else
76             printf "Not doing cleanup\n" >&2
77         fi
78     elif [[ $command == restore ]]; then
79         # for now (and quick development) we override
80         rm -R /data/postgres/data
81
82         if [[ -d /data/postgres/data ]]; then
83             printf "error\n"
84             exit 1
85         fi
86         printf "postgres base\n"
87         mkdir -p "/data/postgres/data/${postgres_cluster}"
88         chunked decode > /data/postgres/data/pg_base.tar.gz
89         mkdir -p /data/postgres/data/restore
90         while :; do
91             printf "incremental?\n"
92             read -e inc
93             if [[ $inc != "y" ]]; then
94                 break
95             fi
96             chunked decode | tar xvz -C /data/postgres/data/restore >&2
97         done
98         cat > "/data/postgres/data/${postgres_cluster}/recovery.conf" <<EOF
99 restore_command = 'cp /var/lib/postgresql/restore/%f "%p"'
100 archive_cleanup_command = 'pg_archivecleanup /var/lib/postgresql/restore %r'
101 recovery_end_command = 'touch /var/lib/postgresql/postgres-ready'
102 EOF
103         mkdir -p "/data/postgres/conf/${postgres_cluster}"
104         printf "auto\n" > "/data/postgres/conf/${postgres_cluster}/start.conf"
105         touch "/data/postgres/conf/${postgres_cluster}/postgresql.conf"
106     elif [[ $command == journal ]]; then
107         read -e from
108         action=$(date -u -d '00:00 today' +%s)
109         if [[ $action == $from ]]; then
110             printf "no journals\n"
111             continue
112         fi
113         printf "Until: %s\n" "$action"
114         for i in $(sudo lxc-ls); do
115             [[ $i == "base-image" ]] && continue
116             printf "journal: %s\n" "$i"
117             if [[ $from == - ]]; then
118                 lxc-attach -n "$i" -- journalctl --utc --until="@$action" -o export | chunked
119             else
120                 lxc-attach -n "$i" -- journalctl --utc --since="@$from" --until="@$action" -o export | chunked
121             fi
122         done
123         printf "end-of-journals\n"
124     elif [[ $command == end ]]; then
125         printf "end\n"
126     fi
127 done