diff --git a/CHANGELOG.md b/CHANGELOG.md index d796970..520c05a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ CHANGELOG ========= +In Development +-------------- + +* Fixed S3 backups which broke with duplicity 0.8.23. + Version 56 (January 19, 2022) ----------------------------- diff --git a/management/backup.py b/management/backup.py index bc3ac5b..26a61a0 100755 --- a/management/backup.py +++ b/management/backup.py @@ -59,7 +59,7 @@ def backup_status(env): "--archive-dir", backup_cache_dir, "--gpg-options", "--cipher-algo=AES256", "--log-fd", "1", - config["target"], + get_duplicity_target_url(config), ] + get_duplicity_additional_args(env), get_duplicity_env_vars(env), trap=True) @@ -190,13 +190,45 @@ def get_passphrase(env): return passphrase +def get_duplicity_target_url(config): + target = config["target"] + + if get_target_type(config) == "s3": + from urllib.parse import urlsplit, urlunsplit + target = list(urlsplit(target)) + + # Duplicity now defaults to boto3 as the backend for S3, but we have + # legacy boto installed (boto3 doesn't support Ubuntu 18.04) so + # we retarget for classic boto. + target[0] = "boto+" + target[0] + + # In addition, although we store the S3 hostname in the target URL, + # duplicity no longer accepts it in the target URL. The hostname in + # the target URL must be the bucket name. The hostname is passed + # via get_duplicity_additional_args. Move the first part of the + # path (the bucket name) into the hostname URL component, and leave + # the rest for the path. + target[1], target[2] = target[2].lstrip('/').split('/', 1) + + target = urlunsplit(target) + + return target + def get_duplicity_additional_args(env): config = get_backup_config(env) + if get_target_type(config) == 'rsync': return [ "--ssh-options= -i /root/.ssh/id_rsa_miab", "--rsync-options= -e \"/usr/bin/ssh -oStrictHostKeyChecking=no -oBatchMode=yes -p 22 -i /root/.ssh/id_rsa_miab\"", ] + elif get_target_type(config) == 's3': + # See note about hostname in get_duplicity_target_url. + from urllib.parse import urlsplit, urlunsplit + target = urlsplit(config["target"]) + endpoint_url = urlunsplit(("https", target.netloc, '', '', '')) + return ["--s3-endpoint-url", endpoint_url] + return [] def get_duplicity_env_vars(env): @@ -277,7 +309,7 @@ def perform_backup(full_backup): "--volsize", "250", "--gpg-options", "--cipher-algo=AES256", env["STORAGE_ROOT"], - config["target"], + get_duplicity_target_url(config), "--allow-source-mismatch" ] + get_duplicity_additional_args(env), get_duplicity_env_vars(env)) @@ -296,7 +328,7 @@ def perform_backup(full_backup): "--verbosity", "error", "--archive-dir", backup_cache_dir, "--force", - config["target"] + get_duplicity_target_url(config) ] + get_duplicity_additional_args(env), get_duplicity_env_vars(env)) @@ -311,7 +343,7 @@ def perform_backup(full_backup): "--verbosity", "error", "--archive-dir", backup_cache_dir, "--force", - config["target"] + get_duplicity_target_url(config) ] + get_duplicity_additional_args(env), get_duplicity_env_vars(env)) @@ -349,7 +381,7 @@ def run_duplicity_verification(): "--compare-data", "--archive-dir", backup_cache_dir, "--exclude", backup_root, - config["target"], + get_duplicity_target_url(config), env["STORAGE_ROOT"], ] + get_duplicity_additional_args(env), get_duplicity_env_vars(env)) @@ -361,7 +393,7 @@ def run_duplicity_restore(args): "/usr/bin/duplicity", "restore", "--archive-dir", backup_cache_dir, - config["target"], + get_duplicity_target_url(config), ] + get_duplicity_additional_args(env) + args, get_duplicity_env_vars(env)) diff --git a/management/status_checks.py b/management/status_checks.py index c66bf9b..220e23e 100755 --- a/management/status_checks.py +++ b/management/status_checks.py @@ -253,6 +253,18 @@ def check_free_disk_space(rounded_values, env, output): if rounded_values: disk_msg = "The disk has less than 15% free space." output.print_error(disk_msg) + # Check that there's only one duplicity cache. If there's more than one, + # it's probably no longer in use, and we can recommend clearing the cache + # to save space. The cache directory may not exist yet, which is OK. + backup_cache_path = os.path.join(env['STORAGE_ROOT'], 'backup/cache') + try: + backup_cache_count = len(os.listdir(backup_cache_path)) + except: + backup_cache_count = 0 + if backup_cache_count > 1: + output.print_warning("The backup cache directory {} has more than one backup target cache. Consider clearing this directory to save disk space." + .format(backup_cache_path)) + def check_free_memory(rounded_values, env, output): # Check free memory. percent_free = 100 - psutil.virtual_memory().percent