1309 lines
46 KiB
Python
Executable File
1309 lines
46 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
"""
|
|
Program: Rod's Incremental Backup System (ribs)
|
|
Author: Rod Wright
|
|
Date: 03/19/2025
|
|
|
|
Copyright (C) 2025 Rod Wright
|
|
|
|
SPDX-License-Identifier: GPL-2.0
|
|
"""
|
|
|
|
import subprocess
|
|
import sys
|
|
import os
|
|
import shutil
|
|
import glob
|
|
from datetime import datetime, timedelta
|
|
import textwrap
|
|
import argparse
|
|
import configparser
|
|
from pathlib import Path
|
|
import re
|
|
|
|
# App version
|
|
APP_VERSION="2.0"
|
|
|
|
# File paths
|
|
SYS_CONF=Path('/etc/ribs')
|
|
USER_CONF=Path.home().joinpath(Path('.ribs'))
|
|
|
|
AVAILABLE=Path('conf-available')
|
|
ENABLED=Path('conf-enabled')
|
|
CONF_TEMPLATE=Path('ribs.conf.sample')
|
|
|
|
SYS_AVAILABLE=SYS_CONF.joinpath(AVAILABLE)
|
|
SYS_ENABLED=SYS_CONF.joinpath(ENABLED)
|
|
USER_AVAILABLE=USER_CONF.joinpath(AVAILABLE)
|
|
USER_ENABLED=USER_CONF.joinpath(ENABLED)
|
|
|
|
|
|
# Determine if I have superuser privileges
|
|
if os.geteuid()==0:
|
|
superuser=True
|
|
else:
|
|
superuser=False
|
|
|
|
# ****** BEGIN function definitions ******
|
|
|
|
def main():
|
|
"""
|
|
Main function
|
|
|
|
Args:
|
|
No arguments
|
|
|
|
Returns:
|
|
Nothing
|
|
|
|
"""
|
|
|
|
print(f"Rod's Incremental Backup System, version {APP_VERSION}")
|
|
print("")
|
|
|
|
if not superuser:
|
|
if not USER_CONF.exists():
|
|
print("Creating user conf directory")
|
|
USER_CONF.mkdir()
|
|
USER_AVAILABLE.mkdir()
|
|
USER_ENABLED.mkdir()
|
|
user_conf_template=USER_AVAILABLE.joinpath(CONF_TEMPLATE)
|
|
sys_conf_template=SYS_AVAILABLE.joinpath(CONF_TEMPLATE)
|
|
if user_conf_template.is_file():
|
|
user_conf_template.unlink()
|
|
if not user_conf_template.is_symlink():
|
|
user_conf_template.symlink_to(sys_conf_template)
|
|
|
|
print("")
|
|
cmdline=parse_cmdline()
|
|
|
|
# Set flags
|
|
global simulate
|
|
simulate=cmdline['simulate']
|
|
|
|
# Dispatch operation
|
|
match cmdline['operation']:
|
|
case 'config':
|
|
config_menu()
|
|
case 'list':
|
|
config_list()
|
|
case 'enable'|'disable':
|
|
config_action(cmdline['operation'],cmdline['conf_file_arg'])
|
|
case 'run':
|
|
run_backup(cmdline['conf_file_arg'])
|
|
case _:
|
|
print("Invalid operation")
|
|
|
|
|
|
|
|
def parse_cmdline():
|
|
"""
|
|
Parses command line arguments
|
|
|
|
Args:
|
|
No arguments
|
|
|
|
Returns:
|
|
dict(cmdline):
|
|
{
|
|
simulate: True | False,
|
|
operation: 'run' | 'config' | 'enable' | 'disable' | None,
|
|
conf_file_arg: 'conf_file'|None
|
|
}
|
|
"""
|
|
cmdline={}
|
|
|
|
parser=argparse.ArgumentParser(description='Rod\'s Incremental Backup System, version '+APP_VERSION)
|
|
group=parser.add_mutually_exclusive_group()
|
|
|
|
parser.add_argument(
|
|
'-s', '--simulate',
|
|
help='simulate operations to be performed',
|
|
action="store_true"
|
|
)
|
|
|
|
group.add_argument(
|
|
'-c', '--config',
|
|
help='create, remove, modify, enable, or disable config files',
|
|
action="store_true"
|
|
)
|
|
|
|
group.add_argument(
|
|
'-l', '--list-configs',
|
|
help='list all config files',
|
|
action="store_true"
|
|
)
|
|
|
|
group.add_argument(
|
|
'-e', '--enable-config',
|
|
metavar='CONF_FILE',
|
|
help='enable a config file',
|
|
)
|
|
|
|
group.add_argument(
|
|
'-d', '--disable-config',
|
|
metavar='CONF_FILE',
|
|
help='disable a config file',
|
|
)
|
|
|
|
group.add_argument(
|
|
"conf_file",
|
|
help="Run RIBS using the specified config file. If none specified, run all enabled config files, looking in '~/.ribs/conf-enabled' for standard users, or in '/etc/ribs/conf-enabled' for superusers.",
|
|
nargs='?'
|
|
)
|
|
|
|
args=parser.parse_args()
|
|
|
|
# Flags
|
|
if args.simulate:
|
|
cmdline['simulate']=True
|
|
else:
|
|
cmdline['simulate']=False
|
|
|
|
# Operations
|
|
if args.config:
|
|
cmdline['operation']='config'
|
|
cmdline['conf_file_arg']=None
|
|
elif args.list_configs:
|
|
cmdline['operation']='list'
|
|
elif args.enable_config:
|
|
cmdline['operation']='enable'
|
|
cmdline['conf_file_arg']=args.enable_config
|
|
elif args.disable_config:
|
|
cmdline['operation']='disable'
|
|
cmdline['conf_file_arg']=args.disable_config
|
|
else:
|
|
cmdline['operation']='run'
|
|
cmdline['conf_file_arg']=args.conf_file
|
|
|
|
|
|
return cmdline
|
|
|
|
|
|
def run_rsync(source, destination, options=None, excludes=None):
|
|
"""
|
|
Runs rsync with the given source, destination, options, and excludes.
|
|
|
|
Args:
|
|
source (str): The source path.
|
|
destination (str): The destination path.
|
|
options (list, optional): A list of rsync options. Defaults to None.
|
|
excludes (list, optional): A list of patterns to exclude. Defaults to None.
|
|
|
|
|
|
Returns:
|
|
int: The return code of the rsync command.
|
|
"""
|
|
command = ["rsync"]
|
|
if options:
|
|
command.extend(options)
|
|
if simulate:
|
|
command.extend(['--dry-run'])
|
|
if excludes:
|
|
for pattern in excludes:
|
|
command.extend(['--exclude='+pattern])
|
|
|
|
command.extend([source, destination])
|
|
#print(command)
|
|
process = subprocess.run(command, capture_output=True, text=True)
|
|
|
|
if '-v' in options or '--verbose' in options:
|
|
print(process.stdout)
|
|
|
|
if process.returncode != 0:
|
|
print(f"Error running rsync: {process.stderr}")
|
|
return process.returncode
|
|
|
|
|
|
def mount_storage(mountpoint,operation,mountparams=None):
|
|
"""
|
|
Handle mounting and unmounting of offline storage.
|
|
|
|
Args:
|
|
mountpoint (str or path object): The directory on the filesystem
|
|
where something will be mounted.
|
|
operation (str): The mount operation. one of:
|
|
'ro': mount read-only
|
|
'rw': mount read-write
|
|
'u': unmount
|
|
mountparams (str, optional): The mount parameters. This would be a string
|
|
containing the arguments to the mount(8) command, up to but
|
|
not including the mount point.
|
|
|
|
Returns:
|
|
int: The return code of the command
|
|
"""
|
|
# Check to see if mountpoint exists
|
|
try:
|
|
if not mountpoint.is_dir():
|
|
raise Exception(f"Error: The mount point {mountpoint} does not exist or is not a directory. This is a fatal error and ribs cannot continue with this config file. Please check the config file: {config}, or create the mountpoint.")
|
|
except Exception as errmsg:
|
|
print("")
|
|
print(errmsg)
|
|
return 1
|
|
|
|
# Check to see if something is already mounted there
|
|
if mountpoint.is_mount():
|
|
mounted=True
|
|
# Check to see if it is writeable
|
|
try:
|
|
writefile=mountpoint.joinpath(Path('writetest.tmp'))
|
|
writefile.touch(exist_ok=True)
|
|
writefile.unlink(missing_ok=True)
|
|
except Exception:
|
|
# May be mounted read-only.
|
|
writeable=False
|
|
else:
|
|
writeable=True
|
|
else:
|
|
mounted=False
|
|
|
|
# Build the command
|
|
if operation=='u':
|
|
mountcmd=['umount']
|
|
mountcmd.extend([str(mountpoint)])
|
|
if not mounted:
|
|
# Nothing to do
|
|
mountcmd.clear()
|
|
elif operation=='ro':
|
|
mountcmd=['mount']
|
|
if mounted:
|
|
# Already mounted
|
|
if not writeable:
|
|
# Already read-only
|
|
mountcmd.clear()
|
|
else:
|
|
# We are mounted read-write. Attempt to remount read-only.
|
|
if mountparams:
|
|
mountcmd.extend([mountparams])
|
|
mountcmd.extend([str(mountpoint)])
|
|
mountcmd.extend(['-oremount,ro'])
|
|
else:
|
|
# Not yet mounted
|
|
if mountparams:
|
|
mountcmd.extend([mountparams])
|
|
mountcmd.extend([str(mountpoint)])
|
|
mountcmd.extend(['-oro'])
|
|
elif operation=='rw':
|
|
mountcmd=['mount']
|
|
if mounted:
|
|
# Already mounted
|
|
if writeable:
|
|
# Already writeable
|
|
mountcmd.clear()
|
|
else:
|
|
# We are mounted read-only. Attempt to remount read-write.
|
|
if mountparams:
|
|
mountcmd.extend([mountparams])
|
|
mountcmd.extend([str(mountpoint)])
|
|
mountcmd.extend(['-oremount,rw'])
|
|
else:
|
|
# Not yet mounted
|
|
if mountparams:
|
|
mountcmd.extend([mountparams])
|
|
mountcmd.extend([str(mountpoint)])
|
|
mountcmd.extend(['-orw'])
|
|
else:
|
|
print("Unknown operation passed to mount_storage")
|
|
return 1
|
|
|
|
if len(mountcmd)>0:
|
|
mountproc=subprocess.run(mountcmd, capture_output=True, text=True)
|
|
if mountproc.returncode != 0:
|
|
print(f"Error occurred while attempting to mount or unmount: {mountproc.stderr}")
|
|
return mountproc.returncode
|
|
else:
|
|
return 0
|
|
|
|
|
|
def config_menu():
|
|
"""
|
|
Show a menu of operations to perform on config files.
|
|
|
|
Args:
|
|
none
|
|
|
|
Returns:
|
|
nothing
|
|
"""
|
|
while True:
|
|
print("Select an action",end="")
|
|
if simulate:
|
|
print("(simulated):\n")
|
|
else:
|
|
print(":\n")
|
|
print("1. [Llist config files")
|
|
print("2. [C]reate a config file")
|
|
print("3. [R]emove an available config file")
|
|
print("4. [M]odify an available config file")
|
|
print("5. [E]nable an available config file")
|
|
print("6. [D]isable an enabled config file")
|
|
#print("H. Config file help")
|
|
print("Q. Quit")
|
|
|
|
selection = input("\nEnter your selection: ")
|
|
|
|
match selection:
|
|
case '1'|'l'|'L':
|
|
config_list()
|
|
input("\nPress Enter to continue...")
|
|
print("")
|
|
case '2'|'c'|'C':
|
|
config_create()
|
|
input("\nPress Enter to continue...")
|
|
print("")
|
|
case '3'|'r'|'R':
|
|
config_action('remove')
|
|
input("\nPress Enter to continue...")
|
|
print("")
|
|
case '4'|'m'|'M':
|
|
config_action('modify')
|
|
input("\nPress Enter to continue...")
|
|
print("")
|
|
case '5'|'e'|'E':
|
|
config_action('enable')
|
|
input("\nPress Enter to continue...")
|
|
print("")
|
|
case '6'|'d'|'D':
|
|
config_action('disable')
|
|
input("\nPress Enter to continue...")
|
|
print("")
|
|
# case 'h' | 'H':
|
|
# config_help()
|
|
# input("\nPress Enter to continue...")
|
|
# print("")
|
|
case 'q' | 'Q':
|
|
print("")
|
|
return 0
|
|
case _:
|
|
print("Invalid selection. Please try again.")
|
|
|
|
print("")
|
|
|
|
|
|
def config_list():
|
|
"""
|
|
Find and print a list of config files.
|
|
|
|
Args:
|
|
No arguments
|
|
|
|
Returns:
|
|
Nothing
|
|
"""
|
|
if superuser:
|
|
conftype='system'
|
|
availconfdir=SYS_AVAILABLE
|
|
enaconfdir=SYS_ENABLED
|
|
else:
|
|
conftype='user'
|
|
availconfdir=USER_AVAILABLE
|
|
enaconfdir=USER_ENABLED
|
|
|
|
configs=find_configs(conftype)
|
|
|
|
print(f"{conftype} config files:\n")
|
|
if len(configs)==0:
|
|
wrap_message(f"No available {conftype} config files found. Refer to the sample config file {SYS_AVAILABLE}/ribs.conf.sample to create one. Note that config files must have the .conf extension to be recognized.")
|
|
else:
|
|
for config in configs:
|
|
if Path(enaconfdir/config).is_symlink():
|
|
enabled_ind='enabled ------>'
|
|
else:
|
|
enabled_ind='available ---->'
|
|
print(f"{enabled_ind} {config}")
|
|
print("")
|
|
|
|
|
|
def config_action(action,configfile=None):
|
|
"""
|
|
Take some action on a config file.
|
|
|
|
Args:
|
|
action (str): The action to take.
|
|
one of: 'enable'|'disable'|'modify'|'remove'
|
|
configfile (str, optional): The filename of a configuration file
|
|
|
|
Returns:
|
|
int: 0 on success or 1 on failure
|
|
"""
|
|
if simulate:
|
|
print("\nAll actions will be simulated")
|
|
|
|
if superuser:
|
|
conftype='system'
|
|
availconfdir=SYS_AVAILABLE
|
|
enaconfdir=SYS_ENABLED
|
|
else:
|
|
conftype='user'
|
|
availconfdir=USER_AVAILABLE
|
|
enaconfdir=USER_ENABLED
|
|
|
|
action_file=None
|
|
|
|
match action:
|
|
case 'disable':
|
|
configs=find_configs(conftype,confstatus='enabled')
|
|
case 'enable':
|
|
aconfigs=find_configs(conftype)
|
|
econfigs=find_configs(conftype,confstatus='enabled')
|
|
configs=[conf for conf in aconfigs if conf not in econfigs]
|
|
case 'modify'|'remove':
|
|
configs=find_configs(conftype)
|
|
|
|
if len(configs)==0:
|
|
print("No config files were found.")
|
|
return 1
|
|
|
|
if configfile:
|
|
configfile=Path(configfile)
|
|
if configfile in configs:
|
|
# We're good. That's what we'll use.
|
|
action_file=configfile
|
|
else:
|
|
# The config file supplied doesn't exist
|
|
print(f"The config file {configfile} does not exist.")
|
|
return 1
|
|
else:
|
|
action_file=select_config(configs,action=action)
|
|
|
|
match action:
|
|
case 'enable':
|
|
if simulate:
|
|
print("Simulating enabling",Path(action_file).name)
|
|
else:
|
|
print("Enabling config:",Path(action_file).name)
|
|
link=Path(enaconfdir/action_file)
|
|
target=Path(availconfdir/action_file)
|
|
try:
|
|
link.symlink_to(target)
|
|
except FileExistsError:
|
|
print("That config file is already enabled.")
|
|
return 1
|
|
|
|
case 'disable':
|
|
if simulate:
|
|
print("Simulating disabling",Path(action_file).name)
|
|
else:
|
|
print("Disabling config:",Path(action_file).name)
|
|
link=Path(enaconfdir/action_file)
|
|
try:
|
|
link.unlink()
|
|
except FileNotFoundError:
|
|
print("That config file is already disabled.")
|
|
return 1
|
|
|
|
case 'modify':
|
|
if simulate:
|
|
print("Simulating modifying",Path(action_file).name)
|
|
else:
|
|
print("Modifying config:",action_file)
|
|
filepath_to_mod=Path(availconfdir/action_file)
|
|
editor = os.environ.get('EDITOR')
|
|
if editor:
|
|
subprocess.run([editor, filepath_to_mod])
|
|
else:
|
|
try:
|
|
result=subprocess.run(['xdg-open', filepath_to_mod], stderr = subprocess.DEVNULL)
|
|
if result.returncode!=0:
|
|
raise ValueError
|
|
except (ValueError, FileNotFoundError):
|
|
try:
|
|
subprocess.run(['vi', filepath_to_mod])
|
|
except FileNotFoundError:
|
|
try:
|
|
subprocess.run(['nano', filepath_to_mod])
|
|
except FileNotFoundError:
|
|
print("No default editor found. Please set the $EDITOR environment variable or install a common text editor.")
|
|
return 1
|
|
|
|
case 'remove':
|
|
if simulate:
|
|
print("Simulating removing",Path(action_file).name)
|
|
else:
|
|
print("Removing config:",action_file)
|
|
ena_to_remove=Path(enaconfdir/action_file)
|
|
avail_to_remove=Path(availconfdir/action_file)
|
|
try:
|
|
if ena_to_remove.is_symlink():
|
|
ena_to_remove.unlink()
|
|
avail_to_remove.unlink()
|
|
except FileNotFoundError:
|
|
print("That config file has already been removed.")
|
|
return 1
|
|
|
|
case _:
|
|
print("Invalid action passed to config_action()")
|
|
return 1
|
|
|
|
return 0
|
|
|
|
|
|
def config_create():
|
|
"""
|
|
Create a config file.
|
|
|
|
Args:
|
|
none
|
|
|
|
Returns:
|
|
Nothing
|
|
"""
|
|
if simulate:
|
|
print("\nAll actions will be simulated")
|
|
if superuser:
|
|
conftype='system'
|
|
availconfdir=SYS_AVAILABLE
|
|
else:
|
|
conftype='user'
|
|
availconfdir=USER_AVAILABLE
|
|
|
|
# Prompt for a filename that doesn't already exist
|
|
print("\nCreating a new config file using ribs.conf.sample as the template.\n")
|
|
print("Please enter a filename. The file must not already exist")
|
|
print("and the filename extension must be '.conf'.")
|
|
print("Enter c to cancel.")
|
|
fnvalid=False
|
|
while not fnvalid:
|
|
newfn=input("\nfilename: ")
|
|
existing_confs=find_configs(conftype)
|
|
if newfn=='c' or newfn=='C':
|
|
return 0
|
|
elif Path(newfn) in existing_confs:
|
|
print("That file already exists.")
|
|
print("Try again or enter c to cancel.")
|
|
elif not newfn.endswith('.conf'):
|
|
print("The filename must end with '.conf'.")
|
|
print("Try again or enter c to cancel.")
|
|
else:
|
|
fnvalid=True
|
|
|
|
# Copy ribs.conf.sample to a new file with the provided filename and edit.
|
|
if simulate:
|
|
print("Simulating creating ",newfn)
|
|
else:
|
|
try:
|
|
shutil.copy(availconfdir.joinpath(Path('ribs.conf.sample')), availconfdir.joinpath(Path(newfn)))
|
|
except Exception as e:
|
|
print(f"An error occurred: {e}")
|
|
return 1
|
|
print(f"New config file {newfn} created. Opening for editing...")
|
|
config_action('modify',newfn)
|
|
print("...done. In order to run this config file, you must enable it.")
|
|
|
|
|
|
def find_configs(conftype,confstatus='available'):
|
|
"""
|
|
Find config files of the specified type.
|
|
|
|
Args:
|
|
conftype (str): 'user' | 'system'.
|
|
confstatus (str, optional): ''available' | 'enabled'
|
|
|
|
Returns:
|
|
list(configs): A list of names of config files
|
|
"""
|
|
if conftype=='user':
|
|
if confstatus=='enabled':
|
|
dirpath=USER_ENABLED
|
|
elif confstatus=='available':
|
|
dirpath=USER_AVAILABLE
|
|
elif conftype=='system':
|
|
if confstatus=='enabled':
|
|
dirpath=SYS_ENABLED
|
|
elif confstatus=='available':
|
|
dirpath=SYS_AVAILABLE
|
|
|
|
pattern="*.conf"
|
|
configs = []
|
|
matching_files=sorted(dirpath.glob(pattern))
|
|
if matching_files:
|
|
for filepath in matching_files:
|
|
filename=Path(filepath.name)
|
|
configs.append(filename)
|
|
|
|
return(configs)
|
|
|
|
|
|
def select_config(configfiles,action=None):
|
|
"""
|
|
Show a menu of config files of the provided list, allow the user to select one, and return the path of the selected one.
|
|
|
|
Args:
|
|
configfiles (list): a list of config files.
|
|
|
|
Returns:
|
|
None or path object: A path object of a config file.
|
|
"""
|
|
if not configfiles:
|
|
return None
|
|
|
|
if action:
|
|
print(f"Select a config file to {action}:\n")
|
|
else:
|
|
print(f"Select a config file:\n")
|
|
|
|
for i, item in enumerate(configfiles):
|
|
print(f"{i + 1}. {item}")
|
|
print("Q. Quit")
|
|
while True:
|
|
try:
|
|
selection = input("\nEnter your selection: ")
|
|
if selection=='q' or selection=='Q':
|
|
return None
|
|
selection=int(selection)
|
|
if 1 <= selection <= len(configfiles):
|
|
return configfiles[selection - 1]
|
|
else:
|
|
print("Invalid selection. Please try again.")
|
|
except ValueError:
|
|
print("Invalid input. Please enter a number.")
|
|
|
|
|
|
def parse_config_file(conf_file):
|
|
"""
|
|
Read the specified config file and return the parameters.
|
|
|
|
Args:
|
|
conf_file (str or path object): The full path of a configuration
|
|
file.
|
|
|
|
Returns:
|
|
dict: A dictionary of option:value pairs
|
|
"""
|
|
config=configparser.ConfigParser(allow_no_value=True)
|
|
config.read(conf_file)
|
|
|
|
# Check for the [General] section and its options
|
|
config_errors=[]
|
|
if config.has_section('General'):
|
|
if config.has_option('General','source_dir'):
|
|
source_dir=config['General']['source_dir']
|
|
else:
|
|
config_errors.append('source_dir option missing from config file')
|
|
if config.has_option('General','backup_root'):
|
|
backup_root=config['General']['backup_root']
|
|
else:
|
|
config_errors.append('backup_root option missing from config file')
|
|
if config.has_option('General','backup_subdir'):
|
|
backup_subdir=config['General']['backup_subdir']
|
|
else:
|
|
backup_subdir=None
|
|
if config.has_option('General','exclude_list'):
|
|
exclude_list=config['General']['exclude_list'].split(',')
|
|
exclude_list=[pattern.strip() for pattern in exclude_list]
|
|
else:
|
|
exclude_list=None
|
|
if config.has_option('General','day_limit'):
|
|
day_limit=config['General']['day_limit']
|
|
else:
|
|
config_errors.append('day_limit option missing from config file')
|
|
if config.has_option('General','storage_type'):
|
|
storage_type=config['General']['storage_type']
|
|
else:
|
|
config_errors.append('storage_type option missing from config file')
|
|
else:
|
|
config_errors.append('General section missing from config file')
|
|
|
|
# Abort on [General] section errors
|
|
if len(config_errors)>0:
|
|
print("The following configuration file errors were found")
|
|
for err_msg in config_errors:
|
|
print(err_msg)
|
|
print("These are fatal errors and ribs cannot continue.")
|
|
print("Please check the config file:",conf_file)
|
|
return 1
|
|
|
|
if storage_type=="local":
|
|
# Check for the [local] section and its options
|
|
config_errors=[]
|
|
if config.has_section('local'):
|
|
# Reserved for future use
|
|
pass
|
|
else:
|
|
pass
|
|
|
|
# Abort on [local] section errors
|
|
if len(config_errors)>0:
|
|
print("The following configuration file errors were found")
|
|
for err_msg in config_errors:
|
|
print(err_msg)
|
|
print("These are fatal errors and ribs cannot continue.")
|
|
print("Please check the config file:",conf_file)
|
|
return 1
|
|
|
|
elif storage_type=="offline":
|
|
# Check for the [offline] section and its options
|
|
config_errors=[]
|
|
if config.has_section('offline'):
|
|
if config.has_option('offline','mount_parameters'):
|
|
mount_parameters=config['offline']['mount_parameters']
|
|
else:
|
|
mount_parameters=None
|
|
if config.has_option('offline','remount_ro'):
|
|
remount_ro=config['offline']['remount_ro']
|
|
if remount_ro=="":
|
|
remount_ro=False
|
|
else:
|
|
remount_ro=False
|
|
else:
|
|
mount_parameters=None
|
|
remount_ro=False
|
|
|
|
# Abort on [offline] section errors
|
|
if len(config_errors)>0:
|
|
print("The following configuration file errors were found")
|
|
for err_msg in config_errors:
|
|
print(err_msg)
|
|
print("These are fatal errors and ribs cannot continue.")
|
|
print("Please check the config file:",conf_file)
|
|
return 1
|
|
|
|
elif storage_type=="remote":
|
|
# Check for the [remote] section and its options
|
|
config_errors=[]
|
|
if config.has_section('remote'):
|
|
if config.has_option('remote','remote_user_host'):
|
|
remote_user_host=config['remote']['remote_user_host']
|
|
else:
|
|
config_errors.append('remote_user_host option missing from config file')
|
|
else:
|
|
config_errors.append('remote section missing from config file')
|
|
|
|
# Abort on [remote] section errors
|
|
if len(config_errors)>0:
|
|
print("The following configuration file errors were found")
|
|
for err_msg in config_errors:
|
|
print(err_msg)
|
|
print("These are fatal errors and ribs cannot continue.")
|
|
print("Please check the config file:",conf_file)
|
|
return 1
|
|
else:
|
|
config_errors.append('invalid storage_type. Must be local, offline, or remote.')
|
|
if len(config_errors)>0:
|
|
print("The following configuration file errors were found")
|
|
for err_msg in config_errors:
|
|
print(err_msg)
|
|
print("These are fatal errors and ribs cannot continue.")
|
|
print("Please check the config file:",conf_file)
|
|
return 1
|
|
|
|
# Check and correct option formatting and abort if necessary
|
|
config_errors=[]
|
|
if not source_dir.startswith('/'):
|
|
config_errors.append('source_dir must be an absolute path, but doesn\'t start with a \'/\'')
|
|
if not backup_root.startswith('/'):
|
|
config_errors.append('backup_root must be an absolute path, but doesn\'t start with a \'/\'')
|
|
if day_limit=='0':
|
|
config_errors.append('day_limit cannot be 0')
|
|
if storage_type=='remote' and not "@" in remote_user_host:
|
|
config_errors.append('remote_user_host must be in the format\'user@host\'')
|
|
|
|
# Abort on unacceptable option format errors
|
|
if len(config_errors)>0:
|
|
print("The following configuration file errors were found")
|
|
for err_msg in config_errors:
|
|
print(err_msg)
|
|
print("These are fatal errors and ribs cannot continue.")
|
|
print("Please check the config file:",conf_file)
|
|
return 1
|
|
|
|
# Clean up variables
|
|
if backup_root.endswith('/'):
|
|
backup_root=backup_root.rstrip('/')
|
|
if backup_subdir:
|
|
if backup_subdir.startswith('/'):
|
|
backup_subdir=backup_subdir.lstrip('/')
|
|
if backup_subdir.endswith('/'):
|
|
backup_subdir=backup_subdir.rstrip('/')
|
|
|
|
# Return a parameter:option dictionary
|
|
if storage_type=="local":
|
|
return {
|
|
'storage_type':storage_type,
|
|
'source_dir':source_dir,
|
|
'backup_root':backup_root,
|
|
'backup_subdir':backup_subdir,
|
|
'exclude_list':exclude_list,
|
|
'day_limit':day_limit
|
|
}
|
|
elif storage_type=="offline":
|
|
return {
|
|
'storage_type':storage_type,
|
|
'source_dir':source_dir,
|
|
'backup_root':backup_root,
|
|
'backup_subdir':backup_subdir,
|
|
'exclude_list':exclude_list,
|
|
'day_limit':day_limit,
|
|
'mount_parameters':mount_parameters,
|
|
'remount_ro':remount_ro
|
|
}
|
|
elif storage_type=="remote":
|
|
return {
|
|
'storage_type':storage_type,
|
|
'source_dir':source_dir,
|
|
'backup_root':backup_root,
|
|
'backup_subdir':backup_subdir,
|
|
'exclude_list':exclude_list,
|
|
'day_limit':day_limit,
|
|
'remote_user_host':remote_user_host
|
|
}
|
|
else:
|
|
# Not sure how we got here, but if we did, something's wrong
|
|
return 1
|
|
|
|
|
|
def update_capacity_file(file_location):
|
|
"""
|
|
Update the capacity file for offline media. Detects the capacity of the specified file location and writes it to a capacity.txt file in that location.
|
|
|
|
Args:
|
|
file_location (path object): The full path of the mount point of offline media.
|
|
|
|
Returns:
|
|
int: The return code of the command
|
|
"""
|
|
cap_file=file_location.joinpath(Path('capacity.txt'))
|
|
|
|
# Check to see if the file_location is mounted
|
|
try:
|
|
if not file_location.is_mount():
|
|
raise Exception(f"Could not find a mounted filesystem at {file_location}. Not creating capacity file.")
|
|
except Exception as errmsg:
|
|
print("")
|
|
print(errmsg)
|
|
return 1
|
|
|
|
# Check to see if the file location is writable
|
|
try:
|
|
cap_file.touch(exist_ok=True)
|
|
cap_file.unlink(missing_ok=True)
|
|
except Exception as errmsg:
|
|
print("")
|
|
print(errmsg)
|
|
print(f"Could not write to the filesystem at {file_location}. Not creating capacity file.")
|
|
return 1
|
|
|
|
# Get free space
|
|
command=['df']
|
|
command.extend(['-h'])
|
|
command.extend([file_location])
|
|
proc=subprocess.run(command, capture_output=True, text=True)
|
|
|
|
if proc.returncode != 0:
|
|
print(f"Error running {command}: {proc.stderr}")
|
|
return proc.returncode
|
|
else:
|
|
capacitytext=proc.stdout
|
|
|
|
# Write capacity info to file
|
|
try:
|
|
cap_file.write_text(capacitytext)
|
|
except Exception as errmsg:
|
|
print(errmsg)
|
|
return 1
|
|
|
|
|
|
def run_backup(configfile=None):
|
|
"""
|
|
Run backup using the specified configfile, or all enabled config
|
|
files if none specified.
|
|
|
|
Args:
|
|
configfile (str or path object, optional): The full path of a
|
|
config file.
|
|
|
|
Returns:
|
|
Nothing
|
|
"""
|
|
runstart=datetime.now()
|
|
print(f"Run started {runstart.ctime()}")
|
|
|
|
if simulate:
|
|
print("\nAll actions will be simulated")
|
|
|
|
if superuser:
|
|
conftype='system'
|
|
availconfdir=SYS_AVAILABLE
|
|
enaconfdir=SYS_ENABLED
|
|
else:
|
|
conftype='user'
|
|
availconfdir=USER_AVAILABLE
|
|
enaconfdir=USER_ENABLED
|
|
|
|
configs=[]
|
|
if configfile==None:
|
|
# Find and run all enabled configs
|
|
print("Finding enabled config files...")
|
|
configs=find_configs(conftype,'enabled')
|
|
if len(configs)>0:
|
|
print(f"Found {conftype} config files:")
|
|
for index, config in enumerate(configs):
|
|
print(config)
|
|
configs[index]=enaconfdir.joinpath(config)
|
|
else:
|
|
print("No enabled config files found")
|
|
else:
|
|
# Check to see if the given config file is in the current directory
|
|
if Path(configfile).is_file():
|
|
configs=[Path(configfile)]
|
|
else:
|
|
# Given file is not in current directory. Look in conf-enabled
|
|
print(f"The config file {configfile} is not in the current directory.")
|
|
print(f"Searching enabled {conftype} config files...")
|
|
enaconfigs=find_configs(conftype,'enabled')
|
|
if len(enaconfigs)>0:
|
|
print(f"Found enabled {conftype} config files...")
|
|
if Path(configfile) in enaconfigs:
|
|
print(f"Found {configfile} in enabled {conftype} config files.")
|
|
configs=[enaconfdir.joinpath(configfile)]
|
|
else:
|
|
print("No enabled config files found.")
|
|
|
|
if len(configs)==0:
|
|
print("Unable to find the config file you specified. Aborting.")
|
|
return 1
|
|
|
|
failed_confs=[]
|
|
for config in configs:
|
|
print("Running backup from the config file:\n",config)
|
|
|
|
# Parse config file and create variables
|
|
conf_parameters=parse_config_file(config)
|
|
|
|
source_dir=conf_parameters.get('source_dir')
|
|
backup_root=Path(conf_parameters.get('backup_root'))
|
|
backup_subdir=conf_parameters.get('backup_subdir')
|
|
if backup_subdir:
|
|
backup_subdir=Path(conf_parameters.get('backup_subdir'))
|
|
exclude_list=conf_parameters.get('exclude_list')
|
|
day_limit=conf_parameters.get('day_limit')
|
|
storage_type=conf_parameters.get('storage_type')
|
|
mount_parameters=conf_parameters.get('mount_parameters')
|
|
remount_ro=conf_parameters.get('remount_ro')
|
|
if remount_ro=='True':
|
|
remount_ro=True
|
|
elif remount_ro=='False':
|
|
remount_ro=False
|
|
remote_user_host=conf_parameters.get('remote_user_host')
|
|
|
|
# Test for ability to mount filesystems
|
|
if storage_type=='offline' and not superuser:
|
|
wrap_message("You have set storage_type to offline in the config file, but you do not have superuser rights, preventing you from mounting filesystems. You can run ribs with sudo or change the storage_type to local and handle the mounting and unmounting outside of ribs. Aborting.")
|
|
failed_confs.append(config)
|
|
continue
|
|
|
|
# Test for existance of backup_root
|
|
if storage_type=='local' or storage_type=='offline':
|
|
print(f"Testing for existance of {backup_root}... ",end="")
|
|
try:
|
|
if not backup_root.is_dir():
|
|
raise Exception(f"{backup_root} does not exist or is not a directory. This is a fatal error and ribs cannot continue. Please check the config file: {config}")
|
|
except Exception as errmsg:
|
|
print("FAILED")
|
|
print("")
|
|
print(errmsg)
|
|
failed_confs.append(config)
|
|
continue
|
|
else:
|
|
print("OK")
|
|
elif storage_type=='remote':
|
|
print(f"Testing for existance of {backup_root}... ",end="",flush=True)
|
|
try:
|
|
command=['ssh', remote_user_host,'if [ -d ',str(backup_root),' ]; then echo \"true\"; else echo \"false\";fi']
|
|
etest=subprocess.run(command, capture_output=True, text=True)
|
|
if etest.stdout=='false':
|
|
raise Exception(f"{backup_root} does not exist or is not a directory. This is a fatal error and ribs cannot continue. Please check the config file: {config}")
|
|
except Exception as errmsg:
|
|
print("FAILED")
|
|
print("")
|
|
print(errmsg)
|
|
failed_confs.append(config)
|
|
continue
|
|
else:
|
|
print("OK")
|
|
|
|
|
|
if storage_type=='offline':
|
|
# Mount offline storage read-write
|
|
try:
|
|
print(f"Mounting {backup_root}... ",end="",flush=True)
|
|
mount_result=mount_storage(backup_root,'rw',mountparams=mount_parameters)
|
|
if mount_result != 0:
|
|
print("FAILED")
|
|
raise Exception(f"Failed to mount {backup_root}")
|
|
else:
|
|
print("OK")
|
|
except Exception as errmsg:
|
|
print("FAILED")
|
|
print("")
|
|
print(errmsg)
|
|
failed_confs.append(config)
|
|
continue
|
|
else:
|
|
print("OK")
|
|
|
|
# Test for writability
|
|
if simulate:
|
|
print("Skipping all write testing since we're only simulating.")
|
|
else:
|
|
testdir=backup_root.joinpath(Path('testdir'))
|
|
testfile=testdir.joinpath(Path('testfile'))
|
|
print(f"Testing for writability of{backup_root}... ",end="",flush=True)
|
|
try:
|
|
if storage_type=='local' or storage_type=='offline':
|
|
testdir.mkdir(exist_ok=True)
|
|
testfile.touch(exist_ok=True)
|
|
testfile.unlink(missing_ok=True)
|
|
shutil.rmtree(testdir)
|
|
elif storage_type=='remote':
|
|
command=['ssh', remote_user_host,'mkdir -p ',str(testdir),'&& touch ',str(testfile),'&& rm -rf ',str(testdir)]
|
|
wtest=subprocess.run(command, capture_output=True, text=True)
|
|
if wtest.returncode != 0:
|
|
raise Exception(f"Could not write to the remote directory {backup_root}")
|
|
except Exception as errmsg:
|
|
print("FAILED")
|
|
print("")
|
|
print(f"{errmsg} \n This is a fatal error and ribs cannot continue. Please check the config file:{config}")
|
|
if storage_type=='offline':
|
|
mount_storage(backup_root,'u')
|
|
failed_confs.append(config)
|
|
continue
|
|
else:
|
|
print("OK")
|
|
|
|
# Test for hard link support
|
|
testlink=testdir.joinpath(Path('testlink'))
|
|
print("Testing for hard link support on",backup_root,"... ",end="",flush=True)
|
|
try:
|
|
if storage_type=='local' or storage_type=='offline':
|
|
testdir.mkdir(exist_ok=True)
|
|
testfile.touch(exist_ok=True)
|
|
testlink.hardlink_to(testfile)
|
|
testlink.unlink(missing_ok=True)
|
|
testfile.unlink(missing_ok=True)
|
|
shutil.rmtree(testdir)
|
|
elif storage_type=='remote':
|
|
command=['ssh', remote_user_host,'mkdir -p ',str(testdir),'&& touch ',str(testfile),'&& ln ',str(testfile),' ',str(testlink),'&& rm -rf ',str(testdir)]
|
|
ltest=subprocess.run(command, capture_output=True, text=True)
|
|
if ltest.returncode != 0:
|
|
raise Exception(f"The remote directory {backup_root} does not appear to support hard links.")
|
|
except Exception as errmsg:
|
|
print("FAILED")
|
|
print("")
|
|
print(f"{errmsg} \n This is a fatal error and ribs cannot continue. Please check the config file:{config}")
|
|
if storage_type=='offline':
|
|
mount_storage(backup_root,'u')
|
|
failed_confs.append(config)
|
|
continue
|
|
else:
|
|
print("OK")
|
|
|
|
# Generate exclude list
|
|
if exclude_list != None:
|
|
for exclude_pattern in exclude_list:
|
|
print("excluding",exclude_pattern)
|
|
|
|
# Determine today's date and generate current datetime directory name
|
|
datetimedir_format='%Y-%m-%d_%H:%M:%S'
|
|
datedir_format='%Y-%m-%d'
|
|
now=datetime.now()
|
|
current_dirname=now.strftime(datetimedir_format)
|
|
this_dirname=Path(current_dirname)
|
|
|
|
# Find previous backup's directory, if present, to use as link destination
|
|
dirname_pattern='????-??-??*'
|
|
prev_backups = []
|
|
if storage_type=='local' or storage_type=='offline':
|
|
# get local or offline listing
|
|
if backup_subdir:
|
|
backups_location=backup_root.joinpath(backup_subdir)
|
|
else:
|
|
backups_location=backup_root
|
|
matches=backups_location.glob(dirname_pattern)
|
|
matching_dirs=[match for match in matches if match.is_dir()]
|
|
if matching_dirs:
|
|
for dirpath in matching_dirs:
|
|
prev_backups.append(dirpath)
|
|
elif storage_type=='remote':
|
|
# get remote listing
|
|
if backup_subdir:
|
|
backups_location=backup_root.joinpath(backup_subdir)
|
|
else:
|
|
backups_location=backup_root
|
|
command=['ssh',remote_user_host,' cd ',str(backups_location),'&& ls -1d ',dirname_pattern]
|
|
remotelist=subprocess.run(command, capture_output=True, text=True)
|
|
matching_dirs=remotelist.stdout.splitlines()
|
|
if matching_dirs:
|
|
for dirpath in matching_dirs:
|
|
prev_backups.append(dirpath)
|
|
|
|
if len(prev_backups)==0:
|
|
print("No previous backups were found")
|
|
has_previous=False
|
|
else:
|
|
has_previous=True
|
|
prev_backups.sort()
|
|
last_dirname=prev_backups[-1]
|
|
|
|
last_backup=backups_location.joinpath(last_dirname)
|
|
print("The last backup was:",last_backup)
|
|
|
|
this_backup=backups_location.joinpath(this_dirname)
|
|
print("This backup will be:",this_backup)
|
|
|
|
|
|
# Set rsync target and link destination directories
|
|
if storage_type=='local' or storage_type=='offline':
|
|
target_dir=this_backup
|
|
if has_previous:
|
|
link_dir=last_backup
|
|
else:
|
|
link_dir=None
|
|
elif storage_type=='remote':
|
|
target_dir=f"{remote_user_host}:{this_backup}"
|
|
if has_previous:
|
|
link_dir=last_backup
|
|
else:
|
|
link_dir=None
|
|
|
|
|
|
# Generate rsync options
|
|
rsync_options=[]
|
|
#rsync_options.append('--verbose')
|
|
rsync_options.append('--secluded-args')
|
|
rsync_options.append('--archive')
|
|
rsync_options.append('--human-readable')
|
|
rsync_options.append('--delete')
|
|
if has_previous:
|
|
rsync_options.append('--link-dest='+str(link_dir)+'/')
|
|
|
|
# Create backup_subdir if necessary
|
|
if simulate:
|
|
print("Unable to create backup subdirectory since we're just simulating.")
|
|
else:
|
|
if backup_subdir:
|
|
print(f"Creating backup subdirectory {backup_subdir} if necessary.")
|
|
if storage_type=='local' or storage_type=='offline':
|
|
backups_location.mkdir(exist_ok=True)
|
|
elif storage_type=='remote':
|
|
command=['ssh',remote_user_host,' mkdir -p ',str(backups_location)]
|
|
subcre=subprocess.run(command, capture_output=True, text=True)
|
|
|
|
# Run the rsync command
|
|
print("Running rsync... ",end="",flush=True)
|
|
|
|
try:
|
|
if storage_type=='local' or storage_type=='offline':
|
|
run_rsync(str(source_dir), str(this_backup)+'/', options=rsync_options, excludes=exclude_list)
|
|
elif storage_type=='remote':
|
|
run_rsync(str(source_dir), remote_user_host+':'+str(this_backup)+'/', options=rsync_options, excludes=exclude_list)
|
|
except Exception as errmsg:
|
|
print("FAILED")
|
|
print("")
|
|
print(errmsg)
|
|
failed_confs.append(config)
|
|
continue
|
|
else:
|
|
print("OK")
|
|
|
|
# Delete backup directories older than day_limit
|
|
try:
|
|
if simulate:
|
|
print(f"Simulating deleting backups that are more than {day_limit} days old.")
|
|
print(f"Deleting backups that are more than {day_limit} days old.")
|
|
if has_previous:
|
|
would_delete=False
|
|
for old_backup in prev_backups:
|
|
if re.search(r'\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2}',str(old_backup)):
|
|
backup_date=datetime.strptime(os.path.basename(old_backup),datetimedir_format)
|
|
else:
|
|
backup_date=datetime.strptime(os.path.basename(old_backup),datedir_format)
|
|
if now-backup_date>timedelta(days=int(day_limit)):
|
|
would_delete=True
|
|
if simulate:
|
|
print(f"Simulating deleting {os.path.basename(old_backup)}")
|
|
else:
|
|
print(f"Deleting {os.path.basename(old_backup)}")
|
|
if storage_type=='local' or storage_type=='offline':
|
|
shutil.rmtree(backups_location.joinpath(old_backup))
|
|
elif storage_type=='remote':
|
|
command=['ssh', remote_user_host,'rm -rf ',str(backups_location.joinpath(old_backup))]
|
|
delbkup=subprocess.run(command, capture_output=True, text=True)
|
|
|
|
if not would_delete:
|
|
print("No backups are more than",day_limit,"days old, so not deleting.")
|
|
else:
|
|
print("There are no previous backups to delete")
|
|
except Exception as errmsg:
|
|
print("")
|
|
print(errmsg)
|
|
|
|
# Synchronize disks
|
|
if storage_type=='offline':
|
|
print("Synchronizing disks...",end="")
|
|
os.sync()
|
|
print("OK")
|
|
|
|
# If STORAGE_TYPE is OFFLINE, update the capacity file
|
|
if storage_type=='offline':
|
|
update_capacity_file(backup_root)
|
|
|
|
# If STORAGE_TYPE is OFFLINE, unmount, and remount read-only if required
|
|
try:
|
|
if storage_type=='offline':
|
|
if remount_ro:
|
|
print(f"Remounting {backup_root} read-only... ",end="")
|
|
operation='ro'
|
|
else:
|
|
print(f"Unmounting {backup_root}... ",end="")
|
|
operation='u'
|
|
mount_result=mount_storage(backup_root,operation,mountparams=mount_parameters)
|
|
if mount_result != 0:
|
|
print("FAILED")
|
|
raise Exception(f"Failed to unmount/remount {backup_root}.")
|
|
else:
|
|
print("OK")
|
|
except Exception as errmsg:
|
|
print("")
|
|
print(errmsg)
|
|
failed_confs.append(config)
|
|
continue
|
|
|
|
runend=datetime.now()
|
|
timediff=(runend-runstart).total_seconds()
|
|
print(f"Run ended: {runend.ctime()}")
|
|
print(f"Total time: {str(int(timediff //3600))} hours, {str(int((timediff % 3600) // 60))} minutes, {str(int(timediff % 60))} seconds")
|
|
|
|
if len(failed_confs)>0:
|
|
print("Failures occurred when running the following config files:")
|
|
for failed_conf in failed_confs:
|
|
print(f" {failed_conf}")
|
|
print("")
|
|
wrap_message(f"The listed backups may have failed completely or partially. Please check the most recent backup to determine the nature of the failure, and check to make sure the settings in the config file are correct.")
|
|
return 1
|
|
|
|
|
|
def wrap_message(message_text, width=72):
|
|
"""
|
|
Display a long message with the necessary line breaks.
|
|
|
|
Args:
|
|
message_text (str): The text to display.
|
|
width (int, optional): The max number of columns in each line.
|
|
|
|
Returns:
|
|
Nothing
|
|
"""
|
|
linelist=textwrap.wrap(message_text,width=width,break_on_hyphens=False)
|
|
for line in linelist:
|
|
print(line)
|
|
|
|
|
|
|
|
# ****** END function definitions ******
|
|
|
|
# Call main function
|
|
main()
|