1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
|
import fcntl
import logging
import os
import os.path
import subprocess
import datetime
import statistics
class StepResult:
success = None
output = None
def __init__(self, success = True, output = ''):
self.success = success
self.output = output
class BackupResult:
target_results = None
par2_result = None
def __init__(self):
self.target_results = {}
self.par2_result = StepResult()
class BackupStats:
def __init__(self, fname):
def parse_line(line):
date, size = line.strip().split()
return datetime.datetime.fromisoformat(date), float(size)
with open(fname, 'r') as f:
self.entries = dict(map(parse_line, f))
self.dist = statistics.NormalDist.from_samples(self.entries.values())
class Repo:
"""
A single Bup repository into which the data will be backed up, plus a
separate directory for extra runtime data.
:param str bup_dir: path to the bup repository, defaults to BUP_DIR or ~/.bup
:param str data_dir: path to the directory for storing the runtime data,
defaults to ~/.local/var/lbup
"""
bup_dir = None
data_dir = None
lock_name = 'lock'
sizestat_name = 'size'
_logger = None
def __init__(self, bup_dir = None, data_dir = None, logger = None):
if bup_dir is None:
if 'BUP_DIR' in os.environ:
bup_dir = os.environ['BUP_DIR']
else:
bup_dir = os.path.expanduser('~/.bup')
if data_dir is None:
data_dir = os.path.expanduser('~/.local/var/lbup/')
if logger is None:
self._logger = logging.getLogger('%s.%s' % (self.__class__.__name__, bup_dir))
else:
self._logger = logger
# create the data dir, if it does not already exist
os.makedirs(data_dir, 0o700, exist_ok = True)
self.bup_dir = bup_dir
self.data_dir = data_dir
def _bup_repo_size(self):
ret = subprocess.run(['du', '--summarize', '--bytes', '--dereference-args',
'--exclude=midx*', '--exclude=bup.bloom', self.bup_dir],
capture_output = True, check = True)
return int(ret.stdout.split()[0].strip())
def backup(self, tgts, *, gen_par2 = True, dry_run = False):
"""
Backup the supplied targets.
:param list of Target tgts: List of targets to back up.
:param bool gen_par2: Whether to generate par2 recovery information
after the backup concludes'
"""
with open(os.path.join(self.data_dir, self.lock_name), 'w') as lockfile:
result = BackupResult()
self._logger.debug('Acquiring repository lock')
fcntl.lockf(lockfile, fcntl.LOCK_EX)
try:
for tgt in tgts:
# make sure the per-target data dir exists
data_dir = os.path.join(self.data_dir, tgt.name)
os.makedirs(data_dir, 0o700, exist_ok = True)
# measure the pre-backup size
size_pre = self._bup_repo_size()
self._logger.info('Backing up %s...' % tgt.name)
try:
res = tgt.save(dry_run)
except Exception as e:
self._logger.exception('Exception backing up %s: %s' % (tgt.name, str(e)))
res = StepResult(False, str(e).encode('utf-8'))
else:
self._logger.info('Backing up %s done' % tgt.name)
# measure the post-backup size
size_post = self._bup_repo_size()
backup_time = datetime.datetime.now(datetime.timezone.utc)
with open(os.path.join(data_dir, self.sizestat_name), 'a') as f:
f.write('%s %d\n' % (backup_time.isoformat(timespec = 'seconds'), size_post - size_pre))
result.target_results[tgt.name] = res
if gen_par2:
self._logger.info('Generating par2 files...')
res = subprocess.run(['bup', 'fsck', '-g'],
capture_output = True)
self._logger.info('Generating par2 files done')
result.par2_result = StepResult(res.returncode == 0,
res.stderr + res.stdout)
finally:
self._logger.debug('Releasing repository lock')
fcntl.lockf(lockfile, fcntl.LOCK_UN)
self._logger.debug('Backup finished')
return result
def read_stats(self, tgt_name):
filename = os.path.join(self.data_dir, tgt_name, self.sizestat_name)
return BackupStats(filename)
|
