1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38 """
39 Implements the standard 'store' action.
40 @sort: executeStore, writeImage, writeStoreIndicator, consistencyCheck
41 @author: Kenneth J. Pronovici <pronovic@ieee.org>
42 @author: Dmitry Rutsky <rutsky@inbox.ru>
43 """
44
45
46
47
48
49
50
51 import sys
52 import os
53 import logging
54 import datetime
55 import tempfile
56
57
58 from CedarBackup2.filesystem import compareContents
59 from CedarBackup2.util import isStartOfWeek
60 from CedarBackup2.util import mount, unmount, displayBytes
61 from CedarBackup2.actions.util import createWriter, checkMediaState, buildMediaLabel, writeIndicatorFile
62 from CedarBackup2.actions.constants import DIR_TIME_FORMAT, STAGE_INDICATOR, STORE_INDICATOR
63
64
65
66
67
68
69 logger = logging.getLogger("CedarBackup2.log.actions.store")
70
71
72
73
74
75
76
77
78
79
80
82 """
83 Executes the store backup action.
84
85 @note: The rebuild action and the store action are very similar. The
86 main difference is that while store only stores a single day's staging
87 directory, the rebuild action operates on multiple staging directories.
88
89 @note: When the store action is complete, we will write a store indicator to
90 the daily staging directory we used, so it's obvious that the store action
91 has completed.
92
93 @param configPath: Path to configuration file on disk.
94 @type configPath: String representing a path on disk.
95
96 @param options: Program command-line options.
97 @type options: Options object.
98
99 @param config: Program configuration.
100 @type config: Config object.
101
102 @raise ValueError: Under many generic error conditions
103 @raise IOError: If there are problems reading or writing files.
104 """
105 logger.debug("Executing the 'store' action.")
106 if sys.platform == "darwin":
107 logger.warn("Warning: the store action is not fully supported on Mac OS X.")
108 logger.warn("See the Cedar Backup software manual for further information.")
109 if config.options is None or config.store is None:
110 raise ValueError("Store configuration is not properly filled in.")
111 if config.store.checkMedia:
112 checkMediaState(config.store)
113 rebuildMedia = options.full
114 logger.debug("Rebuild media flag [%s]", rebuildMedia)
115 todayIsStart = isStartOfWeek(config.options.startingDay)
116 stagingDirs = _findCorrectDailyDir(options, config)
117 writeImageBlankSafe(config, rebuildMedia, todayIsStart, config.store.blankBehavior, stagingDirs)
118 if config.store.checkData:
119 if sys.platform == "darwin":
120 logger.warn("Warning: consistency check cannot be run successfully on Mac OS X.")
121 logger.warn("See the Cedar Backup software manual for further information.")
122 else:
123 logger.debug("Running consistency check of media.")
124 consistencyCheck(config, stagingDirs)
125 writeStoreIndicator(config, stagingDirs)
126 logger.info("Executed the 'store' action successfully.")
127
128
129
130
131
132
134 """
135 Builds and writes an ISO image containing the indicated stage directories.
136
137 The generated image will contain each of the staging directories listed in
138 C{stagingDirs}. The directories will be placed into the image at the root by
139 date, so staging directory C{/opt/stage/2005/02/10} will be placed into the
140 disc at C{/2005/02/10}.
141
142 @note: This function is implemented in terms of L{writeImageBlankSafe}. The
143 C{newDisc} flag is passed in for both C{rebuildMedia} and C{todayIsStart}.
144
145 @param config: Config object.
146 @param newDisc: Indicates whether the disc should be re-initialized
147 @param stagingDirs: Dictionary mapping directory path to date suffix.
148
149 @raise ValueError: Under many generic error conditions
150 @raise IOError: If there is a problem writing the image to disc.
151 """
152 writeImageBlankSafe(config, newDisc, newDisc, None, stagingDirs)
153
154
155
156
157
158
160 """
161 Builds and writes an ISO image containing the indicated stage directories.
162
163 The generated image will contain each of the staging directories listed in
164 C{stagingDirs}. The directories will be placed into the image at the root by
165 date, so staging directory C{/opt/stage/2005/02/10} will be placed into the
166 disc at C{/2005/02/10}. The media will always be written with a media
167 label specific to Cedar Backup.
168
169 This function is similar to L{writeImage}, but tries to implement a smarter
170 blanking strategy.
171
172 First, the media is always blanked if the C{rebuildMedia} flag is true.
173 Then, if C{rebuildMedia} is false, blanking behavior and C{todayIsStart}
174 come into effect::
175
176 If no blanking behavior is specified, and it is the start of the week,
177 the disc will be blanked
178
179 If blanking behavior is specified, and either the blank mode is "daily"
180 or the blank mode is "weekly" and it is the start of the week, then
181 the disc will be blanked if it looks like the weekly backup will not
182 fit onto the media.
183
184 Otherwise, the disc will not be blanked
185
186 How do we decide whether the weekly backup will fit onto the media? That is
187 what the blanking factor is used for. The following formula is used::
188
189 will backup fit? = (bytes available / (1 + bytes required) <= blankFactor
190
191 The blanking factor will vary from setup to setup, and will probably
192 require some experimentation to get it right.
193
194 @param config: Config object.
195 @param rebuildMedia: Indicates whether media should be rebuilt
196 @param todayIsStart: Indicates whether today is the starting day of the week
197 @param blankBehavior: Blank behavior from configuration, or C{None} to use default behavior
198 @param stagingDirs: Dictionary mapping directory path to date suffix.
199
200 @raise ValueError: Under many generic error conditions
201 @raise IOError: If there is a problem writing the image to disc.
202 """
203 mediaLabel = buildMediaLabel()
204 writer = createWriter(config)
205 writer.initializeImage(True, config.options.workingDir, mediaLabel)
206 for stageDir in stagingDirs.keys():
207 logger.debug("Adding stage directory [%s].", stageDir)
208 dateSuffix = stagingDirs[stageDir]
209 writer.addImageEntry(stageDir, dateSuffix)
210 newDisc = _getNewDisc(writer, rebuildMedia, todayIsStart, blankBehavior)
211 writer.setImageNewDisc(newDisc)
212 writer.writeImage()
213
214 -def _getNewDisc(writer, rebuildMedia, todayIsStart, blankBehavior):
215 """
216 Gets a value for the newDisc flag based on blanking factor rules.
217
218 The blanking factor rules are described above by L{writeImageBlankSafe}.
219
220 @param writer: Previously configured image writer containing image entries
221 @param rebuildMedia: Indicates whether media should be rebuilt
222 @param todayIsStart: Indicates whether today is the starting day of the week
223 @param blankBehavior: Blank behavior from configuration, or C{None} to use default behavior
224
225 @return: newDisc flag to be set on writer.
226 """
227 newDisc = False
228 if rebuildMedia:
229 newDisc = True
230 logger.debug("Setting new disc flag based on rebuildMedia flag.")
231 else:
232 if blankBehavior is None:
233 logger.debug("Default media blanking behavior is in effect.")
234 if todayIsStart:
235 newDisc = True
236 logger.debug("Setting new disc flag based on todayIsStart.")
237 else:
238
239 logger.debug("Optimized media blanking behavior is in effect based on configuration.")
240 if blankBehavior.blankMode == "daily" or (blankBehavior.blankMode == "weekly" and todayIsStart):
241 logger.debug("New disc flag will be set based on blank factor calculation.")
242 blankFactor = float(blankBehavior.blankFactor)
243 logger.debug("Configured blanking factor: %.2f", blankFactor)
244 available = writer.retrieveCapacity().bytesAvailable
245 logger.debug("Bytes available: %s", displayBytes(available))
246 required = writer.getEstimatedImageSize()
247 logger.debug("Bytes required: %s", displayBytes(required))
248 ratio = available / (1.0 + required)
249 logger.debug("Calculated ratio: %.2f", ratio)
250 newDisc = (ratio <= blankFactor)
251 logger.debug("%.2f <= %.2f ? %s", ratio, blankFactor, newDisc)
252 else:
253 logger.debug("No blank factor calculation is required based on configuration.")
254 logger.debug("New disc flag [%s].", newDisc)
255 return newDisc
256
257
258
259
260
261
263 """
264 Writes a store indicator file into staging directories.
265
266 The store indicator is written into each of the staging directories when
267 either a store or rebuild action has written the staging directory to disc.
268
269 @param config: Config object.
270 @param stagingDirs: Dictionary mapping directory path to date suffix.
271 """
272 for stagingDir in stagingDirs.keys():
273 writeIndicatorFile(stagingDir, STORE_INDICATOR,
274 config.options.backupUser,
275 config.options.backupGroup)
276
277
278
279
280
281
283 """
284 Runs a consistency check against media in the backup device.
285
286 It seems that sometimes, it's possible to create a corrupted multisession
287 disc (i.e. one that cannot be read) although no errors were encountered
288 while writing the disc. This consistency check makes sure that the data
289 read from disc matches the data that was used to create the disc.
290
291 The function mounts the device at a temporary mount point in the working
292 directory, and then compares the indicated staging directories in the
293 staging directory and on the media. The comparison is done via
294 functionality in C{filesystem.py}.
295
296 If no exceptions are thrown, there were no problems with the consistency
297 check. A positive confirmation of "no problems" is also written to the log
298 with C{info} priority.
299
300 @warning: The implementation of this function is very UNIX-specific.
301
302 @param config: Config object.
303 @param stagingDirs: Dictionary mapping directory path to date suffix.
304
305 @raise ValueError: If the two directories are not equivalent.
306 @raise IOError: If there is a problem working with the media.
307 """
308 logger.debug("Running consistency check.")
309 mountPoint = tempfile.mkdtemp(dir=config.options.workingDir)
310 try:
311 mount(config.store.devicePath, mountPoint, "iso9660")
312 for stagingDir in stagingDirs.keys():
313 discDir = os.path.join(mountPoint, stagingDirs[stagingDir])
314 logger.debug("Checking [%s] vs. [%s].", stagingDir, discDir)
315 compareContents(stagingDir, discDir, verbose=True)
316 logger.info("Consistency check completed for [%s]. No problems found.", stagingDir)
317 finally:
318 unmount(mountPoint, True, 5, 1)
319
320
321
322
323
324
325
326
327
328
330 """
331 Finds the correct daily staging directory to be written to disk.
332
333 In Cedar Backup v1.0, we assumed that the correct staging directory matched
334 the current date. However, that has problems. In particular, it breaks
335 down if collect is on one side of midnite and stage is on the other, or if
336 certain processes span midnite.
337
338 For v2.0, I'm trying to be smarter. I'll first check the current day. If
339 that directory is found, it's good enough. If it's not found, I'll look for
340 a valid directory from the day before or day after I{which has not yet been
341 staged, according to the stage indicator file}. The first one I find, I'll
342 use. If I use a directory other than for the current day I{and}
343 C{config.store.warnMidnite} is set, a warning will be put in the log.
344
345 There is one exception to this rule. If the C{options.full} flag is set,
346 then the special "span midnite" logic will be disabled and any existing
347 store indicator will be ignored. I did this because I think that most users
348 who run C{cback --full store} twice in a row expect the command to generate
349 two identical discs. With the other rule in place, running that command
350 twice in a row could result in an error ("no unstored directory exists") or
351 could even cause a completely unexpected directory to be written to disc (if
352 some previous day's contents had not yet been written).
353
354 @note: This code is probably longer and more verbose than it needs to be,
355 but at least it's straightforward.
356
357 @param options: Options object.
358 @param config: Config object.
359
360 @return: Correct staging dir, as a dict mapping directory to date suffix.
361 @raise IOError: If the staging directory cannot be found.
362 """
363 oneDay = datetime.timedelta(days=1)
364 today = datetime.date.today()
365 yesterday = today - oneDay
366 tomorrow = today + oneDay
367 todayDate = today.strftime(DIR_TIME_FORMAT)
368 yesterdayDate = yesterday.strftime(DIR_TIME_FORMAT)
369 tomorrowDate = tomorrow.strftime(DIR_TIME_FORMAT)
370 todayPath = os.path.join(config.stage.targetDir, todayDate)
371 yesterdayPath = os.path.join(config.stage.targetDir, yesterdayDate)
372 tomorrowPath = os.path.join(config.stage.targetDir, tomorrowDate)
373 todayStageInd = os.path.join(todayPath, STAGE_INDICATOR)
374 yesterdayStageInd = os.path.join(yesterdayPath, STAGE_INDICATOR)
375 tomorrowStageInd = os.path.join(tomorrowPath, STAGE_INDICATOR)
376 todayStoreInd = os.path.join(todayPath, STORE_INDICATOR)
377 yesterdayStoreInd = os.path.join(yesterdayPath, STORE_INDICATOR)
378 tomorrowStoreInd = os.path.join(tomorrowPath, STORE_INDICATOR)
379 if options.full:
380 if os.path.isdir(todayPath) and os.path.exists(todayStageInd):
381 logger.info("Store process will use current day's stage directory [%s]", todayPath)
382 return { todayPath:todayDate }
383 raise IOError("Unable to find staging directory to store (only tried today due to full option).")
384 else:
385 if os.path.isdir(todayPath) and os.path.exists(todayStageInd) and not os.path.exists(todayStoreInd):
386 logger.info("Store process will use current day's stage directory [%s]", todayPath)
387 return { todayPath:todayDate }
388 elif os.path.isdir(yesterdayPath) and os.path.exists(yesterdayStageInd) and not os.path.exists(yesterdayStoreInd):
389 logger.info("Store process will use previous day's stage directory [%s]", yesterdayPath)
390 if config.store.warnMidnite:
391 logger.warn("Warning: store process crossed midnite boundary to find data.")
392 return { yesterdayPath:yesterdayDate }
393 elif os.path.isdir(tomorrowPath) and os.path.exists(tomorrowStageInd) and not os.path.exists(tomorrowStoreInd):
394 logger.info("Store process will use next day's stage directory [%s]", tomorrowPath)
395 if config.store.warnMidnite:
396 logger.warn("Warning: store process crossed midnite boundary to find data.")
397 return { tomorrowPath:tomorrowDate }
398 raise IOError("Unable to find unused staging directory to store (tried today, yesterday, tomorrow).")
399