1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38 """
39 Spans staged data among multiple discs
40
41 This is the Cedar Backup span tool. It is intended for use by people who stage
42 more data than can fit on a single disc. It allows a user to split staged data
43 among more than one disc. It can't be an extension because it requires user
44 input when switching media.
45
46 Most configuration is taken from the Cedar Backup configuration file,
47 specifically the store section. A few pieces of configuration are taken
48 directly from the user.
49
50 @author: Kenneth J. Pronovici <pronovic@ieee.org>
51 """
52
53
54
55
56
57
58 import sys
59 import os
60 import logging
61 import tempfile
62
63
64 from CedarBackup3.release import AUTHOR, EMAIL, VERSION, DATE, COPYRIGHT
65 from CedarBackup3.util import displayBytes, convertSize, mount, unmount
66 from CedarBackup3.util import UNIT_SECTORS, UNIT_BYTES
67 from CedarBackup3.config import Config
68 from CedarBackup3.filesystem import BackupFileList, compareDigestMaps, normalizeDir
69 from CedarBackup3.cli import Options, setupLogging, setupPathResolver
70 from CedarBackup3.cli import DEFAULT_CONFIG, DEFAULT_LOGFILE, DEFAULT_OWNERSHIP, DEFAULT_MODE
71 from CedarBackup3.actions.constants import STORE_INDICATOR
72 from CedarBackup3.actions.util import createWriter
73 from CedarBackup3.actions.store import writeIndicatorFile
74 from CedarBackup3.actions.util import findDailyDirs
75 from CedarBackup3.util import Diagnostics
76
77
78
79
80
81
82 logger = logging.getLogger("CedarBackup3.log.tools.span")
83
84
85
86
87
88
90
91 """
92 Tool-specific command-line options.
93
94 Most of the cback3 command-line options are exactly what we need here --
95 logfile path, permissions, verbosity, etc. However, we need to make a few
96 tweaks since we don't accept any actions.
97
98 Also, a few extra command line options that we accept are really ignored
99 underneath. I just don't care about that for a tool like this.
100 """
101
103 """
104 Validates command-line options represented by the object.
105 There are no validations here, because we don't use any actions.
106 @raise ValueError: If one of the validations fails.
107 """
108 pass
109
110
111
112
113
114
115
116
117
118
120 """
121 Implements the command-line interface for the C{cback3-span} script.
122
123 Essentially, this is the "main routine" for the cback3-span script. It does
124 all of the argument processing for the script, and then also implements the
125 tool functionality.
126
127 This function looks pretty similiar to C{CedarBackup3.cli.cli()}. It's not
128 easy to refactor this code to make it reusable and also readable, so I've
129 decided to just live with the duplication.
130
131 A different error code is returned for each type of failure:
132
133 - C{1}: The Python interpreter version is < 3.4
134 - C{2}: Error processing command-line arguments
135 - C{3}: Error configuring logging
136 - C{4}: Error parsing indicated configuration file
137 - C{5}: Backup was interrupted with a CTRL-C or similar
138 - C{6}: Error executing other parts of the script
139
140 @note: This script uses print rather than logging to the INFO level, because
141 it is interactive. Underlying Cedar Backup functionality uses the logging
142 mechanism exclusively.
143
144 @return: Error code as described above.
145 """
146 try:
147 if list(map(int, [sys.version_info[0], sys.version_info[1]])) < [3, 4]:
148 sys.stderr.write("Python 3 version 3.4 or greater required.\n")
149 return 1
150 except:
151
152 sys.stderr.write("Python 3 version 3.4 or greater required.\n")
153 return 1
154
155 try:
156 options = SpanOptions(argumentList=sys.argv[1:])
157 except Exception as e:
158 _usage()
159 sys.stderr.write(" *** Error: %s\n" % e)
160 return 2
161
162 if options.help:
163 _usage()
164 return 0
165 if options.version:
166 _version()
167 return 0
168 if options.diagnostics:
169 _diagnostics()
170 return 0
171
172 if options.stacktrace:
173 logfile = setupLogging(options)
174 else:
175 try:
176 logfile = setupLogging(options)
177 except Exception as e:
178 sys.stderr.write("Error setting up logging: %s\n" % e)
179 return 3
180
181 logger.info("Cedar Backup 'span' utility run started.")
182 logger.info("Options were [%s]", options)
183 logger.info("Logfile is [%s]", logfile)
184
185 if options.config is None:
186 logger.debug("Using default configuration file.")
187 configPath = DEFAULT_CONFIG
188 else:
189 logger.debug("Using user-supplied configuration file.")
190 configPath = options.config
191
192 try:
193 logger.info("Configuration path is [%s]", configPath)
194 config = Config(xmlPath=configPath)
195 setupPathResolver(config)
196 except Exception as e:
197 logger.error("Error reading or handling configuration: %s", e)
198 logger.info("Cedar Backup 'span' utility run completed with status 4.")
199 return 4
200
201 if options.stacktrace:
202 _executeAction(options, config)
203 else:
204 try:
205 _executeAction(options, config)
206 except KeyboardInterrupt:
207 logger.error("Backup interrupted.")
208 logger.info("Cedar Backup 'span' utility run completed with status 5.")
209 return 5
210 except Exception as e:
211 logger.error("Error executing backup: %s", e)
212 logger.info("Cedar Backup 'span' utility run completed with status 6.")
213 return 6
214
215 logger.info("Cedar Backup 'span' utility run completed with status 0.")
216 return 0
217
218
219
220
221
222
223
224
225
226
228 """
229 Prints usage information for the cback3-span script.
230 @param fd: File descriptor used to print information.
231 @note: The C{fd} is used rather than C{print} to facilitate unit testing.
232 """
233 fd.write("\n")
234 fd.write(" Usage: cback3-span [switches]\n")
235 fd.write("\n")
236 fd.write(" Cedar Backup 'span' tool.\n")
237 fd.write("\n")
238 fd.write(" This Cedar Backup utility spans staged data between multiple discs.\n")
239 fd.write(" It is a utility, not an extension, and requires user interaction.\n")
240 fd.write("\n")
241 fd.write(" The following switches are accepted, mostly to set up underlying\n")
242 fd.write(" Cedar Backup functionality:\n")
243 fd.write("\n")
244 fd.write(" -h, --help Display this usage/help listing\n")
245 fd.write(" -V, --version Display version information\n")
246 fd.write(" -b, --verbose Print verbose output as well as logging to disk\n")
247 fd.write(" -c, --config Path to config file (default: %s)\n" % DEFAULT_CONFIG)
248 fd.write(" -l, --logfile Path to logfile (default: %s)\n" % DEFAULT_LOGFILE)
249 fd.write(" -o, --owner Logfile ownership, user:group (default: %s:%s)\n" % (DEFAULT_OWNERSHIP[0], DEFAULT_OWNERSHIP[1]))
250 fd.write(" -m, --mode Octal logfile permissions mode (default: %o)\n" % DEFAULT_MODE)
251 fd.write(" -O, --output Record some sub-command (i.e. tar) output to the log\n")
252 fd.write(" -d, --debug Write debugging information to the log (implies --output)\n")
253 fd.write(" -s, --stack Dump a Python stack trace instead of swallowing exceptions\n")
254 fd.write("\n")
255
256
257
258
259
260
262 """
263 Prints version information for the cback3-span script.
264 @param fd: File descriptor used to print information.
265 @note: The C{fd} is used rather than C{print} to facilitate unit testing.
266 """
267 fd.write("\n")
268 fd.write(" Cedar Backup 'span' tool.\n")
269 fd.write(" Included with Cedar Backup version %s, released %s.\n" % (VERSION, DATE))
270 fd.write("\n")
271 fd.write(" Copyright (c) %s %s <%s>.\n" % (COPYRIGHT, AUTHOR, EMAIL))
272 fd.write(" See CREDITS for a list of included code and other contributors.\n")
273 fd.write(" This is free software; there is NO warranty. See the\n")
274 fd.write(" GNU General Public License version 2 for copying conditions.\n")
275 fd.write("\n")
276 fd.write(" Use the --help option for usage information.\n")
277 fd.write("\n")
278
279
280
281
282
283
285 """
286 Prints runtime diagnostics information.
287 @param fd: File descriptor used to print information.
288 @note: The C{fd} is used rather than C{print} to facilitate unit testing.
289 """
290 fd.write("\n")
291 fd.write("Diagnostics:\n")
292 fd.write("\n")
293 Diagnostics().printDiagnostics(fd=fd, prefix=" ")
294 fd.write("\n")
295
296
297
298
299
300
301
303 """
304 Implements the guts of the cback3-span tool.
305
306 @param options: Program command-line options.
307 @type options: SpanOptions object.
308
309 @param config: Program configuration.
310 @type config: Config object.
311
312 @raise Exception: Under many generic error conditions
313 """
314 print("")
315 print("================================================")
316 print(" Cedar Backup 'span' tool")
317 print("================================================")
318 print("")
319 print("This the Cedar Backup span tool. It is used to split up staging")
320 print("data when that staging data does not fit onto a single disc.")
321 print("")
322 print("This utility operates using Cedar Backup configuration. Configuration")
323 print("specifies which staging directory to look at and which writer device")
324 print("and media type to use.")
325 print("")
326 if not _getYesNoAnswer("Continue?", default="Y"):
327 return
328 print("===")
329
330 print("")
331 print("Cedar Backup store configuration looks like this:")
332 print("")
333 print(" Source Directory...: %s" % config.store.sourceDir)
334 print(" Media Type.........: %s" % config.store.mediaType)
335 print(" Device Type........: %s" % config.store.deviceType)
336 print(" Device Path........: %s" % config.store.devicePath)
337 print(" Device SCSI ID.....: %s" % config.store.deviceScsiId)
338 print(" Drive Speed........: %s" % config.store.driveSpeed)
339 print(" Check Data Flag....: %s" % config.store.checkData)
340 print(" No Eject Flag......: %s" % config.store.noEject)
341 print("")
342 if not _getYesNoAnswer("Is this OK?", default="Y"):
343 return
344 print("===")
345
346 (writer, mediaCapacity) = _getWriter(config)
347
348 print("")
349 print("Please wait, indexing the source directory (this may take a while)...")
350 (dailyDirs, fileList) = _findDailyDirs(config.store.sourceDir)
351 print("===")
352
353 print("")
354 print("The following daily staging directories have not yet been written to disc:")
355 print("")
356 for dailyDir in dailyDirs:
357 print(" %s" % dailyDir)
358
359 totalSize = fileList.totalSize()
360 print("")
361 print("The total size of the data in these directories is %s." % displayBytes(totalSize))
362 print("")
363 if not _getYesNoAnswer("Continue?", default="Y"):
364 return
365 print("===")
366
367 print("")
368 print("Based on configuration, the capacity of your media is %s." % displayBytes(mediaCapacity))
369
370 print("")
371 print("Since estimates are not perfect and there is some uncertainly in")
372 print("media capacity calculations, it is good to have a \"cushion\",")
373 print("a percentage of capacity to set aside. The cushion reduces the")
374 print("capacity of your media, so a 1.5% cushion leaves 98.5% remaining.")
375 print("")
376 cushion = _getFloat("What cushion percentage?", default=4.5)
377 print("===")
378
379 realCapacity = ((100.0 - cushion)/100.0) * mediaCapacity
380 minimumDiscs = (totalSize/realCapacity) + 1
381 print("")
382 print("The real capacity, taking into account the %.2f%% cushion, is %s." % (cushion, displayBytes(realCapacity)))
383 print("It will take at least %d disc(s) to store your %s of data." % (minimumDiscs, displayBytes(totalSize)))
384 print("")
385 if not _getYesNoAnswer("Continue?", default="Y"):
386 return
387 print("===")
388
389 happy = False
390 while not happy:
391 print("")
392 print("Which algorithm do you want to use to span your data across")
393 print("multiple discs?")
394 print("")
395 print("The following algorithms are available:")
396 print("")
397 print(" first....: The \"first-fit\" algorithm")
398 print(" best.....: The \"best-fit\" algorithm")
399 print(" worst....: The \"worst-fit\" algorithm")
400 print(" alternate: The \"alternate-fit\" algorithm")
401 print("")
402 print("If you don't like the results you will have a chance to try a")
403 print("different one later.")
404 print("")
405 algorithm = _getChoiceAnswer("Which algorithm?", "worst", [ "first", "best", "worst", "alternate", ])
406 print("===")
407
408 print("")
409 print("Please wait, generating file lists (this may take a while)...")
410 spanSet = fileList.generateSpan(capacity=realCapacity, algorithm="%s_fit" % algorithm)
411 print("===")
412
413 print("")
414 print("Using the \"%s-fit\" algorithm, Cedar Backup can split your data" % algorithm)
415 print("into %d discs." % len(spanSet))
416 print("")
417 counter = 0
418 for item in spanSet:
419 counter += 1
420 print("Disc %d: %d files, %s, %.2f%% utilization" % (counter, len(item.fileList),
421 displayBytes(item.size), item.utilization))
422 print("")
423 if _getYesNoAnswer("Accept this solution?", default="Y"):
424 happy = True
425 print("===")
426
427 counter = 0
428 for spanItem in spanSet:
429 counter += 1
430 if counter == 1:
431 print("")
432 _getReturn("Please place the first disc in your backup device.\nPress return when ready.")
433 print("===")
434 else:
435 print("")
436 _getReturn("Please replace the disc in your backup device.\nPress return when ready.")
437 print("===")
438 _writeDisc(config, writer, spanItem)
439
440 _writeStoreIndicator(config, dailyDirs)
441
442 print("")
443 print("Completed writing all discs.")
444
445
446
447
448
449
451 """
452 Returns a list of all daily staging directories that have not yet been
453 stored.
454
455 The store indicator file C{cback.store} will be written to a daily staging
456 directory once that directory is written to disc. So, this function looks
457 at each daily staging directory within the configured staging directory, and
458 returns a list of those which do not contain the indicator file.
459
460 Returned is a tuple containing two items: a list of daily staging
461 directories, and a BackupFileList containing all files among those staging
462 directories.
463
464 @param stagingDir: Configured staging directory
465
466 @return: Tuple (staging dirs, backup file list)
467 """
468 results = findDailyDirs(stagingDir, STORE_INDICATOR)
469 fileList = BackupFileList()
470 for item in results:
471 fileList.addDirContents(item)
472 return (results, fileList)
473
474
475
476
477
478
490
491
492
493
494
495
506
507
508
509
510
511
525
527 """
528 Initialize an ISO image for a span item.
529 @param config: Cedar Backup configuration
530 @param writer: Writer to use
531 @param spanItem: Span item to write
532 """
533 complete = False
534 while not complete:
535 try:
536 print("Initializing image...")
537 writer.initializeImage(newDisc=True, tmpdir=config.options.workingDir)
538 for path in spanItem.fileList:
539 graftPoint = os.path.dirname(path.replace(config.store.sourceDir, "", 1))
540 writer.addImageEntry(path, graftPoint)
541 complete = True
542 except KeyboardInterrupt as e:
543 raise e
544 except Exception as e:
545 logger.error("Failed to initialize image: %s", e)
546 if not _getYesNoAnswer("Retry initialization step?", default="Y"):
547 raise e
548 print("Ok, attempting retry.")
549 print("===")
550 print("Completed initializing image.")
551
552
554 """
555 Writes a ISO image for a span item.
556 @param config: Cedar Backup configuration
557 @param writer: Writer to use
558 """
559 complete = False
560 while not complete:
561 try:
562 print("Writing image to disc...")
563 writer.writeImage()
564 complete = True
565 except KeyboardInterrupt as e:
566 raise e
567 except Exception as e:
568 logger.error("Failed to write image: %s", e)
569 if not _getYesNoAnswer("Retry this step?", default="Y"):
570 raise e
571 print("Ok, attempting retry.")
572 _getReturn("Please replace media if needed.\nPress return when ready.")
573 print("===")
574 print("Completed writing image.")
575
577 """
578 Run a consistency check on an ISO image for a span item.
579 @param config: Cedar Backup configuration
580 @param writer: Writer to use
581 @param spanItem: Span item to write
582 """
583 if config.store.checkData:
584 complete = False
585 while not complete:
586 try:
587 print("Running consistency check...")
588 _consistencyCheck(config, spanItem.fileList)
589 complete = True
590 except KeyboardInterrupt as e:
591 raise e
592 except Exception as e:
593 logger.error("Consistency check failed: %s", e)
594 if not _getYesNoAnswer("Retry the consistency check?", default="Y"):
595 raise e
596 if _getYesNoAnswer("Rewrite the disc first?", default="N"):
597 print("Ok, attempting retry.")
598 _getReturn("Please replace the disc in your backup device.\nPress return when ready.")
599 print("===")
600 _discWriteImage(config, writer)
601 else:
602 print("Ok, attempting retry.")
603 print("===")
604 print("Completed consistency check.")
605
606
607
608
609
610
612 """
613 Runs a consistency check against media in the backup device.
614
615 The function mounts the device at a temporary mount point in the working
616 directory, and then compares the passed-in file list's digest map with the
617 one generated from the disc. The two lists should be identical.
618
619 If no exceptions are thrown, there were no problems with the consistency
620 check.
621
622 @warning: The implementation of this function is very UNIX-specific.
623
624 @param config: Config object.
625 @param fileList: BackupFileList whose contents to check against
626
627 @raise ValueError: If the check fails
628 @raise IOError: If there is a problem working with the media.
629 """
630 logger.debug("Running consistency check.")
631 mountPoint = tempfile.mkdtemp(dir=config.options.workingDir)
632 try:
633 mount(config.store.devicePath, mountPoint, "iso9660")
634 discList = BackupFileList()
635 discList.addDirContents(mountPoint)
636 sourceList = BackupFileList()
637 sourceList.extend(fileList)
638 discListDigest = discList.generateDigestMap(stripPrefix=normalizeDir(mountPoint))
639 sourceListDigest = sourceList.generateDigestMap(stripPrefix=normalizeDir(config.store.sourceDir))
640 compareDigestMaps(sourceListDigest, discListDigest, verbose=True)
641 logger.info("Consistency check completed. No problems found.")
642 finally:
643 unmount(mountPoint, True, 5, 1)
644
645
646
647
648
649
651 """
652 Get a yes/no answer from the user.
653 The default will be placed at the end of the prompt.
654 A "Y" or "y" is considered yes, anything else no.
655 A blank (empty) response results in the default.
656 @param prompt: Prompt to show.
657 @param default: Default to set if the result is blank
658 @return: Boolean true/false corresponding to Y/N
659 """
660 if default == "Y":
661 prompt = "%s [Y/n]: " % prompt
662 else:
663 prompt = "%s [y/N]: " % prompt
664 answer = input(prompt)
665 if answer in [ None, "", ]:
666 answer = default
667 if answer[0] in [ "Y", "y", ]:
668 return True
669 else:
670 return False
671
673 """
674 Get a particular choice from the user.
675 The default will be placed at the end of the prompt.
676 The function loops until getting a valid choice.
677 A blank (empty) response results in the default.
678 @param prompt: Prompt to show.
679 @param default: Default to set if the result is None or blank.
680 @param validChoices: List of valid choices (strings)
681 @return: Valid choice from user.
682 """
683 prompt = "%s [%s]: " % (prompt, default)
684 answer = input(prompt)
685 if answer in [ None, "", ]:
686 answer = default
687 while answer not in validChoices:
688 print("Choice must be one of %s" % validChoices)
689 answer = input(prompt)
690 return answer
691
693 """
694 Get a floating point number from the user.
695 The default will be placed at the end of the prompt.
696 The function loops until getting a valid floating point number.
697 A blank (empty) response results in the default.
698 @param prompt: Prompt to show.
699 @param default: Default to set if the result is None or blank.
700 @return: Floating point number from user
701 """
702 prompt = "%s [%.2f]: " % (prompt, default)
703 while True:
704 answer = input(prompt)
705 if answer in [ None, "" ]:
706 return default
707 else:
708 try:
709 return float(answer)
710 except ValueError:
711 print("Enter a floating point number.")
712
714 """
715 Get a return key from the user.
716 @param prompt: Prompt to show.
717 """
718 input(prompt)
719
720
721
722
723
724
725 if __name__ == "__main__":
726 sys.exit(cli())
727