Python Backup Script

Part one, take the first full backup. At the present time, this code is still under development and should not be used on a production machine. However, I am posting it here for reference.

Eventually, this code is going to be included in a backup client I am developing that will interface with glusterfs and Amazon S3 storage.

Currently, this code is tested to run on Python v2.7.4 on a Fedora 18 machine. With all three python files, and any number of properly defined job xml files in the jobs.d/ directory, these scripts are currently functional.

seed_files.py

This is the controller file for taking the first backup.

#!/usr/bin/python

# Create first full backup

import os, stat, time, seed_functions

def printHello():
  print "hello";

#--- This part of the script does the heavy lifting.
def seedMain(myFindPath,myJobId,myFullTempPath,myTargetMetaPath,myTargetTarFilePath,myExcludeFiles):
  #---execute the first backup for the job.
  seed_functions.findFiles(myFullTempPath,myFindPath,myExcludeFiles);
  seed_functions.storeMetaData(myFullTempPath,myTargetMetaPath);
  seed_functions.mkTarFile(myFullTempPath,myTargetTarFilePath);
  print "Job completed successfully";

#--- If this is being run as a script, set temporary variables
if __name__ == '__main__':
  myFindPath = '/home/myuser/findfiles';
  myJobId = str('106');
  myTempPath = '/tmp/';
  myTempFileList = 'files.tmp';
  myFullTempPath = os.path.join(myTempPath,myTempFileList); 
  myTargetPath = '/home/target1/';
  myTargetMeta = 'job'+myJobId+'.meta';
  myTargetTarFile = 'job'+myJobId+'.tar';
  myTargetMetaPath = myTargetPath+myTargetMeta;
  myTargetTarFilePath = myTargetPath+myTargetTarFile;

  #---Path names below should be absolute path names (start with / ) and should not end with '/'
  myExlcudeFiles = list();
  myExlcudeFiles.append('/home/myuser/findfiles/dontbackup');
  myExlcudeFiles.append('/home/myuser/findfiles/somebigfiles');
  myExlcudeFiles.append('*.adf');
  myExlcudeFiles.append('badfilez*');
  
  seedMain(myFindPath,myJobId,myFullTempPath,myTargetMetaPath,myTargetTarFilePath,myExcludeFiles);

#myJdate = seed_functions.getJulianDate();

seed_functions.py
This is the file that does the actual real work

#!/usr/bin/python

#Shared Functions and Classes
import os, sys, stat, time, glob
from datetime import datetime

def getJulianDate():
#--- This function returns an interger value of today's Julian Date 
#--- preceded by two digit year.  IE: 01JAN2013 -> 13001
  nowtime =  str(datetime.now());
  (year, month, day) = nowtime.split('-');
  day = int(day[:2]);
  month = int(month);
  year = int(year);
  t = time.mktime((year, month, day, 0, 0, 0, 0, 0, 0));
  jdate = (year % 2000) * 1000;
  jdate = jdate + time.gmtime(t)[7];
  return jdate;


import re
def findFiles(filepath,myFindPath,excludeFiles):
#--- Variables: fileListPath = string, myFindPath = string, excludeFiles = list() of strings)
#--- This function writes/overwrites the file @ 'fileListPath' which should be absolute path.
#--- The file @ fileListPath is a list of files found within myFindPath.
#--- myFindPath is desgined to be an absolute directory path.
#--- excludeFiles is a list() of absolute paths which should be excluded during the finding operation.
  with open(filepath, 'w') as ftemp:
    #--- Generate list of files that shouldn't be added
    #removeExcludedFiles();
    for dirname, dirnames, filenames in os.walk(myFindPath):
      #--- Remove exlcuded directories
      for badfile in dirnames:
        if os.path.join(myFindPath,dirname,badfile) in excludeFiles:
          dirnames.remove(badfile);

      #--- Gather the other directories
      for subdirname in dirnames:
	  #--- we just want to add empty directories
	  if os.path.islink(os.path.join(dirname, subdirname)) or not os.listdir(os.path.join(dirname, subdirname)):
	    ftemp.write(os.path.join(dirname, subdirname)+'\n');

      #--- Add files to the list.
      for filename in filenames:
	  #--- check to see if the file is a regular file or a link:
	  if os.path.islink(os.path.join(dirname, filename)) or os.path.isfile(os.path.join(dirname, filename)):
            #if not filename in myWildList:
	      ftemp.write(os.path.join(dirname, filename)+'\n');

  ftemp.closed;	

def storeMetaData(fileListPath, fileMetaPath):
#---This function creates an output file with
#---Filename and absolute path of fileMetaPath
#---File format is:
#---  /path/to/file ::: modified datetime ::: seconds since 1970 ::: md5 hash
  with open(fileMetaPath, 'w') as fmeta:
    myFileList = open(fileListPath, 'r');
    for filez in myFileList:
      #--- create an array to append information.
      myFileMeta = list();
      #--- Append absolute path and file name as first item
      myFileMeta.append(os.path.abspath(filez.strip()));
      #--- Get file meta information from os.stat()
      myStat = os.stat(filez.strip());
      #---Append human readable date/time stamp.
      myFileMeta.append(time.ctime(myStat.st_mtime));
      #---Append unix timestamp for easy comparison in the future.
      myFileMeta.append(myStat.st_mtime);
      if not os.path.isdir(filez.strip()):
        myHash = md5(filez.strip());
      else:
        myHash = "---None: directory";
      myFileMeta.append(myHash);
      metaDataString = str(myFileMeta[0]+":::"+myFileMeta[1]+":::"+str(myFileMeta[2])+":::"+str(myFileMeta[3]));
      fmeta.write(metaDataString+'\n');
    myFileList.closed;
  fmeta.closed;

import hashlib,os
def md5(filename):
    ''' function to get md5 of file '''
    d = hashlib.md5();
    try:
        d.update(open(filename).read());
    except Exception,e:
        print e;
    else:
        return d.hexdigest();

import tarfile
def mkTarFile(fileList, tarOutPath):
    thisTarOut = tarOutPath+".lzo"
    thisFileList = "-T "+fileList
    os.system("tar {options} {tarfile} {filex} &> /dev/null".format(options="cpvfa", tarfile=thisTarOut, filex=thisFileList));

#--------
#--- Not yet impemented functions below:
#--------

def findFiles2(fileListPath,myFindPath,excludeFiles):
#--- This function is for testing purposes only
  with open(fileListPath, 'w') as ftemp:
   for dirname, dirnames, filenames in os.walk(myFindPath):
    for subdirname in dirnames:
	#--- we just want to add empty directories
	if not os.listdir(os.path.join(dirname, subdirname)):
		ftemp.write(os.path.join(dirname, subdirname)+'\n');
    for filename in filenames:
	#--- check to see if the file is a regular file or a link:
	if os.path.islink(os.path.join(dirname, filename)) or os.path.isfile(os.path.join(dirname, filename)):
		ftemp.write(os.path.join(dirname, filename)+'\n');
  ftemp.closed;

def removeExcludedFiles():
    #--- Not implemented yet.
    myWildList = list();
    wildMatch = re.compile("^\*");
    wildMatch2 = re.compile(".*\*");
    print "Excluding the following"
    for badfile in excludeFiles:
      print badfile;
      result = wildMatch.match(badfile);
      if not result:
        result2 = wildMatch2.match(badfile);
        print result2;
      if result or result2:
        myWildList.append(badfile);
    print myWildList;

start_seeds.py
This is near completion; it parses the jobs in jobs.d/, verifies them, and runs them.

#!/usr/bin/python
import os, re
import xml.etree.ElementTree as ET
import seed_files



def readConfFile():
  #--- Future feature to read a specified jobs.d from config file.
  jobdir = 'jobs.d';
  return jobdir;

def findJobs(jobdir):
  myJobList = list();
  confMatch = re.compile(".*\.xml$");
  for dirname, dirnames, filenames in os.walk(jobdir):
    for jobid in filenames:
      result = confMatch.match(jobid);
      if result:
        myJobList.append(os.path.join(jobdir,jobid));
  return myJobList;

def checkPath(pathText):
  confMatch = re.compile("^\/");
  result = confMatch.match(pathText);
  confMatch2 = re.compile("^\/$");
  result2 = confMatch2.match(pathText);
  if result2:
    exit('Path cannot be / ');
  if not result:
    exit('Directory path must be absolute path: '+pathText);
  if os.path.isdir(pathText) or os.path.ismount(pathText):
    print 'Path seems valid: ',pathText;
  else:
    exit('Invalid path: '+pathText);

def parseJobs(myFoundJobs):
  print "Found the following config files: ",myFoundJobs;
  print "------------------------------------------------";
  myJobList = list();
  for myJob in myFoundJobs:
    mySubList = list();
    print "Parsing and testing: ",myJob;
    tree = ET.parse(myJob);
    root = tree.getroot();

    for child in root:
      #--- Validate backup path is valide.
      #----future feature: master excludes in config file
      if child.tag == 'backupdir':
        print "Checking Backup Directory";
        checkPath(child.text);
      if child.tag == 'backuptarget':
        print "Checking Backup Target"
        checkPath(child.text);

      #--- Create another sublist for excluded directories.
      if child.tag == 'exclude':
        myExcludeList = list();
        for subchild in child:
	  myExcludeList.append(subchild.text);
        mySubList.append(myExcludeList);

      #--- Since it's not a sublist, we append directly
      elif not child.tag == 'exclude':
        mySubList.append(child.text);
    print "------------------------------------------------";
    myJobList.append(mySubList);

  #--- Ensure some jobs were actually found.
  if myJobList.__len__() == 0:
    exit('Exit on Error: No Jobs Found!');
  #--- If we didn't exit above, we returned the parsed job list
  return myJobList;

def performBackup(myJobList):
  for job in myJobList:
    #--- job[0]:  JobID
    #--- job[1];  Backup Path
    #--- job[2]:  Backup Target
    #--- job[3]:  temp directory
    #--- job[4]:  Excluded directories
    myJobId = job[0];
    myFindPath = job[1];
    myTarget = job[2];
    myTempPath = job[3];
    myExcludes = job[4];
    myTargetPath = os.path.join(myTarget,str('job'+myJobId),'master');
    myTempFileList = 'backup_job'+myJobId+'.tmp';
    myFullTempPath = os.path.join(myTargetPath,myTempFileList); 
    myTargetMeta = 'job'+myJobId+'_master.meta';
    myTargetTarFile = 'job'+myJobId+'_master_seed.tar';
    myTargetMetaPath = os.path.join(myTargetPath,myTargetMeta);
    myTargetTarFilePath = os.path.join(myTargetPath,myTargetTarFile);

    if os.path.exists(os.path.join(myTarget,str('job'+myJobId))):
      print os.path.join(myFindPath,str('job'+myJobId));
      exit('Critical Error on JobID: '+myJobId+'\n This job directory already exists!  Exiting to preserve data!');
    else:
      os.makedirs(os.path.join(myTarget,str('job'+myJobId)));
    if os.path.exists(myTargetPath):
      exit('Critical Error on JobID: '+myJobId+'\n This job directory already exists!  Exiting to preserve data!');
    else:
      os.makedirs(myTargetPath);

    myExcludeFiles = list();
    for excludes in myExcludes:
      myExcludeFiles.append(excludes);
    myExcludeFiles.append(myTarget);
    print "------------------------------------------------";
    print "Starting Job: ",myJobId;
    seed_files.seedMain(myFindPath,myJobId,myFullTempPath,myTargetMetaPath,myTargetTarFilePath,myExcludeFiles);



#--- Execute the script.
myJobList = list();
jobdir = readConfFile();
myFoundJobs = findJobs(jobdir);
myJobList = parseJobs(myFoundJobs);
print "Number of jobs: ",myJobList.__len__();
#promptContinue() #--- Let user review backup jobs, prompt for continue.
performBackup(myJobList);

job103.xml
Job file in jobs.d/ directory

<?xml version="1.0"?>
<data>
  <jobid>103</jobid>
  <backupdir>/home/myuser/files</backupdir>
  <backuptarget>/offsitenfs/client1/target2</backuptarget>
  <temppath>/tmp</temppath>
  <exclude>
    <directory>/home/myuser/files/badfolder1</directory>
    <directory>/home/myuser/files/music/badfolder2</directory>
  </exclude>
</data>