#!/bin/ksh -p # For more detailed documentation, see zrep.txt or zrep.overview.txt ZREP_VERSION=1.0.1 ######## start of included files from zrep_top here ########### zrep_vars # This should basically be included as common vars before all zrep stuff. # It contains all 'constant' definitions, as well as a few crucial # shared routines, such as lock handling ones. # These first three, are user tunables SSH=${SSH:-ssh} ZREP_PATH=${ZREP_PATH:-zrep} #Set to /path/to/zrep, if needed, for remote #ZREP_CREATE_FLAGS="-o whatever" #Set for extra options on remote zfs create ######################################################################### # Everyting else below here, should not be touched. First we have autodetect # routines, and then internal utilities such as locking functions. Z_LOCAL_HOST=`uname -n` Z_LOCAL_HOST=${Z_LOCAL_HOST%%.*} # Solaris hack, to use native perl, which isnt always in $PATH,but should # always be there. It's also simple, straightforward, and non-extended. # But on other OSs will fall back to use default perl. PERL_BIN=${PERL_BIN:-/usr/perl5/bin} # Capability for this does not show in usage. # So, just tie this to MU6 related check,like HAS_SNAPPROPS PROPTYPES="local,received" # dump the usage message, and check for capabilities # make sure we dont spew for non-root, so that "zrep status" works case `id` in *\(root\)) zrep_checkfile=/var/run/zrep.check.$$ ;; *) zrep_checkfile=/tmp/zrep.check.$$ ;; esac zfs >$zrep_checkfile 2>&1 # Sigh. solaris 11 changes output yet again, there is no global usage msg, # which makes it much less efficient to parse. # So first 'if' clause is a bit of a hack. # Just assume has everything, if usage message is newer style if grep 'help' $zrep_checkfile >/dev/null ;then Z_HAS_X=1 # can use recv -x Z_HAS_SNAPPROPS=1 # can set properties on snapshots # This also lets me set accurate "last synced" timestamps # otherwise cant use zrep:sent sanely # would lose it on rollbacks DEPTHCAP="-d 1" # limits "list -r" else if grep 'receive.*-x' $zrep_checkfile >/dev/null ;then Z_HAS_X=1 # can use recv -x else Z_HAS_X=0 fi if grep 'set.*snapshot' $zrep_checkfile >/dev/null ;then Z_HAS_SNAPPROPS=1 # can set properties on snapshots else Z_HAS_SNAPPROPS=0 PROPTYPES="local" fi if grep 'list.*-d' $zrep_checkfile >/dev/null ;then DEPTHCAP="-d 1" # limits "list -r" else DEPTHCAP="" print WARNING: old ZFS version detected print WARNING: You may not nest zrep managed filesystems fi fi if ((Z_HAS_X)) ; then Z_HAS_O=1 # can recv use -o option else Z_HAS_O=0 fi rm $zrep_checkfile Z_LOCK_RETRY=${Z_LOCK_RETRY:-10} # default 10 second retry, 1 per sec Z_SAVE_COUNT=${Z_SAVE_COUNT:-5} Z_GLOBAL_LOCKFILE=/var/run/zrep.lock if [[ "$Z_GLOBAL_PID" == "" ]] ; then export Z_GLOBAL_PID=$$ fi Z_SETHOLD=${Z_SETHOLD:-"zfs hold"} # if your zfs isnt new enough, and you like to live dangerously, # you can skip setting holds by using this instead. # Although I may not have gotten around to using this in the code either! #Z_SETHOLD="echo skipping zfs hold on" # return 0 if "we" are holding lock, 1 otherwise # Note that we check for "us, OR our global parent", if different # zrep_has_global_lock(){ lockpid=`ls -l $Z_GLOBAL_LOCKFILE 2>/dev/null |awk -F/ '{print $NF}'` if [[ "$lockpid" == "" ]] ; then return 1 ; fi if [[ "$lockpid" != "$Z_GLOBAL_PID" ]] ; then if [[ "$lockpid" != "$$" ]] ; then return 1 fi fi return 0 } #Note: it is an ERROR to call this if you already have lock #It is binary, not recursive ownership. zrep_get_global_lock(){ typeset retry_count=$Z_LOCK_RETRY ln -s /proc/$Z_GLOBAL_PID $Z_GLOBAL_LOCKFILE && return 0 # otherwise, deal with fail # Check for dead old holder first. # CANNOT CLEAN UP OURSELVES: race condition problems. while (( retry_count > 0 )); do sleep 1 ln -s /proc/$Z_GLOBAL_PID $Z_GLOBAL_LOCKFILE && return 0 retry_count=$((retry_count-1)) done print Failed to acquire global lock return 1 } zrep_release_global_lock(){ if zrep_has_global_lock ; then rm $Z_GLOBAL_LOCKFILE return $? else print ERROR: zrep_release_global_lock called, but do not own lock return 1 fi } # returns PID of zrep process holding a lock on filesystem, if there is one. # NOTE: prints "-" NOT "", if lock unheld zrep_fs_lock_pid(){ zfs get -H -o value zrep:lock-pid $1 } zrep_has_fs_lock(){ typeset check=`zfs get -H -o value zrep:lock-pid $1` if ((check == $$)) ; then return 0 else return 1 fi } # use global lock first (if not already), then # grab lock on individual fs # return 1 on fail, 0 on lock acquired # Note that it is an ERROR to call this, if you already have lock # Note2: if a dead process has lock, it will forcibly override and # acqure lock zrep_lock_fs(){ # global lock is slow. so do quickcheck first. typeset check=`zrep_fs_lock_pid $1` newcheck if [[ "$check" != "-" ]] ; then # validate fs lock before giving up ls -d /proc/$check >/dev/null 2>&1 && return 1 fi zrep_get_global_lock || return 1 # Double-check if needed, now that we have global lock if [[ "$check" != "-" ]] ; then newcheck=`zrep_fs_lock_pid $1` if [[ "$newcheck" != "$check" ]] && [[ "$newcheck" != "-" ]] then # oops. someone else must have dealt with it. # If they havent reset it to "-" then give up zrep_release_global_lock return 1 fi print DEBUG: overiding stale lock on $1 from pid $check >/dev/fd/2 fi zfs set zrep:lock-pid=$$ $1 zfs set zrep:lock-time=`date +%Y%m%d%H%M%S` $1 zrep_release_global_lock } # release lock, if we have it. # Since this could be called by an exit cleanup routine blindly, # dont exit program if we dont have lock. But do return error zrep_unlock_fs(){ typeset lockpid=`zrep_fs_lock_pid $1` if ((lockpid != $$)) ; then return 1; fi #since "we" already have it locked, no need to get global lock first zfs inherit zrep:lock-time $1 zfs inherit zrep:lock-pid $1 return 0 } # Quit whole program with error status, outputting args to stderr # Release global lock if we are holding it # Unless we're running in parallel batch mode # I'll need to plan that out more carefully! # zrep_errquit(){ print Error: "$@" >/dev/fd/2 if zrep_has_global_lock ; then if [[ "$$" -ne "$Z_GLOBAL_PID" ]] ; then print EXTRA-ERROR: Running in child proc. print 'Not sure whether to release global lock. NOT releasing!' exit 1 else zrep_release_global_lock fi fi exit 1 } # Optimization wrapper for ssh: if destination host is ourself, dont use ssh. # Just run the local command mentioned # Be careful about quotes here. In fact, try not to use any. # Usage: zrep_ssh desthost commands_for_ssh go_here zrep_ssh(){ typeset ssh_cmd case "$1" in localhost|$Z_LOCAL_HOST) ssh_cmd="" ;; *) ssh_cmd="$SSH $1" ;; esac shift $ssh_cmd "$@" } zrep_gettimeinseconds(){ # unfortunately, solaris date doesnt do '%s', so need to use perl typeset PATH=$PERL_BIN:$PATH perl -e 'print int(time);' } ###### zrep_status # be sure to have included zrep_vars # This file contains all "status" related routines. # It should be folded into final "zrep" script # #Give this a top level zrep registered filesystem, NOT snapshot. # Will print out various status points, such as last sync date. # Or if given no args, will print out sync date for all zrep mastered fs # Note that the date given is time of SNAPSHOT, not time sync completed. # zrep_status(){ typeset check fs srcfs jdesthost destfs date lastsnap verbose=0 typeset printall=0 if [[ "$1" == "-v" ]] ; then verbose=1 ; shift fi if [[ "$1" == "" ]] ; then set -- `zrep_list_master` elif [[ "$1" == "-a" ]] ; then set -- `zrep_list` printall=1 fi while [[ "$1" != "" ]] ; do fs="$1" destfs=`zfs get -H -o value zrep:dest-fs $fs` if [[ "$destfs" == "-" ]] || [[ "$destfs" == "" ]]; then zrep_errquit "$fs is not a zrep registered filesystem" fi lastsnap=`getlastsnapsent $fs` if [[ "$lastsnap" == "" ]] ; then date="[NEVER]" else date=`zfs get -H -o value creation $lastsnap` fi if ((printall)) && ((verbose)) ; then # If we are printing out ALL filesystems, # then we have to make sure left side is always # "src filesystem", not "named filesystem" # then we have to check what the src fs is srcfs=`zfs get -H -o value zrep:src-fs $fs` else # Yes, okay, if -a is used, then # technically, this isnt always "src". # but it prints out right, so close enough :) srcfs="$fs" fi if ((verbose)) ; then desthost=`zfs get -H -o value zrep:dest-host $srcfs` printf "%-25s->%-35s %s\n" $srcfs "$desthost:$destfs" "${date#????}" else printf "%-47s" $srcfs print "last synced $date" fi shift done } # convenience function to list only local filesystems for which we are # zrep master for. # In contrast, zrep_list, lists ALL zrep registered filesystem, at the moment. # # Annoyingly... it would be way faster if we could just stick with the # pure "zfs get" implementation, but we also need to deal with the zone # issue. When a single zfs filesystem is visible aross multiple zones, # we dont want them all thinking they are master # # Durn. Individual validation required. zrep_list_master(){ typeset srchost for fs in `zfs get -H -o name -s local zrep:master "$@"` ; do srchost=`zfs get -H -o value zrep:src-host $fs` if [[ "$srchost" == "$Z_LOCAL_HOST" ]] ; then print $fs fi done } # Given ONE filesystem, print all zrep properties for it. # Note that this is internal routine. we do not validate input. list_verbose(){ print $1: zfs get -H -o property,value -s $PROPTYPES all $1 } # Note: called by both user, AND by zrep_status # # Usage: # zrep_list [-v] # zrep_list [-L] # zrep_list [-v] fs1 fs2 # list all zrep-initialized filesystems (NOT snapshots..) # If no specific fs listed, will show master, AND received filesystems, # unless -L given (in which case, only local masters will be shown) # # Normal output is one line per fs. # # -v gives all properties of each filesystem # Give only one of -L or -v # # This works because we only set this property on the actual fs. # "source type" on snapshots for this property is "inherited" not local # or "received" zrep_list(){ typeset fslist="" verbose=0 typeset printcmd="zfs get -H -o name -s $PROPTYPES zrep:dest-fs" case $1 in -v) verbose=1 printcmd=list_verbose shift ;; -L) printcmd="zrep_list_master" shift ;; esac if [[ "$1" != "" ]] ; then while [[ "$1" != "" ]] ; do if zfs list -t filesystem $1 >/dev/null 2>&1 ; then $printcmd $1 else zrep_errquit "Expecting filesystem, but got $1" fi shift done return fi # Must be "list all" now. But which output format? # if not verbose, we have a nice shortcut if (( $verbose == 0)) ; then $printcmd return fi # oh well. have to step through them one by one now, to # print out the properties associated with each zrep filesystem fslist=`zfs get -H -o name -s $PROPTYPES zrep:dest-fs` for fs in $fslist ; do print ${fs}: # sneaky cheat: only user-set properties will # match these 'source' types. So "grep zrep:" is not # neccessary. Although we may pick up other user-set values, # but that is not neccessarily a bad thing zfs get -H -o property,value -s $PROPTYPES all $fs print "" done } ################ zrep_snap # be sure to have included zrep_vars # This file contains routines related to # "make new snapshot, using next sequence number". # So it thus includes all snap sequence related routines # It may contain "sync snapshot" related routines for now. # It also is definitive for the format of snapshot names # It also contains most "query status of snaps" type routines, # such as "getlastsnapsent" # # Normal style for making a snapshot and syncing it: # 1. create a snapshot. # 2. sync it over # 3. set "zrep:sent" on *snapshot*, with timestamp in seconds # Old-nasty-zfs compat mode: # Step 3. Add/update "zrep:lastsent->snapname", and # "zrep:lastsenttime->timestamp", on *filesystem* # ###################################################################### # By observation, 'zfs list' shows snapshots order of creation. # last listed, should be last in sequence. # But, dont take chances!! getlastsequence(){ typeset lastval #remember, filesystems can have '_' in them getlastsnap $1|sed 's/.*@zrep_\(......\).*/\1/' } # prints out last snapshot zrep created, going purely by sequence. # Note: "last created", which may or may NOT be "last successfully synced". # This is basically "getallsnaps |tail -1" getlastsnap(){ zfs list -t snapshot -H -o name $DEPTHCAP -r $1 | sed -n '/@zrep_[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]/'p | sort | tail -1 } # Usage: getlastsnapsent zpool/FSNAME getlastsnapsent(){ # arg. more efficient if we can just return value directly, # but i'm using backwards compat :( typeset lastsent lastsent=`zfs get -H -o name -r -s local zrep:sent $1 | sort | tail -1` if [[ "$lastsent" != "" ]] ; then print $lastsent return fi # Fallback method, for backwards compat with older ZFS code, # since it cant set properties on snapshots zfs get -H -o value -s local zrep:lastsent $1 } # outputs time in seconds, of when the last successful sync for the # filesystem was done. (in format compatible with zrep_gettimeinseconds() ) # Note that this is time of actual sync, not snapshot creation time. # # This unfortunately needs to be compatible with both new way, and # old-nasty-hack-way # # In future, may take optional argument of which HOST to check # sync with. But since I currently only suport one host per fs... oh well. # If never synced, will return 1, and print "-" # getlastsynctime(){ typeset fs lastsent senttime if [[ "$1" == "" ]] ; then zrep_errquit Internal error: no arg to getlastsynctime fi fs="$1" # Deal with possibly upgraded system; # Check "lastsent", only as fallback. # copy from getlastsnapsent, but only using newest method lastsent=`zfs get -H -o name -r -s local zrep:sent $fs | sort | tail -1` senttime=`zfs get -H -o value zrep:sent $lastsent` if [[ "$senttime" != "-" ]] ; then print $senttime ; return 0; fi # ooops. try fallback to nasty old zfs-compat style senttime=`zfs get -H -o value zrep:lastsent $fs` print $senttime if [[ "$senttime" != "-" ]] ; then return 0; fi return 1 } #This is for synctosnap, and also zrep_expire getallsnaps(){ zfs list -t snapshot -H -o name $DEPTHCAP -r $1 | sed -n '/@zrep_[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]/'p | sort } # list all snapshots of the given filesystem, that are made by this prog # arg: fs list_autosnaps(){ if [[ "$1" == "" ]] ; then zrep_errquit "zrep internalerror: no arg for list_autosnaps" fi zfs list $DEPTHCAP -r -H -o name -t snapshot $1 | grep '@zrep_[0-9a-f][0-9a-f]' # Make sure this format matches other routines in here # Okay to just check first few digits though } # User entrypoint. Part of pair: snaponly, sendonly # Just makes snapshot. zrep_snaponly(){ typeset srcfs while [[ "$1" != "" ]] ; do srcfs="$1" ;shift zrep_lock_fs $srcfs if [[ $? -ne 0 ]] ; then # this function is supposed to be coordinated by user # therefore, if something else is competing, # coordination has failed. no retry. zrep_errquit zrep snaponly failed for $srcfs: cannot get lock fi makesnap $srcfs ||zrep_errquit snaponly for $srcfs failed zrep_unlock_fs $srcfs done } # # creates next snapshot in sequence # consider holding lock here # Caller must have zrep lock on filesystem: # we verify with zrep_has_fs_lock makesnap(){ typeset check oldseq newseq="" newseqX newsnap #sanity checks first! check="`zfs get -H -o value zrep:src-host $1`" if [[ "$check" != "$Z_LOCAL_HOST" ]] ; then print ERROR: we are not master host for $1 >/dev/fd/2 print master is $check, we are $Z_LOCAL_HOST >/dev/fd/2 exit 1 fi zrep_has_fs_lock $1 if [[ $? -ne 0 ]] ; then print Internal error: makesnap fail, no lock on $1 >/dev/fd/2 exit 1 fi oldseq=`getlastsequence $1` newseq=$((0x$oldseq)) newseqX=$(printf "%.6x" $(($newseq + 1)) ) #print DEBUG old=$oldseq new=$newseqX >/dev/fd/2 newsnap="$1@zrep_$newseqX" zfs snapshot $newsnap if [[ $? -eq 0 ]] ; then print $newsnap; return 0 else return 1 fi } ## This is the implentation for the "zrep clear" command ## Purpose is to remove all zrep related hooks from a local filesystem. ## (NOT delete it) ## Will remove zrep snapshots and zfs zrep: properties zrep_clear(){ print "WARNING: Removing all zrep configs and snapshots from $1" print Continuing in 10 seconds sleep 10 print Destroying any zrep-related snapshots from $1 snaplist=`list_autosnaps $1` for snap in $snaplist ; do zfs destroy -r $snap done print Removing zrep-related properties from $1 proplist=`zfs get -H -o property all $1|grep zrep:` for prop in $proplist ; do zfs inherit $prop $1 done } ## This is a special internal routine, used only by zrep_init. ## call with "srcfs errmsg1 errmsg2..." ## It will REMOVE REMOTEFS if set in PROPERTIES!! clearquit(){ remhost=`zfs get -H -o value zrep:dest-host $1` remfs=`zfs get -H -o value zrep:dest-fs $1` if [[ $? -eq 0 ]] && [[ "$remhost" != "-" ]] && [[ "$remfs" != "-" ]]; then zrep_ssh $remhost zfs destroy -r $remfs fi zrep_clear $1 shift zrep_errquit "$@" } # Shared internal routine. # Set the to/from properties on a fs for zrep # Called by zrep_init and zrep_changeconfig setfsconfigs(){ typeset srcfs="$1" desthost="$2" destfs="$3" fsname if [[ "$destfs" == "" ]] ; then zrep_errquit "zrep: no dest fs specified" fi zfs list $srcfs >/dev/null ||zrep_errquit "filesystem $srcfs must exist already" fsname=${srcfs##*/} case $destfs in # Originally, I had this passthrough only if fsname was at end # However,we must allow destfs to have different leaf name, # for circumstances such as replication to same host */*) : ;; *) # Only the pool name given. Let's make it explicit. destfs=$destfs/$fsname ;; esac zfs set zrep:src-fs=$srcfs $srcfs zfs set zrep:src-host=$Z_LOCAL_HOST $srcfs zfs set zrep:dest-fs=$destfs $srcfs zfs set zrep:dest-host=$desthost $srcfs zfs set zrep:savecount=$Z_SAVE_COUNT $srcfs } # Follow "initial set up" in workflow.txt # Some day, will allow init from pre-existing snaps< But not today! # Note that remote fs must share same stem name as source. (for now?) zrep_init(){ typeset srcfs="$1" desthost="$2" destfs="$3" fsname snap check vol=0 if [[ "$srcfs" == "" ]] ; then zrep_errquit "zrep: no fs specified" fi #sanity checks check="`zfs get -H -o value zrep:dest-fs $srcfs`" if [[ "$check" != "-" ]] ; then print "$srcfs is at least partially configured by zrep" zrep_errquit "To re-initialize, first use zrep clear $srcfs" fi check="`zfs get -H -o value type $srcfs`" if [[ "$check" == "volume" ]] ; then vol=1 if ((! Z_HAS_O )) ; then print "Sorry, your zfs is too old for zrep to handle volume initialization" zrep_errquit "Please initialize volume target by hand, if you won't upgrade" fi fi print Setting properties on $srcfs setfsconfigs $srcfs $desthost $destfs #setfsconfigs may do some "smarts" to adjust value, so get it again. destfs=`zfs get -H -o value zrep:dest-fs $srcfs` if (( Z_HAS_O )) ; then READONLYPROP="-o readonly=on" else READONLYPROP="" print Ancient local version of ZFS detected. print Creating destination filesystem as separate step zrep_ssh $desthost zfs create $ZREP_CREATE_FLAGS -o readonly=on $destfs || zrep_errquit "Cannot create $desthost:$destfs" fi snap="${srcfs}@zrep_000000" print Creating snapshot $snap zfs snapshot $snap || clearquit $srcfs "Cannot create initial snapshot $snap" # Note that we may not want to use -p for normal zrep syncs # We also should not use -F for normal recv. See workflow.txt # Note: we may have to reset readonly=on, if we used -p on send... # print Sending initial replication stream to $desthost:$destfs if (( $Z_HAS_X )) ; then # This is the nice, clean, modern codepath, to send # zrep settings over automatically at first init # # But check to see if sending volume or filesystem first, # and act appropriately # typeset MOUNTFILTER if (( vol )) ; then MOUNTFILTER="" else MOUNTFILTER="-x mountpoint" fi zfs send -p $snap | zrep_ssh $desthost zfs recv $MOUNTFILTER $READONLYPROP -F $destfs else ## arg.. Patch your systems!! # Doesn't support "recv -x mountpoint", so cant use -p in send # This means we have to manually set props lower down as well. zfs send $snap | zrep_ssh $desthost zfs recv $READONLYPROP -F $destfs fi if [[ $? -ne 0 ]] ; then clearquit $srcfs "Error transferring $snap to $desthost:$destfs. Resetting" fi # Successful initial sync! Woo! okay record that, etc. # ... after stupid old-zfs-compat junk, that is if (( ! Z_HAS_X )) ; then print Debug: Because you have old zfs support, setting remote properties by hand zrep_ssh $desthost zfs set readonly=on $destfs || clearquit Could not set readonly for $desthost:$destfs zrep_ssh $desthost zfs set zrep:src-fs=$srcfs $destfs zrep_ssh $desthost zfs set zrep:src-host=$Z_LOCAL_HOST $destfs zrep_ssh $desthost zfs set zrep:dest-fs=$destfs $destfs zrep_ssh $desthost zfs set zrep:dest-host=$destfs $destfs zrep_ssh $desthost zfs set zrep:savecount=$Z_SAVE_COUNT $destfs fi # Make sure to set format to match what zrep_sync() looks for! if (( Z_HAS_SNAPPROPS )) ; then typeset sentprop="zrep:sent=`zrep_gettimeinseconds`" zfs set $sentprop ${snap} else # Arg stupidold stuff cant set props on a snapshot # So we have to manually set these on both sides also, # "Just in case" zfs set zrep:lastsent=${snap} $srcfs zrep_ssh $desthost zfs set zrep:lastsent=${snap} $destfs fi # make sure the above ' set 's (sent, lastsent) # match what zrep_sync() does !!! # Note: we have to set master property NOW, not before, # because "recv -x zrep:master" Does Not Work properly # Also, it avoids things like "zrep sync all" from attempting # to sync it before initial sync has been done. # We don't even have to zrep_lock_fs until this is set zfs set zrep:master=yes $srcfs print Initialization copy of $srcfs to $desthost:$destfs complete } zrep_changeconfig(){ typeset srcfs="$1" desthost="$2" destfs="$3" check if [[ "$srcfs" == "" ]] ; then zrep_errquit "zrep: no fs specified" fi check=`getlastsnap $srcfs` if [[ "$check" == "" ]] ; then print "No pre-existing zrep snapshots found on $srcfs" >/dev/fd/2 print $srcfs is not initialized for zrep. cannot change config. >/dev/fd/2 zrep_errquit Use zrep init on $srcfs instead fi setfsconfigs $srcfs $desthost $destfs } ##### zrep_sync # contains meat of the "sync" level operations. # basic snap routines, and init rountines, are in zrep_snap #################### # synctosnap: called by zrep_sync, if a specific snapshot is specified. # # This LOCAL side, *and* REMOTE side, match up with local zrep_created # snapshot. ... # # Note that it uses zrep_lock_fs # # WARNING: if we force other side to roll to snap.... # we should NOT BE SYNCING ANY more. # At the moment, it is up to the user to ensure that nothing is going on # locally, and future zrep syncs wont just effectively roll forward again # on the remote side. # zrep sync jobs should probably be halted, until it is decided that # you want to sync again. # # In the future, I should support some kind of "pause" option, for # zrep sync all to ignore a rolled back filesystem # # synctosnap(){ typeset srcsnap=$1 destfs=$2 desthost=$3 typeset newsentlist typeset srcfs snapname destsnap if [[ "$desthost" == "" ]] ; then print ERROR: synctosnap did not receive all required args zrep_errquit "args=$@" fi srcfs=${srcsnap%@*} snapname=${srcsnap#*@} destsnap=${snapname} # Have to enforce OUR syntax. otherwise, any future attempt to # continue sync will fail. # ( getlastsnap() wont find it! ) # case $snapname in zrep_[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]*) : ;; *) zrep_errquit $srcsnap is not zrep snapshot. Cannot roll with it. ;; esac print Validating remote snap zrep_ssh $desthost zfs list -t snapshot $destfs@$destsnap >/dev/null if [[ $? -ne 0 ]] ; then zrep_errquit $destfs@$destsnap does not exist. Cannot roll to snap fi print "WARNING: We will be rolling back $destfs, on $desthost" print -n " to $snapname, made at: " zfs get -H -o value creation $srcsnap print "" print "All newer snapshots on remote side will be destroyed" print "You should have paused ongoing sync jobs for $destfs before continuing" print "Continuing in 20 seconds...." sleep 10 print "Continuing in 10 seconds...." sleep 10 zrep_lock_fs $srcfs || zrep_errquit "Cannot lock $srcfs" zrep_ssh $desthost zfs rollback -Rr $destfs@$destsnap || zrep_errquit roll failed print $desthost:$destfs rolled back successfully to $destsnap print Now cleaning up local snapshots # need to undo whatever zrep_sync does newsentlist=`getallsnaps $srcfs|sed "1,/@$snapname/d"` for snap in $newsentlist ; do zfs inherit zrep:sent $snap done zrep_unlock_fs $srcfs } # Usage: _snapandsync fs desthost destfs # internal routine called by zrep_sync and zrep_failover, # to do an incremental send. # You must hold filesystem lock before calling this # WE DO NOT DO ANY SAFETY OR LOCK CHECKS HERE. # # Wil create a new snap on srcfs, and sync it over to given destination # Sets our 'synced' marker on it as well. # _snapandsync(){ typeset srcfs=$1 desthost=$2 destfs=$3 typeset sentsnap newsnap snapname #srchost=`zfs get -H -o value zrep:src-host $srcfs` #if [[ "$srchost" != "$Z_LOCAL_HOST" ]] ; then # zrep_errquit _sync: We are not master for $srcfs #fi # Find incremental send starting point # Do this BEFORE creating new snap, because we should make new snap # if we cant do incremental anyway sentsnap=`getlastsnapsent $srcfs` if [[ "$sentsnap" == "" ]] ; then print zrep_sync could not find sent snap for $srcfs. zrep_errquit You must initialize $srcfs for zrep fi newsnap=`makesnap $srcfs` if [[ "$newsnap" == "" ]] ; then zrep_errquit zrep_sync could not create new snapshot for $srcfs fi _sync $srcfs $desthost $destfs $sentsnap $newsnap } # called by _snapandsync, and also zrep_synconly # Usage: sourcefs destinationhost destinationfs (lastsent (newsnap)) _sync(){ typeset srcfs=$1 desthost=$2 destfs=$3 typeset lastsent=$4 newsnap=$5 typeset snapname if [[ "$lastsent" == "" ]] ; then lastsent=`getlastsnapsent $srcfs` if [[ "$lastsent" == "" ]] ; then print zrep_sync could not find sent snap for $srcfs. zrep_errquit You must initialize $srcfs for zrep fi fi if [[ "$newsnap" == "" ]] ; then newsnap=`getlastsnap $srcfs` if [[ "$newsnap" == "" ]] ; then print zrep_sync could not find sent snap for $srcfs. zrep_errquit You must initialize $srcfs for zrep fi fi if [[ "$newsnap" == "$lastsent" ]] ; then print $newsnap already sent return 0 fi snapname=${newsnap#*@} print sending $newsnap to $desthost:$destfs typeset timeinsec=`zrep_gettimeinseconds` typeset senttimeprop="zrep:sent=$timeinsec" # Note: doing "-o $senttimeprop" sets prop on FILESYSTEM, not snap. # So we dont do that usually # other than zrep_init, this should be the ONLY place we do a send # Sigh. but now we also do in _refreshpull zfs send -I $lastsent $newsnap | zrep_ssh $desthost zfs recv $destfs if [[ $? -ne 0 ]] ; then zfs rename ${newsnap} ${newsnap}_unsent zrep_errquit Problem doing sync for $newsnap. renamed to ${newsnap}_unsent fi #Even if we are "old mode", other side may not be. # So try newer way first. zrep_ssh $desthost zfs set $senttimeprop $destfs@$snapname if [[ $? -ne 0 ]] ; then print WARNING: setting zrep:sent failed on $desthost:$destfs@$snapname print Using fallback. Go patch your system zrep_ssh $desthost zfs set zrep:lastsent=${newsnap} $destfs zrep_ssh $desthost zfs set zrep:lastsenttime=${timeinsec} $destfs fi if (( Z_HAS_SNAPPROPS )) ; then zfs set $senttimeprop ${newsnap} else #note that this is only for old-ZFS compatibility. # We dont really want to use this style if possible! zfs set zrep:lastsent=${newsnap} $srcfs zfs set zrep:lastsenttime=${timeinsec} $srcfs fi } #User entrypoint, for synconly, which is the pair of snaponly zrep_synconly(){ # annoyingly..need to make this almost identical to our current full # zrep_sync. but just skipping first steps :( # we can skip retries, though. typeset srcfs desthost destfs # at one point, accept multiple args. But not for now...? srcfs=$1 [[ "$srcfs" == "" ]] && zrep_errquit No fileystem specified for synconly desthost=`zfs get -H -o value zrep:dest-host $srcfs` destfs=`zfs get -H -o value zrep:dest-fs $srcfs` if [[ $? -ne 0 ]] || [[ "$desthost" == "-" ]] || [[ "$destfs" == "-" ]]; then zrep_errquit Problem getting zrep properties for fs $srcfs fi zrep_lock_fs $srcfs if [[ $? -ne 0 ]] ; then zrep_errquit Failed to acquire zrep lock for $srcfs fi _sync $srcfs $desthost $destfs || zrep_errquit sync failed for $srcfs _expire $srcfs #dont care so much if this fails zrep_unlock_fs $srcfs } #zrep_sync # make a new snapshot and copy it over. # Usage: zrep_sync [-q quiettime] (all|fs1 .. fsX) # See workflow.txt # Will call synctosnap if a snapshot is given instead of fsname # Normally, will bail out if another instance of zrep holds lock. # -q option says to check last update time of locked filesystems. # If sync more recent than given quiettime, then quietly ignore # zrep_sync(){ typeset srcfs destfs desthost sentsnap newsnap typeset quiettime=0 if [[ "$1" == "-q" ]] ; then quiettime="$2" shift shift if (( quiettime < 30 )) ; then zrep_errquit "-q must use value greater than 30" fi fi if [[ "$1" == "all" ]] ; then set -- `zrep_list_master` if [[ "$1" == "" ]] ; then # Stay quiet, so we dont spew if in cron #print No zrep mastered filesystems found exit fi fi if [[ "$1" == "" ]] ; then print Error: no filesystems specified for sync >/dev/fd/2 return 1 fi while [[ "$1" != "" ]] ; do srcfs="$1" [[ "$srcfs" == "" ]] && zrep_errquit No fileystem specified for sync desthost=`zfs get -H -o value zrep:dest-host $srcfs` destfs=`zfs get -H -o value zrep:dest-fs $srcfs` if [[ $? -ne 0 ]] || [[ "$desthost" == "-" ]] || [[ "$destfs" == "-" ]]; then zrep_errquit Problem getting zrep properties for fs $srcfs fi case $srcfs in *@*) synctosnap $srcfs $destfs $desthost return ;; esac zrep_lock_fs $srcfs if [[ $? -ne 0 ]] ; then # retry for lock for a while, if (quiettime>0 ) if ((quiettime==0)); then zrep_errquit Cannot lock $srcfs. Cannot continue fi typeset currtime=`zrep_gettimeinseconds` snaptime elapsed snaptime=`getlastsynctime $srcfs` if (( snaptime == 0 )) ; then zrep_errquit quiet mode set, but no last snap for $srcfs fi elapsed=$((currtime - snaptime)) if ((elapsed > quiettime)) ; then print DEBUG: $elapsed seconds have elapsed since last sync of $srcfs zrep_errquit quiet time limit of $quiettime seconds exceeded for busy fs $srcfs else print Quiet mode: skipping busy fs $srcfs at `date` return fi fi _snapandsync $srcfs $desthost $destfs # Make this message match what zrep_expire uses.. print Expiring zrep snaps on $srcfs _expire $srcfs zrep_unlock_fs $srcfs shift done } # zrep_refresh is a "pull" version of "zrep_sync" # The concept is a bit of a hack. # It primarily exists so people can run a secure backup server, that # has ssh access to all hosts, but not vice versa # # Implementation is a bit sketchy. # For initial, non-optimal run, perhaps take advantage of # ssh host zrep synconly # to avoid too much duplication of things? # but will still need to set all the perms n things. Nastyyy.. # The MAIN nastiness, is that all our locks are on the "master" side. # Which depends on the PID still being there!! # But if we start now running things on the "slave" side.. # There is potential for problems # Examine critical points and reasons for lock: # 1. while doing analysis of which snap to send # 2. to avoid paralel "zfs send"s running. # 3. for update of timestamp # # We can still wrap #1 and #2 in a single lock call. # (and still on the src side!) # The ugly comes when updating zrep:sent. Dont want to update wrong snap! # So long as we do some kind of check to see that we're not going # backwards when we get lock a second time ... we should be relatively okay. # However.. for simplicity... going to just cross fingers and wrap # all three in single remote lock call, through _refreshpull # zrep_refresh(){ typeset srcfs destfs desthost newsnap newseq master # for now, just handle ONE arg, not multiple fs list destfs="$1" if [[ "$1" == "" ]] ; then print Error: no filesystems specified for refresh >/dev/fd/2 return 1 fi master=`zfs get -H -o value -s local zrep:master $destfs` if [[ "$master" == "yes" ]] ; then zrep_errquit Sorry, you cant run refresh on a master mode fs $destfs fi srchost=`zfs get -H -o value zrep:src-host $destfs` srcfs=`zfs get -H -o value zrep:src-fs $destfs` zrep_lock_fs $destfs if [[ $? -ne 0 ]] ; then zrep_errquit Cannot lock $destfs. Cannot continue fi print DEBUG: refresh step 1: Going to $srchost to snapshot $destfs newsnap=`zrep_ssh $srchost $ZREP_PATH snaponly $srcfs` if [[ $? -ne 0 ]] ; then zrep_errquit snap of src $srcfs on $srchost failed fi # yes, MORE paranoia.. case $newsnap in *@zrep_*) newseq=${newsnap#*@} ;; *) zrep_errquit Unrecognized output from src snap. Cannot continue ;; esac typeset timeinsec=`zrep_gettimeinseconds` typeset senttimeprop="zrep:sent=$timeinsec" print DEBUG: refresh step 2: Pulling $newsnap zrep_ssh $srchost $ZREP_PATH _refreshpull $newsnap | zfs recv $destfs if [[ $? -ne 0 ]] ; then zrep_errquit Unforseen error pulling snapshot $newsnap from $srchost fi zfs set $senttimeprop $destfs@$newseq if [[ $? -ne 0 ]] ; then print WARNING: expected local copy $destfs@newseq does not exist >/dev/fd/2 fi zrep_unlock_fs $destfs } # Hidden command-line option for "zrep refresh" # This is the "remote call" to support zrep refresh # ( aka zrep_refresh ) # In principle, its kinda like "zrep expire" being callable by # both the user, and the program itself. # However, this routine is definitely not supposed to be user visible # .. eh... maybe someday. but initial design is "private" _refreshpull(){ typeset fs snapname lastsent latest timeinsec senttimeprop snapname="$1" fs=${snapname%@*} print DEBUG: _refreshpull: snapname=$snapname, fs=$fs >/dev/fd/2 zrep_lock_fs $fs if [[ $? -ne 0 ]] ; then zrep_errquit Could not lock $fs fi #We should now; # 1. compare to latest snap. quit if not latest # 2. get timestamp # 3. trigger a zfs send # 4. set timestamp if no errors. # I think it is reasonable to presume that if the receive failed, # we will see an error by the pipe blowing up. # lastsent=`getlastsnapsent $fs` if [[ "$lastsent" == "" ]] ; then zrep_errquit Canthappen: _refreshpull cant findlastsent snap fi latest=`getlastsnap $fs` if [[ "$latest" != "$snapname" ]] ; then zrep_errquit Sync error: $snapname is not latest snap for $fs fi timeinsec=`zrep_gettimeinseconds` senttimeprop="zrep:sent=$timeinsec" if (( Z_HAS_SNAPPROPS ==0)) ; then zrep_errquit Error: we currently only support modern ZFS that allows setting props on snaps fi zfs send -I $lastsent $latest if [[ $? -ne 0 ]] ; then zrep_errquit Some kind of error during sending. Bailing out of _refreshpull fi zfs set $senttimeprop $latest zrep_unlock_fs $fs } # _expire: # get rid of "old" snapshots for a specifically named filesystem # # Note0: you must hold local(master) fs lock first # # Note1: expire BOTH SIDES, if we are master # Keep in mind that sometimes master and dest are on same system # # Note2: Be sure to NEVER delete most recent sent snapshot!! # INTERNAL routine. For external-facing routine, see zrep_expire _expire(){ typeset savecount currcount lastsent remotehost remotefs sanity typeset tmpfile=/var/run/zrep_expire.$$ typeset local=0 master if [[ "$1" == "-L" ]] ; then local=1; shift fi master=`zfs get -H -o value -s local zrep:master $1` zrep_has_fs_lock $1 || zrep_errquit zrep_expire Internal Err caller did not hold fs lock savecount=`zfs get -H -o value zrep:savecount $1` # do not use (()) in case value unset if [[ $savecount < 1 ]] ; then zrep_errquit zrep:savecount on $1 set to improper value $savecount fi if [[ "$master" == "yes" ]] ; then lastsent=`getlastsnapsent $1` if [[ "$lastsent" == "" ]] ; then zrep_errquit corrupted zrep data: no last sent detected. Stopping expire fi getallsnaps $1 |egrep -v $lastsent >$tmpfile savecount=$((savecount-1)) else getallsnaps $1 >$tmpfile fi currcount=`wc -l < $tmpfile` if ((currcount > savecount )) ; then currcount=$((currcount - savecount)) head -$currcount $tmpfile >$tmpfile.2 mv $tmpfile.2 $tmpfile for snap in `cat $tmpfile` ; do print DEBUG: expiring $snap zfs destroy $snap done fi rm $tmpfile if [[ "$master" != "yes" ]] || ((local ==1)) ; then #This fs is dest fs. We are done. return #otherwise, go expire on remote side as well fi remotehost=`zfs get -H -o value zrep:dest-host $1` remotefs=`zfs get -H -o value zrep:dest-fs $1` print Also running expire on $remotehost:$remotefs now... sanity=`zrep_ssh $remotehost zfs get -H -o value -s local zrep:master $remotefs` # Normally, dont quit on error. But this is super-bad. if [[ "$sanity" == "yes" ]] ; then zrep_errquit "Remote side also marked as master ** $remotehost:$remotefs" fi zrep_ssh $remotehost $ZREP_PATH expire $remotefs ||print REMOTE expire failed } # top-level user-facing routine. # expire old snaps for some or all zrep filesystems. # Different ways of calling: # zrep expire all Run expire on all zrep fs # zrep expire Run expire on zrep fs we are master for, plus remote # zrep expire -L Run expire on zrep fs we are master for. SKIP remote # zrep expire fs .. Run expire only on fs, plus remote if it is a master # zrep expire -L fs Run expire only on fs. Skip remote # # If no arg given, expire only filesystems we are master for # If "all" given, expire literally all. # zrep_expire() { typeset local if [[ "$1" == "-L" ]] ; then local="-L" shift fi if [[ "$1" == "all" ]] ; then set -- `zrep_list` elif [[ "$1" == "" ]] ; then set -- `zrep_list_master` fi # Note: we should continue if we hit problems with an individual # filesystem. Otherwise we risk letting server selfdestruct fill # over one troublesome filesystem # while [[ "$1" != "" ]] ; do zrep_lock_fs $1 print Expiring zrep snaps on $1 _expire $local $1 || print WARNING: expire failed for $1 zrep_unlock_fs $1 shift done } # run this on 'master' side, to make other side master zrep_failover(){ typeset local=0 fs snap="" remotehost remotefs check if [[ "$1" == "-L" ]] ; then local=1 shift fi if [[ "$1" == "" ]] ; then usage exit 1 fi zfs list $1 >/dev/null || zrep_errquit invalid filesystem $1 check=`zfs get -H -o value -s local zrep:master $1` if [[ "$check" != "yes" ]] ; then zrep_errquit $1 not master. Cannot fail over fi fs="$1" case $fs in *@*) snap=$fs fs=${srcsnap%@*} ;; esac zrep_lock_fs $fs ||zrep_errquit could not lock $fs remotehost=`zfs get -H -o value zrep:dest-host $fs` remotefs=`zfs get -H -o value zrep:dest-fs $fs` print setting readonly on local $fs, then syncing zfs set readonly=on $fs if ((local ==1)) ; then print failover for $1 in LOCAL mode if [[ "$snap" == "" ]] ; then snap=`getlastsnapsent $1` zfs list $1 >/dev/null || zrep_errquit No last synced snap found for $1. Cannot fail over print Rolling back to last sync $snap else print Rolling back to specified snap $snap fi zfs rollback -Rr $snap ||zrep_errquit Rollback to $snap failed else ## Need to sync both sides before mode switch! ## If named snap, roll back. ## otherwise, "roll forward" by doing one last sync if [[ "$snap" != "" ]] ; then typeset snapname snapname=${snap#*@} print Rolling back to local $snap zfs rollback -Rr $snap || zrep_errquit Rollback to $snap failed print Rolling back $remotehost to $remotefs@$snapname zrep_ssh $remotehost zfs rollback $remotefs@$snapname || zrep_errquit remote rollback failed else # makes new snapshot, and syncs _snapandsync $fs $remotehost $remotefs || zrep_errquit final sync failed. failover failed. fi fi print Reversing master properties for $Z_LOCAL_HOST:$fs zfs set zrep:dest-fs=$fs $fs zfs set zrep:dest-host=$Z_LOCAL_HOST $fs zfs set zrep:src-fs=$remotefs $fs zfs set zrep:src-host=$remotehost $fs zfs inherit zrep:master $fs zrep_unlock_fs $fs if (( local ==0)) ;then print Setting master on $remotehost:$remotefs zrep_ssh $remotehost $ZREP_PATH takeover -L $remotefs fi } # run this on 'dest' side, to promote it to master zrep_takeover(){ typeset fs snap remotehost remotefs check local=0 if [[ "$1" == "-L" ]] ; then local=1 shift fi if [[ "$1" == "" ]] ; then usage exit 1 fi fs="$1" zfs list $fs >/dev/null || zrep_errquit invalid filesystem $fs check=`zfs get -H -o value -s local zrep:master $fs` if [[ "$check" = "yes" ]] ; then zrep_errquit $fs is already master. Cannot takeover fi remotehost=`zfs get -H -o value zrep:src-host $fs` remotefs=`zfs get -H -o value zrep:src-fs $fs` if (( local == 0 )) ; then print starting failover from remote side $remotehost zrep_ssh $remotehost $ZREP_PATH failover $remotefs exit $? fi # If here, we must be in local mode. # So... just set properties! # (and roll back, if desired) case $fs in *@*) snap=$fs fs=${srcsnap%@*} ;; esac zrep_lock_fs $fs zfs inherit readonly $fs if [[ "$snap" != "" ]] ; then print "WARNING: Before takeover, we will be rolling $fs" print -n " to $snapname, made at: " zfs get -H -o value creation $snap print "" print "All newer snapshots will be destroyed" print Continuing in 10 seconds... sleep 10 zfs rollback -Rr $snap || zrep_errquit Rollback to $snap failed fi print Setting master properties for $Z_LOCAL_HOST:$fs zfs set zrep:src-fs=$fs $fs zfs set zrep:src-host=$Z_LOCAL_HOST $fs zfs set zrep:dest-fs=$remotefs $fs zfs set zrep:dest-host=$remotehost $fs zfs set zrep:master=yes $fs zrep_unlock_fs $fs } ######## zrep_top continues here usage(){ print zrep v${ZREP_VERSION}: a program to replicate a zfs filesystem to another print in an ongoing basis. print More documentation at a later date. print " Philip Brown, 2012" print "" print Simple usage summary: print 'zrep (init|-i) ZFS/fs remotehost remoteZFSpool/fs' print 'zrep (sync|-S) [-q seconds] ZFS/fs' print 'zrep (sync|-S) [-q seconds] all' print 'zrep (sync|-S) ZFS/fs@snapshot -- temporary retroactive sync' print 'zrep (status|-s) [-v] [(-a|ZFS/fs)]' print 'zrep refresh ZFS/fs -- pull version of sync' print 'zrep (list|-l) [-v]' print 'zrep (expire|-e) [-L] (ZFS/fs ...)|(all)|()' print 'zrep (changeconfig|-C) ZFS/fs remotehost remoteZFSpool/fs' print 'zrep failover [-L] ZFS/fs' print 'zrep takeover [-L] ZFS/fs' print 'zrep clear ZFS/fs -- REMOVE ZREP CONFIG AND SNAPS FROM FILESYSTEM' print print ' -q option says to Quietly ignore locked filesystems that have synced' print ' more recently than the given amount of seconds' print print 'Paired commands for high-transaction systems:' print ' zrep snaponly ZFS/fs' print ' zrep synconly ZFS/fs' print 'The above two commands split the simple sync subcommand, into two' print 'separate steps, so that a database, etc. may resume while the sync' print 'completes in the background' print '' print ' More detailed examples can be found at:' print http://www.bolthole.com/solaris/zrep/zrep.documentation.html } case "$1" in "") usage ;; clear) shift zrep_clear $1 ;; expire|-e) shift zrep_expire "$@" ;; init|-i) shift zrep_init "$@" ;; changeconfig|-C) shift zrep_changeconfig "$@" ;; sync|-S) shift zrep_sync "$@" ;; snaponly) shift zrep_snaponly "$@" ;; synconly) shift zrep_synconly "$@" ;; refresh) # yes keep this in this order shift zrep_refresh "$@" ;; status|-s) shift zrep_status "$@" ;; list|-l) shift zrep_list "$@" ;; failover) shift zrep_failover "$@" ;; takeover) shift zrep_takeover "$@" ;; _refreshpull) # Secret option DO NOT PUT IN USAGE!! shift _refreshpull $1 ;; *) usage ;; esac