From c8c632e8acd1b0af0d8502a9f912f6ca09addb58 Mon Sep 17 00:00:00 2001 From: Ralph Ronnquist Date: Thu, 6 Jun 2024 22:14:42 +1000 Subject: [PATCH] inital capture --- Makefile | 7 ++++++ timeliner | 64 ++++++++++++++++++++++++++++++++++++++++++++++++ timeliner-backup | 49 ++++++++++++++++++++++++++++++++++++ timeliner-cron | 17 +++++++++++++ timeliner.8.adoc | 49 ++++++++++++++++++++++++++++++++++++ 5 files changed, 186 insertions(+) create mode 100644 Makefile create mode 100755 timeliner create mode 100755 timeliner-backup create mode 100644 timeliner-cron create mode 100644 timeliner.8.adoc diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0d06bc8 --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ +MANPAGES = $(basename $(wildcard *.8.adoc)) +$(warning MANPAGES = $(MANPAGES)) + +all: $(MANPAGES) + +%.8: %.8.adoc + asciidoctor -bmanpage $< diff --git a/timeliner b/timeliner new file mode 100755 index 0000000..e0710ae --- /dev/null +++ b/timeliner @@ -0,0 +1,64 @@ +#!/bin/sh +# +# Make a local timeline of given directories +# +# $1 = timeline root directory +# $2... directories for the timeline +# +# This is intended for a cron-driven backup script that regularly (eg. +# hourly) snapshots given directory trees $2... into a succession of +# date stamped trees under $1 by using hard links. +# +# NOTE: outside mutex synchronisation may be needed e.g. with flock. +# +# $1/current is the direct store of the current snapshot. This is +# updated by rsync from the local source trees regularly +# +# $1/yyyy-dd-mm form timeline window of daily snapshots by means of +# hard links to "$1/current" files. +# +# This may be combined with timeliner-remote for caching and storing +# the snapshots long term while a shorter, rolling most recent +# timeline window is kept locally. +# + +: ${SNAPTIME:=10:00:00} + +## Helper function to time stamp log lines. +log() { + date +"%Y-%m-%d %H:%M:%S - $*" +} + +log "$0 begin" + +BACKUP=$1 +shift +ERR=false +for D in "$@" ; do + rsync -aR --delete-after /.$(realpath "$D") $BACKUP/current/. || \ + ERR=true +done +$ERR && exit 1 + +HOST="${BACKUP%%:*}" +BASE="${BACKUP#*:}" +if [ "$HOST" = "$BACKUP" ] ; then + cmd() { $* ; } +else + cmd() { ssh $HOST "$*" ; } +fi + +TS=$(date +%s) +TODAY="$(date -d @$TS +%Y-%m-%d)" + +# If snapshot of $TODAY exists then exit +cmd test -d "$BASE/$TODAY" && exit 0 + +# If there is a snap of yesterday and time < SNAPTIME then exit +cmd test -d "$BASE/$(date -d "@$((TS - 86400))" "+%Y-%m-%d")" && \ + test "$TS" -lt "$(date -d "$SNAPTIME" +%s)" && exit 0 + +log ".. snapshot $TODAY" +cmd mkdir -p "$BASE/$TODAY" +cmd cp -axl "$BASE/current/." "$BASE/$TODAY/." +log "$0 end" diff --git a/timeliner-backup b/timeliner-backup new file mode 100755 index 0000000..11f9f42 --- /dev/null +++ b/timeliner-backup @@ -0,0 +1,49 @@ +#!/bin/sh +# +# Make a remote timeline of given local directories by keeping a +# rolling local timewindow and rsync that to the remote host. +# +# $1 = remote host:base +# $2 = local base + +## Helper function to time stamp log lines. +log() { date +"%Y-%m-%d %H:%M:%S - $*" ; } + +## Helper function to test if $1 string is less than $2 string +before() { [ -n "$(echo yes | awk "\"$1\"<\"$2\"")" ] ; } + +log "$0 begin" + +LOCAL=$2 +REMOTE=$1 +WINDOW=4 +LC_ALL=C + +HOST=${REMOTE%:*} +BASE=${REMOTE#*:} + +DDAY="$(date -d "-$WINDOW days" +%Y-%m-%d)" + +# determine day of most recent remote snapshot +RDAY="$(ssh $HOST ls -r $BASE | sed '2,$d')" + +for DAY in $(ls $LOCAL) ; do + [ "$DAY" = "current" ] && continue + log "local backup directory $LOCAL/$DAY" + if [ -z "$RDAY" ] ; then + log ".. remote store is empty" + ssh $HOST mkdir $BASE/$DAY + elif before "$RDAY" "$DAY" ; then + log ".. latest remote $RDAY before local $DAY" + ssh $HOST mkdir $BASE/$DAY '&&' cp -axl $BASE/$RDAY/. $BASE/$DAY/. + elif [ "$DAY" != "$RDAY" ] ; then + log ".. local $DAY before latest remote $RDAY" + before "$DAY" "$DDAY" && rm -rf $LOCAL/$D # drop local snapshot + continue # and skip the copying + fi + log ".. copy %DAY to remote $REMOTE" + rsync -a --delete-after $LOCAL/$DAY/. $REMOTE/$DAY/. + RDAY=$DAY +done + +log "$0 done" diff --git a/timeliner-cron b/timeliner-cron new file mode 100644 index 0000000..27d7161 --- /dev/null +++ b/timeliner-cron @@ -0,0 +1,17 @@ +#!/bin/sh +# +# This is intended to be installed as a cron.hourly script. +# + +#LOGFILE=/var/log/timeliner.log +#LOCAL=/backup +#BASE=/backup +#REMOTE=remote.example.com # needs paswwordless ssh access to root +#DIRS="/root /etc /home /opt /usr" + +[ -z "$DIRS" ] && exit 0 # "not configured yet" + +{ + flock -n 1 || exit 0 + timeliner $LOCAL $DIRS && timeliner-backup $REMOTE:$BASE $LOCAL +} >> $LOGFILE 2>&1 diff --git a/timeliner.8.adoc b/timeliner.8.adoc new file mode 100644 index 0000000..ab57385 --- /dev/null +++ b/timeliner.8.adoc @@ -0,0 +1,49 @@ += timeliner(8) +:doctype: manpage + +== NAME + +timeliner - cron driven backup of directories into local timeline + +== SYNOPSIS + +*timeliner* _target_ _source_* + +*timeliner-backup* _remote_ _target_ + +== DESCRIPTION + +=== timeliner + +In normal use, *timeliner* would be set up as an _hourly_ `cron` task. +It then makes hourly _rsync_ copies of the _source_* directories into +the _target/current_ directory, plus _daily_ hardlink snapshots of +that directory as a series of timeline directories named in the format +of _target/yyyy-mm-dd_ (where _yyyy_ is the year in 4 digits, _mm_ is +the month in 2 digits, and _dd_ is the day of the month in 2 digits). +The daily snapshots are regularly made as early as possible after the +hard coded SNAPTIME of 10:00:00, or possibly earlier in the day if the +previous day snapshot is missing. + +The target directory is either a directory pathname (a relative or +absolute pathname) or a "remote pathname" in the form of +_host:directory_. + +* A plain pathname nominates the base directory for the `current` and + _yyyy-mm-dd_ timeline directories. + +* A remote pathname results in `rsync` and `ssh` being used for + maintaining the timeline ath the nominated remote base directory. + +In both cases, the timeline directory must reside in a filesystem +cabale of hard links. + +== timeliner-backup + +*timeliner-backup* is a companion script that normally is set up as a +follow-on _hourly_ cron task to transfer a _target_ timeline to a +_remote_ store and then maintain the local _target_ timeline as a +rolling window of only the few most recent daily backups. The remote +timeline is built by making a hard link copy of the most recent prior +day before using `rsync` to copy the next day snapshot. The local +recent day snapshot is synchronised hourly to the remote host. -- 2.39.5