From: Ralph Ronnquist Date: Thu, 29 Jun 2023 09:11:10 +0000 (+1000) Subject: Merge branch 'suites/experimental' X-Git-Tag: 1.0~1 X-Git-Url: https://git.rrq.au/?a=commitdiff_plain;h=2c3cea697eeba53a967bea599bb45bbe54f8bd8d;hp=38ceca0fc206b4cf5026c677acb7f97de0e650fb;p=rrq%2Ffusefile.git Merge branch 'suites/experimental' --- diff --git a/Makefile b/Makefile index c2a0ba6..9f02adc 100644 --- a/Makefile +++ b/Makefile @@ -3,10 +3,10 @@ BINS = fusefile default: $(BINS) ifneq (${DEBUG},) -fusefile: CFLAGS += -DDEBUG=1 -g +${BINS}: CFLAGS += -DDEBUG=1 -g endif -fusefile: CFLAGS += -Wall -D_FILE_OFFSET_BITS=64 +${BINS}: CFLAGS += -Wall -D_FILE_OFFSET_BITS=64 fusefile: LDFLAGS = -lfuse -pthread .INTERMEDIATE: fusefile.o @@ -26,7 +26,7 @@ README.html: README.adoc SBINDIR = $(DESTDIR)/usr/sbin MAN8DIR = $(DESTDIR)/usr/share/man/man8 -SBINFILES = fusefile +SBINFILES = fusefile fusedisk MAN8FILES = fusefile.8 INSTALLTARGETS = $(addprefix $(SBINDIR)/,$(SBINFILES)) diff --git a/README.adoc b/README.adoc index 500e81b..7bf9049 100644 --- a/README.adoc +++ b/README.adoc @@ -1,82 +1,29 @@ -fusefile -======== +# fusefile +:author: Ralph Ronnquist +:revdate: Sun, 30 Apr 2023 21:18:34 +1000 -This project implements a "fuse" device to mount as a single file that -is a concatenation of fragments of one or more files. The fused file -allows overwriting the parts files, but not changing their sizes, and -only for parts files that are writable upon first access. +This project implements a "fuse" device to mount a concatenation of +fragments of one or more files as a single file. -FUSE file mount for combining file fragments. - -== SYNOPSIS +The __fused file__ allows writing to fragments (without changing their +sizes); of course only for writable fragment files. The fused file may +be set up with an __overlay file__ to capture changes instead of +writing the underlying fragment files. ==== -*fusefile* [ _fuse options_ ] *mountpoint* +_filename_/from:to+ ... +This is a usage example to set up a fused file C consisting of files A +and B: +---- +$ fusefile C A B +---- ==== -## DESCRIPTION - -*fusefile* is FUSE file mount that presents a series of fragments of -other files as a contiguous concatenation. It bind mounts a driver on -top of the file mountpoint to present the nominated file fragments as -a single, contiguous file. - -The fragment arguments include the filename of a source file, and -optionally start and end byte positions. All in all there five -variations: - - * __filename__ include all of the file. - - * __filename/__ include all of the file named with "/" in the pathname. This case requires a final "/", since the last "/" separates the filename from the position details. - - * __filename/from__ include the file from the given start position, to end. - - * __filename/-to__ include the file from beginning to the given end position (not included). - - * __filename/from:to__ include the file from the given start position, up to the given end position (not included). - -## EXAMPLES - -.Insert file "y" into file "x" at position 1200: - - $ fusefile -ononempty x x/:1200 y x/1200: - -That mount will shadow the original file "x", and presents the -fused file instead. - -.Make file y be a swap of the beginning and end of file "x", at position 2442: - - $ fusefile y x/2442: x/:2442 - -.Replace a partition in an image file with a different file - - $ partx -oNR,START,SECTORS disk.raw - NR START SECTORS - 1 2048 2097152 - 2 2099200 409600 - 3 2508800 14268383 - # Replace partition 2 of 409600 sectors from 2099200 with - # the file "insert.fat" clipped to 409600 sectors. - $ fusefile -ononempty disk.raw \ - disk.raw/0:$(( 2099200*512 )) \ - insert.fat/0:$(( 409600*512 )) \ - disk.raw/$(( (2099200+409600)*512 )): - - -## NOTES - -Note that **fusefile** opens the nominated source file(s) before bind -mounting. With the fuse option __-ononempty__ it will bind over an -non-empty file, which may be useful. The source file descriptors -remain open, but the source fragments are not recomputed. If a source -file changes or reduces in size "behind" the fused file, then anything -may happen. - -If the mountpoint file doesn't exist, then **fusefile** creates it and -removes it when unmounted. - -Unmounting is done with "fusermount -u __fused file__" as usual. +==== +This is an example of tearing down a fused file C: +---- +$ fusermount -u C +---- +==== -## AUTHOR +See the +man page+ for usage details and some more examples. -Ralph Rönnquist diff --git a/debian/changelog b/debian/changelog index b31afc7..4339d31 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,48 @@ +fusefile (1.0) unstable; urgency=medium + + * Version change for submission to debian + + -- Ralph Ronnquist Sun, 25 Jun 2023 17:39:21 +1000 +fusefile (0.5.7) unstable; urgency=medium + + * Updated man page. + + -- Ralph Ronnquist Sun, 25 Jun 2023 13:37:38 +1000 +fusefile (0.5.6) unstable; urgency=medium + + * Added -dump function for dumping fragments including overlay table. + + -- Ralph Ronnquist Wed, 21 Jun 2023 00:09:34 +1000 +fusefile (0.5.5) unstable; urgency=medium + + * Updated man page. + + -- Ralph Ronnquist Thu, 25 May 2023 16:01:40 +1000 +fusefile (0.5.4) unstable; urgency=medium + + * Include fusedisk helper script for block mount + + -- Ralph Ronnquist Wed, 24 May 2023 19:48:53 +1000 +fusefile (0.5.3) unstable; urgency=medium + + * Updated package description. + + -- Ralph Ronnquist Mon, 03 Oct 2022 13:01:26 +1100 +fusefile (0.5.2) unstable; urgency=medium + + * Reworked overlay with new region representaiton. + + -- Ralph Ronnquist Fri, 12 Aug 2022 08:57:37 +1000 +fusefile (0.5.1) unstable; urgency=medium + + * Fixes to overlay function. + + -- Ralph Ronnquist Thu, 11 Aug 2022 01:40:48 +1000 +fusefile (0.5) unstable; urgency=medium + + * Added overlay file option. + + -- Ralph Ronnquist Tue, 09 Aug 2022 21:22:35 +1000 fusefile (0.4) unstable; urgency=medium * Revised into overwriting in sources without size change. diff --git a/debian/control b/debian/control index 9ad6d54..908912b 100644 --- a/debian/control +++ b/debian/control @@ -1,16 +1,21 @@ Source: fusefile Section: admin Priority: optional -Maintainer: Ralph Ronnquist +Maintainer: Ralph Ronnquist Build-Depends: debhelper-compat (= 13), libfuse-dev -Standards-Version: 4.6.0 +Standards-Version: 4.6.1 Homepage: https://git.devuan.org/rrq/fusefile.git Vcs-Git: https://git.devuan.org/rrq/fusefile.git +Rules-Requires-Root: no Package: fusefile Architecture: any Depends: ${shlibs:Depends}, ${misc:Depends}, fuse -Description: FUSE file mount to present a series of fragments of files - as if a real, read-only file. It simply bind mounts a driver on top - of the file mountpoint to present the nominated file fragments as a - single, contiguous file. +Description: Using FUSE to combine file fragments into a single file. + fusefile is a FUSE "file mount" that presents a series of fragments + of other files as a contiguous concatenation. Technically it bind + mounts a driver on top of the filename mountpoint to provide access + to the given file fragments as if in a single, contiguous file. The + fusefile mount driver offers read/write access to the fused file, + distributing written data accross the given fragments or to an + overlay file. diff --git a/debian/copyright b/debian/copyright index 802bc86..f2f4788 100644 --- a/debian/copyright +++ b/debian/copyright @@ -1,5 +1,6 @@ Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: fusefile +Upstream-Contact: Ralph Ronnquist Source: https://git.devuan.org/rrq/fusefile.git Files: * diff --git a/debian/fusefile.links b/debian/fusefile.links new file mode 100644 index 0000000..ed132f0 --- /dev/null +++ b/debian/fusefile.links @@ -0,0 +1 @@ +usr/share/man/man8/fusefile.8 usr/share/man/man8/fusedisk.8 diff --git a/debian/rules b/debian/rules index 9ba3ef6..aac9e81 100755 --- a/debian/rules +++ b/debian/rules @@ -1,5 +1,4 @@ #!/usr/bin/make -f -# See debhelper(7) (uncomment to enable) # output every command that modifies files on the build system. #export DH_VERBOSE = 1 @@ -18,7 +17,7 @@ dh $@ -# dh_make generated override targets +# dh_make generated ecamples of override targets # This is example for Cmake (See https://bugs.debian.org/641051 ) #override_dh_auto_configure: # dh_auto_configure -- # -DCMAKE_LIBRARY_PATH=$(DEB_HOST_MULTIARCH) diff --git a/fusedisk b/fusedisk new file mode 100755 index 0000000..f7df50e --- /dev/null +++ b/fusedisk @@ -0,0 +1,42 @@ +#!/bin/sh +# +# Set up a fusefile as a disk device using device mapper. +# Note that this requires root access. + +if [ $(id -u) != 0 ] ; then + echo "block device set up requires root." >&2 + exit 1 +fi + +# fuse blkdev mounting needs to sniff an existing but unmounted block +# device node for setup. However the device mapping has an empty table +# and the content is only accessible via the fuse mount that links it +# to the fusefile process. The device node (major:minor) are still +# considered in use by the kernel and, and the device node is "open" +# while mounted. + +[ -e /dev/mapper/control ] || modprobe dm_mod || exit 1 + +# Create up to N fusedisk named as fusedisk0..fusediskN, the device +# mapper also creates its dm-X device nodes and we also force +# /dev/mapper/$NAME nodes for them. +N=15 +DEV= +for I in $(seq 0 $N) ; do + NAME=fusedisk$I + C="$(dmsetup info --noheadings -c -o open $NAME 2>/dev/null)" + if [ "$C" != "1" ] ; then + if [ -z "$C" ] ; then + dmsetup create $NAME --notable || exit 1 + dmsetup mknodes $NAME || exit 1 + fi + DEV=/dev/mapper/$NAME + break + fi +done +if [ -z "$DEV" ] ; then + echo "** No more fusedisk devices" >&2 + exit 1 +fi +echo "using $DEV for $*" | logger -t fusedisk +exec fusefile -oblkdev,fsname=$DEV -oallow_other $* diff --git a/fusefile.8 b/fusefile.8 index b4bd50d..6af36f4 100644 --- a/fusefile.8 +++ b/fusefile.8 @@ -1,19 +1,54 @@ .mso www.tmac .TH fusefile 8 .SH NAME -fusefile \- FUSE file mount for combining file fragments +fusefile, fusedisk \- FUSE file mount for combining file fragments .SH SYNOPSIS -.B fusefile \fR[fuse options\fR] \fBmountpoint\fR \fIfilename/from-to\fR ... +.B fusefile \fR[\fIfuse-opts\fR] \fBmountpoint\fR \fR[\fIoverlay\fR] \fIfilename/from-to\fR ... +.br +.B fusefile \fB-dump\fR \fR[\fIfuse-opts\fR] \fBmountpoint\fR \fR[\fIoverlay\fR] \fIfilename/from-to\fR ... +.br +.B fusedisk \fR[\fIfuse-opts\fR] \fBmountpoint\fR \fR[\fIoverlay\fR] \fIfilename/from-to\fR ... .SH DESCRIPTION -\fBfusefile\fR is FUSE file mount that presents a series of fragments -of other files as a contiguous concatenation. It bind mounts a driver -on top of the file mountpoint to present the nominated file fragments -as a single, contiguous file. It accepts over-writing on the fused -file which gets distributed accordingly to the fragments, but cannot -change size. +\fBfusefile\fR is a FUSE \fIfile mount\fR that presents a series of +fragments of other files as a contiguous concatenation. Technically it +bind mounts a driver on top of the filename mountpoint to provide +access to the given file fragments as if in a single, contiguous file. + +\fBfusefile\fR accepts over-writing on the fused file (i.e. the +mountpoint) which gets distributed accordingly to the fragments. But +neither the fused file nor the fragments can change size; any writing +thus merely over-writes content without truncating fragments. All +fragment files are held open while \fBfusefile\fR is active. + +By using the optional \fB-overlay:\fIfilename\fR argument between the +mount point and the fragments, an overlay file may be set up. The +overlay file will then be used by \fBfusefile\fR for capturing writes +to the fused file (i.e. the mountpoint). The overlay file will contain +any new written fused file regions followed by meta data to +distinguish between new, written content and old content that comes +from the fragments. + +The option \fB-dump\fR as first argument together with a fusefile +setup will print the setup to standard output rather than establishing +a fusefile mount. This is of most use with a prior overlay setup, +where then the printout includes the portions of updates that have +been captured in the overlay. The printout is the series of fusefile +fragment argments to give in order to intersperse the captured overlay +portions according to the overlay table. + +\fBfusedisk\fR is a helper script to set up a \fBfusefile\fR as a +block device (via \fIfuseblk\fR) by using the device mapper +(\fBdmsetup\fR) to manage an empty block device mapping where content +is handled at the mountpoint via \fBfusefile\fR. (Note that the same +thing may be done with the device manager directly, but then all +fragments need to be in sectors of N*512 bytes whereas with +\fBfusedisk\fR, only the fused file as a whole is "clipped" at nearest +N*512 bytes below actual size) + +.SH FRAGMENT ARGUMENTS The fragment arguments include the filename of a source file, and optionally start and end byte positions. All in all there five @@ -39,46 +74,153 @@ relative to the end of the file. If "length" is negative or omitted it means that position relative to the end. .TP -\fIfilename/start\fR -include bytes from the given start. This is the same as "/start+" +\fIfilename/start\fR include bytes from the given start. This is the +same as "/start+" .P Note that a negative start position is clipped to 0 and a too large -end position is clipped to the end of the file. +end position is clipped to the end of the file. .P -Charater devices are treated as being of any given finite size, but +Character devices are treated as being of any given finite size, but have size 0 by default. For example, "/dev/zero/:100" means a fragment of 100 NUL bytes. +.SH OPTIONS + +This section enumerates the most interesting options to use with +\fBfuesfile\fR. See "man fuse" and "man mount" for more options. + +.TP +\fB-dump\fR + +The \fB-dump\fR "option" tells \fBfusefile\fR to print out the +applicable fragment sequence for the current setup, including the +overlay table, if any. The printout is done instead of setting up a +mount point. + +.TP +\fB-o\fIallow_other\fB + +The fuse option \fI-oallow_other\fR is needed for sharing the fused +file with other users who otherwise will not have access to it +(including "root"). Note however that this must first be enabled in +\fI/etc/fuse.conf\fR. + +.TP +\fB-o\fInonempty\fR + +The fuse option \fI-ononempty\fR may need to be used when reusing an +existing file as mountpoint. + +.TP +\fB-o\fIuid=...\fR and \fB-o\fIgid=...\fR, + +These mount options, where \fI...\fR is a user or group id +respectively, are useful for root when using \fBfusedisk\fR and +thereby give user or group ownership for the mount to the nominated +user or group. + .SH EXAMPLES +This section illustrates uses of \fBfusefile\fR. -Insert file "y" into file "x" at position 1200: +.SS Exanple 1. +Insert a file "y" into a file "x" at position 1200. .RS \fB$ fusefile -ononempty x x/:1200 y x/1200:\fR .RE -The bind mount shadows the original file "x", and presents the -fused file instead. +This also shadows the original file "x" and presents the fused file +instead. +.SS Example 2. Make fused file y be a swap of the beginning and end of file "x", at -position 2442: +position 2442. .RS \fB$ fusefile y x/2442: x/:2442\fR .RE +.SS Example 3. +Replace partition 2 of an image file, \fIA\fR, with a different +file, \fIX\fR. For this example the partition table looks as follows. +.RS +\fB$ partx -oNR,START,SECTORS \fIA\fR + NR START SECTORS + 1 2048 2097152 + 2 2099200 409600 + 3 2508800 14268383 +.RE +.br +As each sector is 512 bytes the clipping points around partition 2 are +1074790400 and 1284505600 and the insertion size is 209715200 bytes. +The \fBfusefile\fR comman will therefore be as follows. +.RS +\fB$ fusefile -ononempty \fIA\fB \fIA\fB/:1074790400 \fIX\fB/:209715200 \fIA\fB/1284505600\fR +.RE +Note that the fused file shadows the original file \fIA\fR. + +.SS Example 4. +Protect raw disk image file with an overlay: +.RS +\fB$ fusefile -ononempty disk.raw -overlay:today disk.raw\fR +.RE +By that set up, the overlay file, "today", will protect the disk image +file, "disk.raw" from changes, and also override the pathname +"disk.raw" to be the fused file. + +.SS Example 5. +A fusefile mount with an \fIoverlay file\fR is writable regardless of +the fused fragments, but all updates are written to the overlay file +instead of to the fragments. + +.RS +\fB$ fusefile -ononempty \fIA\fR \fB-overlay:\fIB\fB \fIA\fR +.RE + +The overlay file, \fIB\fR in the example, contains all changes to the +shadowed original file, \fIA\fR. The overlay file contains only the +newly written regions and content is otherwise obtained from the +original file. + +To that end, the overlay file also contains a "marker table" at the +end as if appended to the fused file. That part of the file is outside +of the fused file; and it's simply an element count followed by pairs +of byte addresses that tell which regions of the fused file have been +captured into the overlay file. (The marker table is of course +maintained so that adjoining regions are collapsed) + +Thus, an overlay file may be reused to later re-establish the same +fused file with overlay as previously, to continue capturing more +changes. + +.SS Example 6. +As final example, we set up a fused block device \fIy\fR as a swap of +the beginning and end of file "x", at position 2442: +.RS +\fB$ sudo fusedisk -ouid=1000 y x/2442: x/:2442\fR +.RE +Note the use of \fBsudo\fR for becoming \fIroot\fR, which is required +for block device handling, and also the \fB-ouid=1000\fR option so as +to make the block device \fIy\fR be owned by the user with id 1000. + .SH NOTES -Note that \fBfusefile\fR opens the nominated source file or files -before bind mounting. With the fuse option \fI-ononempty\fR it will -bind over an non-empty file, which may be useful. The source file -descriptors remain open, but the source fragments are not recomputed. -If a source file changes the fused file will present the new content. -If a source is reduced in size, access will be inconsistent. +\fBfusefile\fR opens the nominated source files before any bind +mounting. With the fuse option \fI-ononempty\fR it will bind over an +non-empty file, which may be useful. The source files remain open, but +the source fragments are not recomputed. If a source file changes the +fused file will present the new content. If a source is reduced in +size, access will be inconsistent. + +If the mountpoint file doesn't exist, then \fBfusefile\fR creates it. -If the mountpoint file doesn't exist, then \fBfusefile\fR creates it, -and removes it when unmounted. +Unmounting is done with "\fBfusermount -u\fR \fImountpoint\fR" as +usual. A \fBfusedisk\fR mount is unmounted by \fIroot\fR using +\fBumount\fR. +.SH SEE ALSO +\fBfuse, fusermount, mount, dmsetup\fR .SH AUTHOR -Ralph Rönnquist +Ralph Rönnquist . + diff --git a/fusefile.c b/fusefile.c index 59dcaa5..5f7850b 100644 --- a/fusefile.c +++ b/fusefile.c @@ -1,8 +1,8 @@ /*** fusefile - overlay a file path with a concatenation of parts of - other files, read only. + other files. - Copyright (C) 2019 Ralph Ronnquist + Copyright (C) 2019- Ralph Ronnquist This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as @@ -33,6 +33,13 @@ #include #include #include +#include +#include + +struct Region { + off_t beg; + off_t end; +}; struct Source { char *filename; @@ -40,6 +47,7 @@ struct Source { ssize_t to; ssize_t start; // starting position in concatenated file int fd; + int dirty; }; static struct { @@ -53,7 +61,224 @@ static struct { time_t mtime; time_t ctime; } times; - + +/** + * Overlay + */ +static struct { + struct Source source; + struct Region *table; + size_t count; + size_t limit; +} overlay; + +static void usage(); + +/** + * Find the nearest overlay.table region below pos. Returns the index, + * or -1 if there is none, i.e. pos < overlay.table[0]. + */ +static ssize_t overlay_prior_fragment(off_t pos) { + size_t lo = 0, hi = overlay.count; + while ( lo < hi ) { + size_t m = ( lo + hi ) / 2; + if ( m == lo ) { + return overlay.table[m].beg <= pos? m : -1; + } + if ( overlay.table[m].beg <= pos ) { + lo = m; + } else { + hi = m; + } + } + return -1; +} + +/** + * Save the entry count for overlay.table as 64-bit integer + * immediately following the overlay content at the index + * corresponding to the fused file size. + */ +static void overlay_save_count() { + lseek( overlay.source.fd, overlay.source.to, SEEK_SET ); + size_t size = sizeof( overlay.count ); + char *p = (char *) &overlay.count ; + while ( size > 0 ) { + size_t n = write( overlay.source.fd, p, size ); + if ( n < 0 ) { + perror( overlay.source.filename ); + exit( 1 ); + } + size -= n; + p += n; + } + if ( overlay.source.dirty++ > 1000 ) { + fsync( overlay.source.fd ); + overlay.source.dirty = 0; + } +} + +/** + * Update the on-disk cache of overlay.table between the given + * indexes. The table is laid out immediately following the table + * count with each region saved as two 64-bit unsigned integers. + */ +static void overlay_save_table(size_t lo,size_t hi) { + char *p = (char *) &overlay.table[ lo ]; + size_t pos = overlay.source.to + sizeof( overlay.count ) + + lo * sizeof( struct Region ); + size_t size = ( hi - lo ) * sizeof( struct Region ); + if ( pos != lseek( overlay.source.fd, pos, SEEK_SET ) ) { + fprintf( stderr, "%s: seek error\n", overlay.source.filename ); + exit( 1 ); + } + while ( size > 0 ) { + size_t n = write( overlay.source.fd, p, size ); + if ( n < 0 ) { + perror( overlay.source.filename ); + exit( 1 ); + } + size -= n; + p += n; + } + if ( overlay.source.dirty++ > 1000 ) { + fsync( overlay.source.fd ); + overlay.source.dirty = 0; + } +} + +/** + * Insert a new region at index p, with previous portion [p,count] + * moved up to make space. + */ +static void overlay_insert(size_t p,off_t beg,off_t end) { + size_t bytes; + // Grow the table if needed + if ( overlay.count >= overlay.limit ) { + overlay.limit = overlay.count + 10; + bytes = overlay.limit * sizeof( struct Region ); + overlay.table = overlay.table? + realloc( overlay.table, bytes ) : malloc( bytes ); + } + bytes = ( overlay.count++ - p ) * sizeof( struct Region ); + if ( bytes ) { + memmove( (char*) &overlay.table[ p+1 ], + (char*) &overlay.table[ p ], + bytes ); + } + overlay.table[ p ].beg = beg; + overlay.table[ p ].end = end; + overlay_save_count(); +} + +/** + * Delete the region entry at p by moving the portion [p+1,count] + * down. + */ +static void overlay_delete(size_t p) { + size_t bytes = ( --overlay.count - p ) * sizeof( struct Region ); + if ( bytes ) { + memmove( (char*) &overlay.table[ p ], + (char*) &overlay.table[ p+1 ], + bytes ); + } +} + +/** + * Mark the given region as updated, i.e. written to the overlay. The + * mark region may attach to prior marked regions or be a new, + * separate region. If attaching, it causes the prior regions to + * expand and the table adjusted by deleting any regions that become + * fully contained in other regions. + */ +static void overlay_mark(off_t beg,off_t end) { +#if DEBUG + fprintf( stderr, "overlay_mark( %ld, %ld )\n", beg, end ); +#endif + int deleted = 0; + ssize_t q; + ssize_t p = overlay_prior_fragment( beg ); + // p is the nearest region below or at beg (or -1) + if ( p >= 0 && beg <= overlay.table[p].end ) { + // p overlaps mark region + if ( end <= overlay.table[p].end ) { + // region p covers mark region already +#if DEBUG + fprintf( stderr, "overlay covering ( %ld %ld )\n", + overlay.table[p].beg, overlay.table[p].end ); +#endif + return; + } + // the new mark region extends region p + overlay.table[p].end = end; + q = p+1; + while ( q < overlay.count && + overlay.table[q].beg <= overlay.table[p].end ) { + // Extended region merges with subsequent region + if ( overlay.table[p].end < overlay.table[q].end ) { + overlay.table[p].end = overlay.table[q].end; + } + overlay_delete( q ); + deleted++; + } + if ( deleted ) { + overlay_save_count(); + q = overlay.count; + } + overlay_save_table( p, q ); +#if DEBUG + fprintf( stderr, "overlay expand ( %ld %ld ) deleted %d\n", + overlay.table[p].beg, overlay.table[p].end, deleted ); +#endif + return; + } + // The prior region p does not expand into new mark region + p++; // subsequent region + if ( p >= overlay.count || end < overlay.table[p].beg ) { + // New mark region is a separate region at p + overlay_insert( p, beg, end ); +#if DEBUG + fprintf( stderr, "overlay new ( %ld %ld )\n", + overlay.table[p].beg, overlay.table[p].end ); +#endif + overlay_save_table( p, overlay.count ); + return; + } + // New marks start before and overlap with region p => change p + // and handle any subsequent regions being covered + overlay.table[p].beg = beg; + q = p+1; + if ( overlay.table[p].end < end ) { + overlay.table[p].end = end; + while ( q < overlay.count && + overlay.table[q].beg <= overlay.table[p].end ) { + if ( overlay.table[p].end < overlay.table[q].end ) { + overlay.table[p].end = overlay.table[q].end; + } + overlay_delete( q ); + deleted++; + } + if ( deleted ) { + overlay_save_count(); + q = overlay.count; + } + } + overlay_save_table( p, q ); +#if DEBUG + fprintf( stderr, "overlay before ( %ld %ld ) deleted %d\n", + overlay.table[p].beg, overlay.table[p].end, deleted ); +#endif +} + +static void setup_overlay(char *filename) { + overlay.source.filename = filename; + overlay.source.fd = open( filename, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR ); + if ( overlay.source.fd < 0 ) { + perror( filename ); + usage(); + } +} + #if DEBUG static void print_source(struct Source *p) { fprintf( stderr, "%p { %s, %ld, %ld, %ld, %d }\n", @@ -67,6 +292,87 @@ static int RANGE(int s,int n ) { return ( s == n ) && *(range+c) == 0; } +static int setup_source(struct Source *p,char *frag) { + struct stat filestat; + // Open the fragment file rw if possible, else ro + range = strrchr( frag, '/' ); // last '/' + p->filename = range? strndup( frag, range - frag ) : frag; + p->fd = open( p->filename, O_RDWR ); + int rdonly = 0; + if ( p->fd < 0 ) { + rdonly = 1; + p->fd = open( p->filename, O_RDONLY ); + } + if ( p->fd < 0 ) { + perror( p->filename ); + return 1; // Error return + } + if ( stat( p->filename, &filestat ) ) { + perror( p->filename ); + return 1; + } + if ( rdonly ) { + fprintf( stderr, "** %s opened read-only\n", p->filename ); + } + p->from = 0; + if ( S_ISBLK( filestat.st_mode ) ) { + // Block devices report size differently: + if ( ioctl( p->fd, BLKGETSIZE64, &filestat.st_size ) < 0 ) { + perror( p->filename ); + } +#if DEBUG + fprintf( stderr, "block device size = %ld\n", filestat.st_size ); +#endif + } + p->to = filestat.st_size; + // Process any range variation + if ( range && *(++range) ) { + int a,b; + if ( 0 ) { + } else if ( RANGE( sscanf( range, "%d:%d%n", &a, &b, &c ), 2 )) { + p->from = ( a < 0 )? ( p->to + a ) : a; + p->to = ( b < 0 )? ( p->to + b ) : b; + } else if ( RANGE( sscanf( range, "%d+%d%n", &a, &b, &c ), 2 )) { + p->from = ( a < 0 )? ( p->to + a ) : a; + p->to = ( ( b < 0 )? p->to : p->from ) + b; + } else if ( RANGE( sscanf( range, "%d+%n", &a, &c ), 1 )) { + p->from = ( a < 0 )? ( p->to + a ) : a; + } else if ( RANGE( sscanf( range, ":%d%n", &b, &c ), 1 )) { + p->to = ( b < 0 )? ( p->to + b ) : b; + } else if ( RANGE( sscanf( range, "%d:%n", &a, &c ), 1 )) { + p->from = ( a < 0 )? ( p->to + a ) : a; + } else if ( RANGE( sscanf( range, "%d%n", &a, &c ), 1 )) { + if ( a >= 0 ) { + p->from = a; + } else { + p->from = p->to + a; + } + } else if ( RANGE( sscanf( range, ":%n", &c), 0 ) ) { + // to end from start + } else { + fprintf( stderr, "** BAD RANGE: %s\n", frag ); + return 1; + } + } + if ( ( filestat.st_mode & S_IFMT ) == S_IFCHR ) { + filestat.st_size = p->to; // Pretend size of character device + } + if ( p->from < 0 ) { + p->from = 0; + } + if ( p->to > filestat.st_size ) { + p->to = filestat.st_size; + } + if ( p->from >= p->to || p->from >= filestat.st_size ) { + fprintf( stderr, "** BAD RANGE: %s [%ld:%ld]\n", + frag, p->from, p->to ); + return 1; + } + p->start = sources.size; // the fusefile position of fragment + sources.size += p->to - p->from; + return 0; +} + static int setup_sources(char **argv,int i,int n) { sources.array = calloc( n, sizeof( struct Source ) ); if ( sources.array == 0 ) { @@ -76,75 +382,10 @@ static int setup_sources(char **argv,int i,int n) { int j = 0; sources.size = 0; for ( ; j < n; i++, j++ ) { - struct stat filestat; struct Source *p = sources.array + j; - // Open the fragment file rw if possible, else ro - range = strrchr( argv[i], '/' ); // last '/' - p->filename = range? strndup( argv[i], range - argv[i] ) : argv[i]; - p->fd = open( p->filename, O_RDWR ); - int rdonly = 0; - if ( p->fd < 0 ) { - rdonly = 1; - p->fd = open( p->filename, O_RDONLY ); - } - if ( p->fd < 0 ) { - perror( p->filename ); - return 1; // Error return - } - if ( stat( p->filename, &filestat ) ) { - perror( p->filename ); - return 1; - } - if ( rdonly ) { - fprintf( stderr, "** %s opened read-only\n", p->filename ); - } - p->from = 0; - p->to = filestat.st_size; - // Process any range variation - if ( range && *(++range) ) { - int a,b; - if ( 0 ) { - } else if ( RANGE( sscanf( range, "%d:%d%n", &a, &b, &c ), 2 )) { - p->from = ( a < 0 )? ( p->to + a ) : a; - p->to = ( b < 0 )? ( p->to + b ) : b; - } else if ( RANGE( sscanf( range, "%d+%d%n", &a, &b, &c ), 2 )) { - p->from = ( a < 0 )? ( p->to + a ) : a; - p->to = ( ( b < 0 )? p->to : p->from ) + b; - } else if ( RANGE( sscanf( range, "%d+%n", &a, &c ), 1 )) { - p->from = ( a < 0 )? ( p->to + a ) : a; - } else if ( RANGE( sscanf( range, ":%d%n", &b, &c ), 1 )) { - p->to = ( b < 0 )? ( p->to + b ) : b; - } else if ( RANGE( sscanf( range, "%d:%n", &a, &c ), 1 )) { - p->from = ( a < 0 )? ( p->to + a ) : a; - } else if ( RANGE( sscanf( range, "%d%n", &a, &c ), 1 )) { - if ( a >= 0 ) { - p->from = a; - } else { - p->from = p->to + a; - } - } else if ( RANGE( sscanf( range, ":%n", &c), 0 ) ) { - // to end from start - } else { - fprintf( stderr, "** BAD RANGE: %s\n", argv[i] ); - return 1; - } - } - if ( ( filestat.st_mode & S_IFMT ) == S_IFCHR ) { - filestat.st_size = p->to; // Pretend size of character device - } - if ( p->from < 0 ) { - p->from = 0; - } - if ( p->to > filestat.st_size ) { - p->to = filestat.st_size; - } - if ( p->from >= p->to || p->from >= filestat.st_size ) { - fprintf( stderr, "** BAD RANGE: %s [%ld:%ld]\n", - argv[i], p->from, p->to ); + if ( setup_source( p, argv[i] ) ) { return 1; } - p->start = sources.size; // the fusefile position of fragment - sources.size += p->to - p->from; #if DEBUG print_source( p ); #endif @@ -225,6 +466,37 @@ static int find_source(off_t offset) { return lo; } +static int overlay_merge(char *buf,off_t beg,off_t end) { +#if DEBUG + fprintf( stderr, "merge %ld %ld\n", beg, end ); +#endif + // Find nearest overlay data before or at beg + ssize_t p = overlay_prior_fragment( beg ); + if ( p < 0 ) { + p = 0; + } + for ( ; p < overlay.count && overlay.table[p].beg < end; p++ ) { + if ( overlay.table[p].end < beg ) { + continue; + } + if ( overlay.table[p].beg > beg ) { + size_t delta = overlay.table[p].beg - beg; + buf += delta; + beg += delta; + } + size_t size = ( overlay.table[p].end <= end )? + ( overlay.table[p].end - beg ) : ( end - beg ); + lseek( overlay.source.fd, beg, SEEK_SET ); + while ( size > 0 ) { + size_t n = read( overlay.source.fd, buf, size ); + size -= n; + buf += n; + beg += n; // + } + } + return 0; +} + // Read bytes from in file static int fusefile_read(const char *path, char *buf, size_t size, off_t off, struct fuse_file_info *fi) @@ -259,6 +531,10 @@ static int fusefile_read(const char *path, char *buf, size_t size, if ( n > size ) { n = size; } + if ( sources.array[i].dirty ) { + fsync( sources.array[i].fd ); + sources.array[i].dirty = 0; + } #if DEBUG fprintf( stderr, " seek fd=%d to %ld\n", sources.array[i].fd, b ); #endif @@ -281,6 +557,16 @@ static int fusefile_read(const char *path, char *buf, size_t size, if ( r == 0 ) { break; } + if ( overlay.source.filename ) { + if ( overlay.source.dirty ) { + fsync( overlay.source.fd ); + overlay.source.dirty = 0; + } + int x = overlay_merge( buf + rr, off + rr, off + rr + r ); + if ( x ) { + return x; + } + } rr += r; off += r; size -= r; @@ -309,6 +595,30 @@ int fusefile_poll(const char *path, struct fuse_file_info *fi, return 0; } +static void overlay_load() { + lseek( overlay.source.fd, overlay.source.to, SEEK_SET ); + size_t x = 0; + size_t size = sizeof( overlay.count ); + if ( read( overlay.source.fd, &x, size ) != size ) { + return; + } +#if DEBUG + fprintf( stderr, "overlay: %s with %ld regions\n", + overlay.source.filename, x ); +#endif + struct Region f = { 0, 0 }; + size = sizeof( struct Region ); + while ( x-- > 0 ) { + if ( read( overlay.source.fd, &f, size ) != size ) { + fprintf( stderr, "%s: bad meta data\n", overlay.source.filename ); + exit( 1 ); + } +#if DEBUG + fprintf( stderr, "overlay region: %ld %ld\n", f.beg, f.end ); +#endif + overlay_mark( f.beg, f.end ); + } +} /** * Write a full block of data over the sources at the offset @@ -317,12 +627,16 @@ static int write_block(off_t off,const char *buf,size_t size) { #if DEBUG fprintf( stderr, "write_block( %ld, ?, %ld )\n", off, size ); #endif + if ( overlay.source.filename ) { + overlay_mark( off, off + size ); // Mark region as written + } while ( size > 0 ) { int index = find_source( off ); // index of source file if ( index < 0 ) { return -EIO; // past EOF } - struct Source *source = &sources.array[ index ]; + struct Source *source = overlay.source.filename? + &overlay.source : &sources.array[ index ]; off_t from = off - source->start + source->from; off_t max = source->to - from; if ( lseek( source->fd, from, SEEK_SET ) < 0 ) { @@ -340,6 +654,10 @@ static int write_block(off_t off,const char *buf,size_t size) { size -= n; off += n; } + if ( source->dirty++ >= 1000 ) { + fsync( source->fd ); + source->dirty = 0; + } } return 0; } @@ -403,6 +721,20 @@ static void fusefile_destroy(void *data) { } } +static void fsync_all_dirty() { + int i = 0; + for ( ; i < sources.count; i++ ) { + if ( sources.array[i].dirty ) { + fsync( sources.array[i].fd ); + sources.array[i].dirty = 0; + } + } + if ( overlay.source.filename && overlay.source.dirty ) { + fsync( overlay.source.fd ); + overlay.source.dirty = 0; + } +} + static int fusefile_flush(const char *path, struct fuse_file_info *info) { #if DEBUG fprintf( stderr, "fusefile_flush( %s )\n", path ); @@ -410,6 +742,7 @@ static int fusefile_flush(const char *path, struct fuse_file_info *info) { if ( strcmp( path, "/" ) != 0 ) { return -ENOENT; } + fsync_all_dirty(); return 0; } @@ -430,6 +763,7 @@ static int fusefile_fsync(const char *path, int x, struct fuse_file_info *fi) { if ( strcmp( path, "/" ) != 0 ) { return -ENOENT; } + fsync_all_dirty(); return 0; } @@ -459,8 +793,53 @@ void *fusefile_init(struct fuse_conn_info *fci) { return 0; } +#define ENDSOURCE( S ) ( S.start + ( S.to - S.from ) ) + +/** + * Dump the current fragmentation to stdout. + */ +static int dump_fragments() { + int oly = 0; + int src = 0; + size_t pos = 0; + while ( src < sources.count ) { + size_t x = ( oly < overlay.count )? + overlay.table[ oly ].beg : sources.size; + for ( ; src < sources.count && + ENDSOURCE( sources.array[ src ] ) <= x; src++ ) { + // Dump sources.array[src] in full + fprintf( stdout, "%s/%ld:%ld\n", + sources.array[ src ].filename, + pos - sources.array[ src ].start, + sources.array[ src ].to ); + pos = ENDSOURCE( sources.array[ src ] ); + } + if ( sources.array[ src ].start < x ) { + // Dump sources.array[src] up to x; + fprintf( stdout, "%s/%ld:%ld\n", + sources.array[ src ].filename, + pos - sources.array[ src ].start, + x - sources.array[ src ].start ); + pos = ENDSOURCE( sources.array[ src ] ); + } + if ( oly < overlay.count ) { + fprintf( stdout, "%s/%ld:%ld\n", + overlay.source.filename, + overlay.table[ oly ].beg, + overlay.table[ oly ].end ); + pos = overlay.table[ oly++ ].end; + } + for ( ; src < sources.count && + ENDSOURCE( sources.array[ src ] ) <= pos; src++ ) { + // Just skip these fragments. + } + } + return( 0 ); +} + static struct fuse_operations fusefile_oper = { .getattr = fusefile_getattr, + // NYI .fgetattr = fusefile_fgetattr, .chmod = fusefile_chmod, .open = fusefile_open, .read = fusefile_read, @@ -468,9 +847,11 @@ static struct fuse_operations fusefile_oper = { .write = fusefile_write, .write_buf = fusefile_write_buf, .destroy = fusefile_destroy, + // NYI .access = fusefile_access, .flush = fusefile_flush, .release = fusefile_release, .fsync = fusefile_fsync, + // NYI .ftruncate = fusefile_ftruncate, .truncate = fusefile_truncate, //.truncate = fusefile_truncate, //.release = fusefile_release, @@ -539,9 +920,22 @@ int main(int argc, char *argv[]) } fuseargc = i; mnt = argv[ i++ ]; // First non-option argument is the mount pount + char *overlaytag = "-overlay:"; + int overlaytagsize = strlen( overlaytag ); + if ( strncmp( argv[i], overlaytag, overlaytagsize ) == 0 ) { + // consume "-overlay:filename" + setup_overlay( argv[i++] + overlaytagsize ); // Need a writable file + if ( i >= argc ) { + usage(); + } + } if ( setup_sources( argv, i, argc-i ) ) { return 1; } + if ( overlay.source.filename ) { + overlay.source.to = sources.size; // Register total size. + overlay_load(); + } if ( stat( mnt, &stbuf ) == -1 ) { int fd = open( mnt, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR ); if ( fd < 0 ) { @@ -574,6 +968,9 @@ int main(int argc, char *argv[]) } } fuseargc = setup_argv( fuseargc, &argv ); + if ( strcmp( "-dump", argv[ 1 ] ) == 0 ) { + return dump_fragments(); + } struct fuse_args args = FUSE_ARGS_INIT( fuseargc, argv ); if ( fuse_parse_cmdline( &args, &mnt, &mt, &fg ) ) { return 1; diff --git a/overlaytest.lsp b/overlaytest.lsp new file mode 100755 index 0000000..f9aa9ae --- /dev/null +++ b/overlaytest.lsp @@ -0,0 +1,112 @@ +#!/usr/bin/newlisp +# +# This is a test script for the overlay function of fusefile. +# +# 1) prepare a base image +# 2) set up a fusefile overlay +# 3) run tests +# 4) dismantle the fusefile overlay +# 5) remove test images + +; ID is hour, minute and second values packed into a string +(constant + 'ID (apply string (3 3 (now))) + 'BASE (format "%s.raw" ID) + 'OLY (format "%s.oly" ID) + 'SEGSZ 17000 + 'SEGN 40 + ) + +(constant + 'LIBC6 "/lib/x86_64-linux-gnu/libc.so.6" + 'MINE "mine" + ) + +(import LIBC6 "on_exit" "int" "void*" "void*") + +;; Set up a fusefile +(define (onexit x y) + (write-line 2 (string "terminating: " x " " (get-string y))) + (! (format "fusermount -u %s" BASE)) + (delete-file OLY) + (delete-file BASE) + ) +## note: BASE is set up as a holes file with SEGN segments of size SEGSZ +(! (format "dd if=/dev/zero of=%s bs=%d seek=%d count=0 status=none" + BASE SEGSZ SEGN)) +(unless (= (! (format "fusefile %s %s -overlay:%s %s" + "-ononempty -oallow_other" BASE OLY BASE))) + (exit 1)) +(on_exit (callback 'onexit "void" "int" "void*") MINE) + +(println (list BASE OLY)) + +(define (die) (write-line 2 (apply string (args)))) + +(define (prog1 x) x) + +(define (pos X (OFF 0)) + (+ (* SEGSZ X) OFF)) + +(define (read-segment FILE X (OFF 0) (N SEGSZ)) + (let ((FD (open FILE "r")) (BUFFER "")) + (seek FD (pos X OFF)) + (prog1 (when (= N (read FD BUFFER N)) BUFFER) + (close FD)))) + +(define (write-segment FILE X DATA (OFF 0)) + (let ((FD (open FILE "u"))) + (seek FD (pos X OFF)) + (write FD DATA) + ;(seek FD -1) + (close FD))) + +(define (read-ulong FD) + (let ((BUFFER "")) + (when (= 8 (read FD BUFFER 8)) ((unpack "ld" BUFFER) 0)))) + +(define (read-table) + (let ((AT (file-info BASE 0)) (FD (open OLY "r")) (COUNT 0) (OUT '())) + (seek FD AT) + (unless (setf COUNT (read-ulong FD)) + (write-line 2 "** Bad count") + (exit 1)) + (push COUNT OUT -1) + (dotimes (i COUNT) + (push (list (read-ulong FD) (read-ulong FD)) OUT -1)) + OUT)) + +(define (check-segment AT DATA (OFF 0)) + (write-segment BASE AT DATA OFF) + (println + (format "check %2d %d: %s %s %s" AT + (length DATA) + (if (= (read-segment BASE AT OFF (length DATA)) DATA) "ok" "error") + (if (= (read-segment OLY AT OFF (length DATA)) DATA) "ok" "error") + (string (read-table)))) + ) + +;; Test 1 +(seed (date-value)) +(setf + DATA (pack (dup "b" SEGSZ) (rand 256 SEGSZ)) + DATB (pack (dup "b" (* 4 SEGSZ)) (rand 256 (* 4 SEGSZ))) + AT (- SEGN 4)) +(check-segment 0 DATA 0) + +(check-segment AT DATA) +(check-segment (+ AT 2) DATA) +(check-segment (+ AT 1) DATA) +(check-segment (- AT 1) DATA -10) +(check-segment (- AT 1) DATA 10) + +(check-segment 0 DATA 0) +(check-segment 1 DATA 1) +(check-segment 2 DATA 2) +(check-segment 0 DATB 10) + +(check-segment (- SEGN 1) DATA 0) + +;(setf DATA (pack (dup "b" SEGSZ) (rand 256 SEGSZ)) AT (- SEGN 4)) + +(exit 0)