From: Ralph Ronnquist <rrq@rrq.au>
Date: Sat, 12 Aug 2023 03:44:10 +0000 (+1000)
Subject: Merge branch 'wip/stacked-overlay' into preview
X-Git-Tag: p-1.1.1~5
X-Git-Url: https://git.rrq.au/?a=commitdiff_plain;h=f46816d1356171d1e9b00db4e151ff5230fbe55d;hp=-c;p=rrq%2Ffusefile.git

Merge branch 'wip/stacked-overlay' into preview
---

f46816d1356171d1e9b00db4e151ff5230fbe55d
diff --combined fusefile.8
index 194f4aa,da2b645..b952f55
--- a/fusefile.8
+++ b/fusefile.8
@@@ -8,8 -8,6 +8,8 @@@ fusefile, fusedisk \- FUSE file mount f
  .br
  .B fusefile \fB-dump\fR \fR[\fIfuse-opts\fR] \fBmountpoint\fR \fR[\fIoverlay\fR] \fIfilename/from-to\fR ...
  .br
 +.B fusefile \fB-push\fR \fR[\fIfuse-opts\fR] \fBmountpoint\fR \fR[\fIoverlay\fR] \fIfilename/from-to\fR ...
 +.br
  .B fusedisk \fR[\fIfuse-opts\fR] \fBmountpoint\fR \fR[\fIoverlay\fR] \fIfilename/from-to\fR ...
  
  .SH DESCRIPTION
@@@ -33,20 -31,19 +33,25 @@@ any new written fused file regions foll
  distinguish between new, written content and old content that comes
  from the fragments.
  
+ By instead using the \fB-overlay:\fIlist\fR argument where \fIlist\fR
+ is a colon-separated list of filenames, \fBfusefile\fR will use those
+ as an ordered stack of overlays and "inject" them as fragments on top
+ of each other.
+ 
  The option \fB-dump\fR as first argument together with a fusefile
  setup will print the setup to standard output rather than establishing
 -a fusefile mount. This is of most use with a prior overlay setup,
 -where then the printout includes the portions of updates that have
 -been captured in the overlay. The printout is the series of fusefile
 +a fusefile mount. This is of most use with a prior overlay setup where
 +then the printout includes the portions of updates that have been
 +captured in the overlay. The printout is the series of fusefile
  fragment argments to give in order to intersperse the captured overlay
  portions according to the overlay table.
  
 +The option \fB-push\fR as first argument together with a fusefile
 +setup will push the overlay into the sources (except for
 +write-protected fragments). This is only of use with a prior overlay
 +setup where then the updates that have been captured in the overlay
 +get pushed into the fragments.
 +
  \fBfusedisk\fR is a helper script to set up a \fBfusefile\fR as a
  block device (via \fIfuseblk\fR) by using the device mapper
  (\fBdmsetup\fR) to manage an empty block device mapping where content
diff --combined fusefile.c
index 69750fa,e05b90e..62431d2
--- a/fusefile.c
+++ b/fusefile.c
@@@ -41,6 -41,8 +41,8 @@@ struct Region 
      off_t end;
  };
  
+ #define REGIONKEEP(r) ((r,beg == r.end)?: 0 : 1)
+ 
  struct Source {
      char *filename;
      ssize_t from;
@@@ -50,6 -52,8 +52,8 @@@
      int dirty;
  };
  
+ #define ENDSOURCE( S ) ( S.start + ( S.to - S.from ) )
+ 
  static struct {
      struct Source *array;
      int count;
@@@ -65,12 -69,12 +69,12 @@@ static struct 
  /**
   * Overlay
   */
- static struct {
+ static struct Overlay {
      struct Source source;
      struct Region *table;
      size_t count;
      size_t limit;
- } overlay;
+ } overlay; // The current overlay.
  
  static void usage();
  
@@@ -270,13 -274,24 +274,24 @@@ static void overlay_mark(off_t beg,off_
  #endif
  }
  
- static void setup_overlay(char *filename) {
-     overlay.source.filename = filename;
-     overlay.source.fd = open( filename, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR );
-     if ( overlay.source.fd < 0 ) {
- 	perror( filename );
+ /**
+  * Capture overlay filenames for processing after source fragments.
+  */
+ static void overlay_setup(char *filenames) {
+     overlay.source.filename = filenames;
+ }
+ 
+ // Duplicate the source record data at the given index.
+ static void dup_source_item(int index) {
+     sources.count++;
+     sources.array = realloc(
+ 	sources.array, sources.count * sizeof( struct Source ) );
+     if ( sources.array == 0 ) {
+ 	fprintf( stderr, "** OOM when expanding frament table\n" );
  	usage();
      }
+     memcpy( &sources.array[ index+1 ], &sources.array[ index ],
+ 	    sizeof( struct Source ) );
  }
  
  #if DEBUG
@@@ -626,6 -641,111 +641,111 @@@ static void overlay_load() 
      }
  }
  
+ /**
+  * Inject an overlay fragment into the source table, end return the
+  * end of the injected fragment.
+  */
+ static off_t overlay_inject_from_region(off_t beg,off_t end) {
+     int index = find_source( beg );
+     if ( index < 0 ) {
+ 	fprintf( stderr, "** Injecting %s at %ld failed\n",
+ 		 overlay.source.filename, beg );
+ 	usage();
+     }
+     if ( end > ENDSOURCE( sources.array[ index ] ) ) {
+ 	end = ENDSOURCE( sources.array[ index ] );
+     }
+     struct Region frags[3] = {
+ 	{ sources.array[ index ].start, beg },
+ 	{ beg, ENDSOURCE( sources.array[ index ] ) },
+ 	{ ENDSOURCE( sources.array[ index ] ), end } };
+     ssize_t size = frags[0].end - frags[0].beg;
+     if ( size ) {
+ 	// "Duplicate" the indexed source data, copying the filename
+ 	dup_source_item( index );
+ 	sources.array[ index ].to = sources.array[ index ].from + size;
+ 	index++;
+ 	sources.array[ index ].start = beg;
+ 	sources.array[ index ].from = sources.array[ index-1 ].to;
+     }
+     size = frags[2].end	- frags[2].beg;
+     if ( size ) {
+ 	dup_source_item( index );
+ 	sources.array[ index+1 ].start = frags[2].beg;
+ 	sources.array[ index+1 ].from = sources.array[ index+1 ].to -size;
+     }
+     // Replace the [index] fragment
+     sources.array[ index ].filename = overlay.source.filename;
+     sources.array[ index ].start = beg;
+     sources.array[ index ].from = beg;
+     sources.array[ index ].to = end;
+     sources.array[ index ].fd = overlay.source.fd; //?
+     return end;
+ }
+ 
+ /**
+  * Inject the current (unopened) overlay into the source list.
+  */
+ static void overlay_inject() {
+     overlay.source.fd = open( overlay.source.filename, O_RDONLY );
+     if ( overlay.source.fd < 0 ) {
+ 	perror( overlay.source.filename );
+ 	usage();
+     }
+     
+     if ( lseek( overlay.source.fd, overlay.source.to, SEEK_SET ) < 0 ) {
+ 	perror( overlay.source.filename );
+ 	usage();
+     }
+     size_t count = 0;
+     size_t size = sizeof( overlay.count );
+     if ( read( overlay.source.fd, &count, size ) != size ) {
+ 	fprintf( stderr, "** error injecting %s\n", overlay.source.filename );
+ 	usage();
+     }
+     if ( count == 0 ) {
+ 	close( overlay.source.fd );
+ 	return;
+     }
+     size = count * sizeof( struct Region );
+     overlay.table = calloc( sizeof( struct Region ), count );
+     if ( read( overlay.source.fd, overlay.table, size ) != size ) {
+ 	fprintf( stderr, "** error injecting %s\n", overlay.source.filename );
+ 	usage();
+     }
+     size_t i;
+     for ( i = 0; i < count; i++ ) {
+ 	off_t beg = overlay.table[i].beg;
+ 	while ( beg < overlay.table[i].end ) {
+ 	    beg = overlay_inject_from_region( beg, overlay.table[i].end );
+ 	}
+     }
+     free( overlay.table );
+     overlay.table = 0;
+     close( overlay.source.fd );
+ }
+ 
+ /**
+  * Each stacked overlay file is nested into to source list
+  */
+ static void overlay_post_setup() {
+     char *end;
+     while ( ( end = strchr( overlay.source.filename, ':' ) ) ) {
+ 	*end = 0; //  
+ 	overlay_inject();
+ 	overlay.source.filename = end + 1;
+     }
+     if ( *overlay.source.filename ) {
+ 	overlay.source.fd = open( overlay.source.filename,
+ 				  O_RDWR | O_CREAT, S_IRUSR | S_IWUSR );
+ 	if ( overlay.source.fd < 0 ) {
+ 	    perror( overlay.source.filename );
+ 	    usage();
+ 	}
+ 	overlay_load();
+     }
+ }
+ 
  /**
   * Write a full block of data over the sources at the offset
   */
@@@ -717,40 -837,6 +837,40 @@@ static int fusefile_write(const char *p
      return size;
  }
  
 +#define PUSHBUF 104857600
 +/**
 + * Write data from overlay to source.
 + */
 +static void push_oly(off_t beg, off_t end) {
 +    static char * buffer = 0;
 +    // Pretend that there isn't an overlay
 +    char *filename = overlay.source.filename;
 +    if ( buffer == 0 ) {
 +	buffer = malloc( PUSHBUF );
 +	if ( buffer == 0 ) {
 +	    fprintf( stderr, "** OOM!!\n" );
 +	    exit( 1 );
 +	}
 +    }
 +    overlay.source.filename = 0;
 +    while ( beg < end ) {
 +	off_t size = end - beg;
 +	if ( size > PUSHBUF ) {
 +	    size = PUSHBUF;
 +	}
 +	if ( lseek( overlay.source.fd, beg, SEEK_SET ) < 0 ) {
 +	    fprintf( stderr, "** Cannot seek overlay at %ld\n", beg );
 +	    break;
 +	}
 +	size = read( overlay.source.fd, buffer, size );
 +	if ( write_block( beg, buffer, size ) < 0 ) {
 +	    fprintf( stderr, "** Cannot push %ld bytes at %ld\n", size, beg );
 +	}
 +	beg += size;
 +    }
 +    overlay.source.filename = filename;
 +}
 +
  static void fusefile_destroy(void *data) {
      char *mnt = (char*) data; // As passed to fuse_main
  #if DEBUG
@@@ -833,12 -919,10 +953,10 @@@ void *fusefile_init(struct fuse_conn_in
      return 0;
  }
  
- #define ENDSOURCE( S ) ( S.start + ( S.to - S.from ) )
- 
  /**
   * Dump the current fragmentation to stdout.
   */
 -static int dump_fragments() {
 +static int dump_fragments(int push) {
      int oly = 0;
      int src = 0;
      size_t pos = 0;
@@@ -848,33 -932,25 +966,33 @@@
  	for ( ; src < sources.count && 
  		  ENDSOURCE( sources.array[ src ] ) <= x; src++ ) {
  	    // Dump sources.array[src] in full
 -	    fprintf( stdout, "%s/%ld:%ld\n",
 -		     sources.array[ src ].filename,
 -		     pos - sources.array[ src ].start,
 -		     sources.array[ src ].to );
 +	    if ( !push ) {
 +		fprintf( stdout, "%s/%ld:%ld\n",
 +			 sources.array[ src ].filename,
 +			 pos - sources.array[ src ].start,
 +			 sources.array[ src ].to );
 +	    }
  	    pos = ENDSOURCE( sources.array[ src ] );
  	}
  	if ( ( src < sources.count ) && ( sources.array[ src ].start < x ) ) {
  	    // Dump sources.array[src] up to x;
 -	    fprintf( stdout, "%s/%ld:%ld\n",
 -		     sources.array[ src ].filename,
 -		     pos - sources.array[ src ].start,
 -		     x - sources.array[ src ].start );
 +	    if ( !push ) {
 +		fprintf( stdout, "%s/%ld:%ld\n",
 +			 sources.array[ src ].filename,
 +			 pos - sources.array[ src ].start,
 +			 x - sources.array[ src ].start );
 +	    }
  	    pos = ENDSOURCE( sources.array[ src ] );
  	}
  	if ( oly < overlay.count ) {
 -	    fprintf( stdout, "%s/%ld:%ld\n",
 -		     overlay.source.filename,
 -		     overlay.table[ oly ].beg,
 -		     overlay.table[ oly ].end );
 +	    if ( !push ) {
 +		fprintf( stdout, "%s/%ld:%ld\n",
 +			 overlay.source.filename,
 +			 overlay.table[ oly ].beg,
 +			 overlay.table[ oly ].end );
 +	    } else {
 +		push_oly( overlay.table[ oly ].beg, overlay.table[ oly ].end );
 +	    }
  	    pos = overlay.table[ oly++ ].end;
  	}
  	for ( ; src < sources.count &&
@@@ -968,11 -1044,10 +1086,10 @@@ int main(int argc, char *argv[]
      }
      fuseargc = i;
      mnt = argv[ i++ ]; // First non-option argument is the mount pount
-     char *overlaytag = "-overlay:";
-     int overlaytagsize = strlen( overlaytag );
-     if ( strncmp( argv[i], overlaytag, overlaytagsize ) == 0 ) {
- 	// consume "-overlay:filename"
- 	setup_overlay( argv[i++] + overlaytagsize ); // Need a writable file
+     if ( strcmp( argv[i], "-overlay:" ) == 0 ) {
+ 	// consume "-overlay:filename[,filename]*"
+ 	// Verify file access; to overlay must be writable.
+ 	overlay_setup( argv[i++] + strlen( "-overlay:" ) );
  	if ( i >= argc ) {
  	    usage();
  	}
@@@ -982,7 -1057,7 +1099,7 @@@
      }
      if ( overlay.source.filename ) {
  	overlay.source.to = sources.size; // Register total size.
- 	overlay_load();
+ 	overlay_post_setup();
      }
      if ( stat( mnt, &stbuf ) == -1 ) {
  	int fd = open( mnt, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR );
@@@ -1017,10 -1092,7 +1134,10 @@@
      }
      fuseargc = setup_argv( fuseargc, &argv );
      if ( strcmp( "-dump", argv[ 1 ] ) == 0 ) {
 -	return dump_fragments();
 +	return dump_fragments( 0 );
 +    }
 +    if ( strcmp( "-push", argv[ 1 ] ) == 0 ) {
 +	return dump_fragments( 1 );
      }
      struct fuse_args args = FUSE_ARGS_INIT( fuseargc, argv );
      if ( fuse_parse_cmdline( &args, &mnt, &mt, &fg ) ) {