X-Git-Url: https://git.rrq.au/?a=blobdiff_plain;f=fusefile.c;h=30ec7fa284c7ce1f330daaede1de2fb1835ee9b5;hb=6c93ae8d4f1955020adedfa67a43478dba854ab0;hp=476ca9d8fb1ffa847e094150b44223a816ed7603;hpb=a83a2cd9258ecb0420a019bb5d4cd32db8a444ac;p=rrq%2Ffusefile.git diff --git a/fusefile.c b/fusefile.c index 476ca9d..30ec7fa 100644 --- a/fusefile.c +++ b/fusefile.c @@ -34,12 +34,18 @@ #include #include +struct Region { + off_t pos; + size_t size; +}; + struct Source { char *filename; ssize_t from; ssize_t to; ssize_t start; // starting position in concatenated file int fd; + int dirty; }; static struct { @@ -53,57 +59,193 @@ static struct { time_t mtime; time_t ctime; } times; - -#if DEBUG -static void print_source(struct Source *p) { - fprintf( stderr, "%p { %s, %ld, %ld, %ld, %d }\n", - p, p->filename, p->from, p->to, p->start, p-> fd ); + +/** + * Overlay + */ +static struct { + struct Source source; + struct Region *table; + size_t count; + size_t limit; +} overlay; + +static void usage(); + +#define FRAG(m) (overlay.table+m) +#define BEG(m) (FRAG(m)->pos) +#define END(m) (FRAG(m)->pos + FRAG(m)->size) + +static ssize_t overlay_prior_fragment(off_t pos) { + size_t lo = 0, hi = overlay.count; + while ( lo < hi ) { + size_t m = ( lo + hi ) / 2; + if ( m == lo ) { + return BEG( m ) < pos? m : -1; + } + if ( BEG( m ) <= pos ) { + lo = m; + } else { + hi = m; + } + } + return -1; } -#endif -// Scan the source specification, and return the length of the -// inclusion. "filename/from,to" -// filename -// filename/from -// filename/-to -// filename/from-to -static size_t scan_source(char *in,struct Source *p) { - int e = strlen( in ); - int i = e-1; - int s = -1; - int m = -1; - // scan for last '/' and last '-' - for ( ; i >= 0; i-- ) { - if ( in[i] == '/' ) { - s = i; - break; +static void overlay_save_count() { + lseek( overlay.source.fd, overlay.source.to, SEEK_SET ); + size_t size = sizeof( overlay.count ); + char *p = (char *) &overlay.count ; + while ( size > 0 ) { + size_t n = write( overlay.source.fd, p, size ); + if ( n < 0 ) { + perror( overlay.source.filename ); + exit( 1 ); } - if ( in[i] == '-' ) { - m = i; + size -= n; + p += n; + } + if ( overlay.source.dirty++ > 1000 ) { + fsync( overlay.source.fd ); + overlay.source.dirty = 0; + } +} + +static void overlay_save_table(size_t lo,size_t hi) { + char *p = (char *) FRAG(lo); + size_t pos = overlay.source.to + sizeof( overlay.count ) + + lo * sizeof( struct Region ); + size_t size = ( hi - lo ) * sizeof( struct Region ); + if ( pos != lseek( overlay.source.fd, pos, SEEK_SET ) ) { + fprintf( stderr, "%s: seek error\n", overlay.source.filename ); + exit( 1 ); + } + while ( size > 0 ) { + size_t n = write( overlay.source.fd, p, size ); + if ( n < 0 ) { + perror( overlay.source.filename ); + exit( 1 ); } + size -= n; + p += n; + } + if ( overlay.source.dirty++ > 1000 ) { + fsync( overlay.source.fd ); + overlay.source.dirty = 0; + } +} + +static void overlay_insert(size_t p,off_t pos,size_t size) { + size_t bytes; + if ( overlay.count >= overlay.limit ) { + overlay.limit = overlay.count + 10; + bytes = overlay.limit * sizeof( struct Region ); + overlay.table = overlay.table? + realloc( overlay.table, bytes ) : malloc( bytes ); + } + bytes = ( overlay.count++ - p ) * sizeof( struct Region ); + if ( bytes ) { + memmove( FRAG( p+1 ), FRAG( p ), bytes ); } + FRAG( p )->pos = pos; + FRAG( p )->size = size; + overlay_save_count(); +} + +static void overlay_delete(size_t p) { + if ( p < --overlay.count ) { + size_t size = ( overlay.count - p ) * sizeof( struct Region ); + memmove( FRAG(p), FRAG(p+1), size ); + } + overlay_save_count(); +} + +static void overlay_mark(off_t pos,size_t size) { #if DEBUG - fprintf( stderr, "m=%d s=%d\n", m, s ); + fprintf( stderr, "overlay_mark( %ld, %ld )\n", pos, size ); #endif - // Copy the filename, and set from and to - p->filename = strndup( in, ( s < 0 )? e : s ); - struct stat buf; - if ( stat( p->filename, & buf ) ) { - perror( p->filename ); - return 1; + int deleted = 0; + ssize_t q; + ssize_t p = overlay_prior_fragment( pos ); + // p is the nearest region below pos (or -1) + if ( p >= 0 && pos <= END(p) ) { + // p overlaps mark region + if ( END(p) >= pos + size ) { +#if DEBUG + fprintf( stderr, "overlay size 1( %ld )\n", FRAG(p)->size ); +#endif + return; // new mark within existing. + } + // new mark region extends existing + FRAG(p)->size = pos + size - BEG(p); + q = p+1; + while ( q < overlay.count && BEG(q) <= END(p) ) { + if ( END(q) > END(p) ) { + FRAG(p)->size = END(q) - BEG(p); + } + overlay_delete( q ); + deleted++; + } + overlay_save_table( p, deleted? overlay.count : q ); +#if DEBUG + fprintf( stderr, "overlay size 2( %ld ) deleted %d\n", + FRAG(p)->size, deleted ); +#endif + return; } - p->from = ( s < 0 )? 0 : atol( in+s+1 ); - if ( p->from < 0 ) { - p->from = 0; + // The region p does not expand into new mark region + p++; // subsequent region + if ( p >= overlay.count || BEG(p) > pos + size ) { + // New mark is separate region at p + overlay_insert( p, pos, size); +#if DEBUG + fprintf( stderr, "overlay size 4( %ld )\n", FRAG(p)->size ); +#endif + overlay_save_table( p, overlay.count ); + return; } + // New marks start before and overlap with the region + if ( BEG(p) + FRAG(p)->size < pos + size ) { + FRAG(p)->size = size; // new mark covers old region + } else { + FRAG(p)->size += BEG(p) - pos; + } + BEG(p) = pos; + q = p+1; + while ( q < overlay.count && BEG(q) <= END(p) ) { + if ( END(q) > END(p) ) { + FRAG(p)->size = END(q) - BEG(p); + } + overlay_delete( q ); + deleted++; + } + overlay_save_table( p, deleted? overlay.count : q ); #if DEBUG - fprintf( stderr, "p->from=%ld\n", p->from ); + fprintf( stderr, "overlay size 4( %ld ) deleted %d\n", + FRAG(p)->size, deleted ); #endif - p->to = ( m < 0 )? buf.st_size : atol( in+m+1 ); - if ( p->from > p->to || p->to > buf.st_size ) { - return 1; +} + +static void setup_overlay(char *filename) { + overlay.source.filename = filename; + overlay.source.fd = open( filename, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR ); + if ( overlay.source.fd < 0 ) { + perror( filename ); + usage(); } - return 0; +} + +#if DEBUG +static void print_source(struct Source *p) { + fprintf( stderr, "%p { %s, %ld, %ld, %ld, %d }\n", + p, p->filename, p->from, p->to, p->start, p-> fd ); +} +#endif + +static char *range; +static unsigned int c; +static int RANGE(int s,int n ) { + return ( s == n ) && *(range+c) == 0; } static int setup_sources(char **argv,int i,int n) { @@ -115,22 +257,75 @@ static int setup_sources(char **argv,int i,int n) { int j = 0; sources.size = 0; for ( ; j < n; i++, j++ ) { + struct stat filestat; struct Source *p = sources.array + j; - if ( scan_source( argv[i], p ) ) { - // should free everything malloc-ed - return 1; - } - p->start = sources.size; - sources.size += p->to - p->from; + // Open the fragment file rw if possible, else ro + range = strrchr( argv[i], '/' ); // last '/' + p->filename = range? strndup( argv[i], range - argv[i] ) : argv[i]; p->fd = open( p->filename, O_RDWR ); + int rdonly = 0; if ( p->fd < 0 ) { - fprintf( stderr, "** %s opened read-only\n", p->filename ); + rdonly = 1; p->fd = open( p->filename, O_RDONLY ); } if ( p->fd < 0 ) { perror( p->filename ); + return 1; // Error return + } + if ( stat( p->filename, &filestat ) ) { + perror( p->filename ); + return 1; + } + if ( rdonly ) { + fprintf( stderr, "** %s opened read-only\n", p->filename ); + } + p->from = 0; + p->to = filestat.st_size; + // Process any range variation + if ( range && *(++range) ) { + int a,b; + if ( 0 ) { + } else if ( RANGE( sscanf( range, "%d:%d%n", &a, &b, &c ), 2 )) { + p->from = ( a < 0 )? ( p->to + a ) : a; + p->to = ( b < 0 )? ( p->to + b ) : b; + } else if ( RANGE( sscanf( range, "%d+%d%n", &a, &b, &c ), 2 )) { + p->from = ( a < 0 )? ( p->to + a ) : a; + p->to = ( ( b < 0 )? p->to : p->from ) + b; + } else if ( RANGE( sscanf( range, "%d+%n", &a, &c ), 1 )) { + p->from = ( a < 0 )? ( p->to + a ) : a; + } else if ( RANGE( sscanf( range, ":%d%n", &b, &c ), 1 )) { + p->to = ( b < 0 )? ( p->to + b ) : b; + } else if ( RANGE( sscanf( range, "%d:%n", &a, &c ), 1 )) { + p->from = ( a < 0 )? ( p->to + a ) : a; + } else if ( RANGE( sscanf( range, "%d%n", &a, &c ), 1 )) { + if ( a >= 0 ) { + p->from = a; + } else { + p->from = p->to + a; + } + } else if ( RANGE( sscanf( range, ":%n", &c), 0 ) ) { + // to end from start + } else { + fprintf( stderr, "** BAD RANGE: %s\n", argv[i] ); + return 1; + } + } + if ( ( filestat.st_mode & S_IFMT ) == S_IFCHR ) { + filestat.st_size = p->to; // Pretend size of character device + } + if ( p->from < 0 ) { + p->from = 0; + } + if ( p->to > filestat.st_size ) { + p->to = filestat.st_size; + } + if ( p->from >= p->to || p->from >= filestat.st_size ) { + fprintf( stderr, "** BAD RANGE: %s [%ld:%ld]\n", + argv[i], p->from, p->to ); return 1; } + p->start = sources.size; // the fusefile position of fragment + sources.size += p->to - p->from; #if DEBUG print_source( p ); #endif @@ -186,17 +381,71 @@ static int find_source(off_t offset) { if ( offset >= sources.size ) { return -1; } +#if DEBUG + fprintf( stderr, "find_source( %ld )\n", offset ); +#endif while ( lo + 1 < hi ) { int m = ( lo + hi ) / 2; if ( offset < sources.array[ m ].start ) { +#if DEBUG + fprintf( stderr, " offset < [%d].start: %ld\n", + m, sources.array[ m ].start ); +#endif hi = m; } else { +#if DEBUG + fprintf( stderr, " offset >= [%d].start: %ld\n", + m, sources.array[ m ].start ); +#endif lo = m; } } +#if DEBUG + fprintf( stderr, "found %d\n", lo ); +#endif return lo; } +static int overlay_merge(char *buf,off_t off,size_t size) { +#if DEBUG + fprintf( stderr, "merge %ld %ld\n", off, size ); +#endif + // Find nearest overlay data before or at off + ssize_t p = overlay_prior_fragment( off ); + if ( p < 0 ) { + p = 0; + } + for ( ; p < overlay.count && BEG(p) < off+size; p++ ) { + if ( END(p) < off ) { + continue; + } + size_t delta = FRAG(p)->size; + if ( BEG(p) < off ) { + delta -= off - BEG(p); + } else { + size_t skip = BEG(p) - off; + off += skip; + size -= skip; + buf += skip; + } + if ( delta > size ) { + delta = size; + } + lseek( overlay.source.fd, off, SEEK_SET ); + while ( delta > 0 ) { + size_t n = read( overlay.source.fd, buf, delta ); + off += n; + size -= n; + delta -= n; + buf += n; + } + } +#if DEBUG + fprintf( stderr, "merged\n" ); +#endif + return 0; +} + // Read bytes from in file static int fusefile_read(const char *path, char *buf, size_t size, off_t off, struct fuse_file_info *fi) @@ -210,14 +459,14 @@ static int fusefile_read(const char *path, char *buf, size_t size, #if DEBUG fprintf( stderr, "read %ld %ld\n", off, size ); #endif - size_t rr = 0; + size_t rr = 0; // total reading while ( size > 0 ) { #if DEBUG - fprintf( stderr, "find_source %ld %ld\n", off, size ); + fprintf( stderr, " find_source %ld %ld\n", off, size ); #endif int i = find_source( off ); if ( i < 0 ) { - return ( off == sources.size )? 0 : -ENOENT; + return ( off == sources.size )? rr : -ENOENT; } if ( sources.array[i].fd < 0 ) { return -ENOENT; @@ -225,21 +474,30 @@ static int fusefile_read(const char *path, char *buf, size_t size, #if DEBUG print_source( &sources.array[i] ); #endif + times.atime = time( 0 ); size_t b = off - sources.array[i].start + sources.array[i].from; size_t n = sources.array[i].to - b; if ( n > size ) { n = size; } + if ( sources.array[i].dirty ) { + fsync( sources.array[i].fd ); + sources.array[i].dirty = 0; + } +#if DEBUG + fprintf( stderr, " seek fd=%d to %ld\n", sources.array[i].fd, b ); +#endif if ( lseek( sources.array[i].fd, b, SEEK_SET ) < 0 ) { perror( sources.array[i].filename ); return -ENOENT; } #if DEBUG - fprintf( stderr, "get %ld bytes at %ld\n", n, rr ); + fprintf( stderr, " now read %ld from fd=%d\n", + n, sources.array[i].fd ); #endif ssize_t r = read( sources.array[i].fd, buf + rr, n ); #if DEBUG - fprintf( stderr, "got %ld bytes\n", r ); + fprintf( stderr, " got %ld bytes\n", r ); #endif if ( r < 0 ) { perror( sources.array[i].filename ); @@ -248,14 +506,69 @@ static int fusefile_read(const char *path, char *buf, size_t size, if ( r == 0 ) { break; } + if ( overlay.source.filename ) { + if ( overlay.source.dirty ) { + fsync( overlay.source.fd ); + overlay.source.dirty = 0; + } + int x = overlay_merge( buf + rr, off + rr, r ); + if ( x ) { + return x; + } + } rr += r; off += r; size -= r; } - times.atime = time( 0 ); +#if DEBUG + fprintf( stderr, " total reading %ld bytes\n", rr ); +#endif return rr; } +/** + * Poll for IO readiness. + */ +int fusefile_poll(const char *path, struct fuse_file_info *fi, + struct fuse_pollhandle *ph, unsigned *reventsp ) +{ +#if DEBUG + fprintf( stderr, "fusefile_poll( %s ) %p %d\n", path, ph, *reventsp ); +#endif + if( strcmp( path, "/" ) != 0 ) { + return -ENOENT; + } + if ( ph ) { + return fuse_notify_poll( ph ); + } + return 0; +} + +static void overlay_load() { + lseek( overlay.source.fd, overlay.source.to, SEEK_SET ); + size_t x = 0; + size_t size = sizeof( overlay.count ); + if ( read( overlay.source.fd, &x, size ) != size ) { + return; + } +#if DEBUG + fprintf( stderr, "overlay: %s with %ld regions\n", + overlay.source.filename, x ); +#endif + struct Region f = { 0, 0 }; + size = sizeof( struct Region ); + while ( x-- > 0 ) { + if ( read( overlay.source.fd, &f, size ) != size ) { + fprintf( stderr, "%s: bad meta data\n", overlay.source.filename ); + exit( 1 ); + } +#if DEBUG + fprintf( stderr, "overlay region: %ld %ld\n", f.pos, f.size ); +#endif + overlay_mark( f.pos, f.size ); + } +} + /** * Write a full block of data over the sources at the offset */ @@ -263,14 +576,18 @@ static int write_block(off_t off,const char *buf,size_t size) { #if DEBUG fprintf( stderr, "write_block( %ld, ?, %ld )\n", off, size ); #endif + if ( overlay.source.filename ) { + overlay_mark( off, size ); // Mark region as written + } while ( size > 0 ) { int index = find_source( off ); // index of source file if ( index < 0 ) { return -EIO; // past EOF } - struct Source *source = &sources.array[ index ]; - off_t from = off - source->start; - off_t max = source->to - source->from - from; + struct Source *source = overlay.source.filename? + &overlay.source : &sources.array[ index ]; + off_t from = off - source->start + source->from; + off_t max = source->to - from; if ( lseek( source->fd, from, SEEK_SET ) < 0 ) { return -EIO; } @@ -286,6 +603,10 @@ static int write_block(off_t off,const char *buf,size_t size) { size -= n; off += n; } + if ( source->dirty++ >= 1000 ) { + fsync( source->fd ); + source->dirty = 0; + } } return 0; } @@ -349,6 +670,20 @@ static void fusefile_destroy(void *data) { } } +static void fsync_all_dirty() { + int i = 0; + for ( ; i < sources.count; i++ ) { + if ( sources.array[i].dirty ) { + fsync( sources.array[i].fd ); + sources.array[i].dirty = 0; + } + } + if ( overlay.source.filename && overlay.source.dirty ) { + fsync( overlay.source.fd ); + overlay.source.dirty = 0; + } +} + static int fusefile_flush(const char *path, struct fuse_file_info *info) { #if DEBUG fprintf( stderr, "fusefile_flush( %s )\n", path ); @@ -356,6 +691,7 @@ static int fusefile_flush(const char *path, struct fuse_file_info *info) { if ( strcmp( path, "/" ) != 0 ) { return -ENOENT; } + fsync_all_dirty(); return 0; } @@ -376,6 +712,7 @@ static int fusefile_fsync(const char *path, int x, struct fuse_file_info *fi) { if ( strcmp( path, "/" ) != 0 ) { return -ENOENT; } + fsync_all_dirty(); return 0; } @@ -392,11 +729,25 @@ static int fusefile_truncate(const char *path, off_t len) { return -EIO; } +void *fusefile_init(struct fuse_conn_info *fci) { +#if DEBUG + fprintf( stderr, "fusefile_init( %d, %d )\n", fci->async_read, fci->want ); +#endif + // Disable asynchronous reading + fci->async_read = 0; + fci->want &= ~FUSE_CAP_ASYNC_READ; +#if DEBUG + fprintf( stderr, "fusefile_init( %d, %d )\n", fci->async_read, fci->want ); +#endif + return 0; +} + static struct fuse_operations fusefile_oper = { .getattr = fusefile_getattr, .chmod = fusefile_chmod, .open = fusefile_open, .read = fusefile_read, + .poll = fusefile_poll, .write = fusefile_write, .write_buf = fusefile_write_buf, .destroy = fusefile_destroy, @@ -406,13 +757,13 @@ static struct fuse_operations fusefile_oper = { .truncate = fusefile_truncate, //.truncate = fusefile_truncate, //.release = fusefile_release, - //void *(*init) (struct fuse_conn_info *conn); + .init = fusefile_init, }; static void usage() { char *usage = "Usage: fusefile [ ] ... \n" -"Mounts a virtual, read-only file that is a concatenation of file fragments\n" +"Mounts a virtual, file that is a concatenation of file fragments\n" ; fprintf( stderr, "%s", usage ); exit( 1 ); @@ -420,28 +771,31 @@ static void usage() { /** * Set up the arguments for the fuse_main call, adding our own. + * argv[argc] is the mount point argument */ static int setup_argv(int argc,char ***argv) { + // note: (*argv)[ argc ] is the mount point argument char *OURS[] = { "-odefault_permissions", - (*argv)[ --argc ] // note: (*argv)[ argc-1 ] = the mount point + (*argv)[ argc ] }; #define OURSN ( sizeof( OURS ) / sizeof( char* ) ) - int N = argc + OURSN; // new argv-tobe size, excluding null - char **out = malloc( ( N + 1 ) * sizeof( char* ) ); + int N = argc + OURSN; + // Allocate new arg array plus terminating null pointer + char **out = malloc( ( N + 1 ) * sizeof( char* ) ); int i; for ( i = 0; i < argc; i++ ) { out[ i ] = (*argv)[i]; - fprintf( stderr, " %s", out[ i ] ); + //fprintf( stderr, " %s", out[ i ] ); } for ( i = 0; i < OURSN; i++ ) { out[ argc + i ] = OURS[i]; - fprintf( stderr, " %s", out[ i ] ); + //fprintf( stderr, " %s", out[ i ] ); } out[ N ] = 0; - fprintf( stderr, "\n" ); + //fprintf( stderr, "\n" ); (*argv) = out; - return N; + return N; // Don't include the terminating null pointer } /** @@ -466,11 +820,24 @@ int main(int argc, char *argv[]) if ( i > argc - 2 ) { // At least mount point plus one source usage(); } - mnt = argv[ i++ ]; // First non-option argument is the mount pount fuseargc = i; + mnt = argv[ i++ ]; // First non-option argument is the mount pount + char *overlaytag = "-overlay:"; + int overlaytagsize = strlen( overlaytag ); + if ( strncmp( argv[i], overlaytag, overlaytagsize ) == 0 ) { + // consume "-overlay:filename" + setup_overlay( argv[i++] + overlaytagsize ); // Need a writable file + if ( i >= argc ) { + usage(); + } + } if ( setup_sources( argv, i, argc-i ) ) { return 1; } + if ( overlay.source.filename ) { + overlay.source.to = sources.size; // Register total size. + overlay_load(); + } if ( stat( mnt, &stbuf ) == -1 ) { int fd = open( mnt, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR ); if ( fd < 0 ) { @@ -492,6 +859,16 @@ int main(int argc, char *argv[]) times.ctime = stbuf.st_ctime; } + { + int fd = open( mnt, O_RDWR, S_IRUSR | S_IWUSR ); + if ( fd < 0 ) { + perror( mnt ); + return 1; + } + if ( lseek( fd, sources.size, SEEK_SET ) < 0 ) { + return -EIO; + } + } fuseargc = setup_argv( fuseargc, &argv ); struct fuse_args args = FUSE_ARGS_INIT( fuseargc, argv ); if ( fuse_parse_cmdline( &args, &mnt, &mt, &fg ) ) {