diff options
| -rw-r--r-- | Makefile | 1 | ||||
| -rw-r--r-- | README | 26 | ||||
| -rw-r--r-- | TODO | 2 | ||||
| -rw-r--r-- | pgfuse.c | 148 | ||||
| -rw-r--r-- | test.sql | 7 |
5 files changed, 164 insertions, 20 deletions
@@ -23,6 +23,7 @@ test: pgfuse -mkdir mnt/dir/dir2 -echo "hello" > mnt/dir/dir2/afile -cp Makefile mnt/dir/dir2/bfile + -cat mnt/dir/dir2/afile -ls -al mnt -ls -al mnt/dir/dir2 fusermount -u mnt @@ -7,26 +7,32 @@ drop of efficiency and incremental backups for instance). :-) Nevertheless, there are special situations, where a filesystem in a database is usefull. With FUSE this is also rather simple to write. -The reason I wrote one was a projects with lots of data on a ReiserFS, which -was more or less immutable and should be efficiently stored away (almost in -an archive mode). Backup is no issue there (as the files are more or less -static after an initial load), but space efficiency is an issue. +The reason I wrote one was a project with lots of data on a ReiserFS (at +least in 2001, this was), which was more or less immutable and should be +efficiently stored away (almost in an archive mode). Backup is no issue +here (as the files are more or less static after an initial load), +but space efficiency is an issue. Most other projects try to map an existing database structure somehow as -files. This here should strictly deal with files/dirs as the only model. +files. This here should strictly deal withs files/dirs as the only +available model. Other projects -------------- -Pgfs: store a filesystem in Postgres, expose it as NFS server, rather old - and I was unable to find the sources +Pgfs: stores a filesystem in Postgres, exposes it as a NFS server, rather old + and I was unable to find the sources. mysqlfs: the blueprint I used and got inspired from. +curlftpfs: blueprint for some debugging code. + +zip-fs: blueprint for handling files in memory (in my current implementation, + this should be changed of course) + References ---------- +http://www.postgresql.org/docs/ +http://fuse.sourceforge.net/ http://www.cs.hmc.edu/~geoff/classes/hmc.cs135.201109/homework/fuse/fuse_doc.html -http://www.postgresql.org/docs/8.3/static/libpq-example.html -http://www.postgresql.org/docs/8.0/static/libpq-example.html -http://fuse.sourceforge.net/doxygen/structfuse__operations.html#dc6dc71274f185de72217e38d62142c4 @@ -10,4 +10,6 @@ file, or sequential read) efficient together with random access? db securityy on blobs? Use COPY TO/COPY FROM? +- establish self-containment (with respect to + a temporarily unavailable Postgresql server) @@ -324,9 +324,7 @@ static int pgfuse_create( const char *path, mode_t mode, struct fuse_file_info * if( data->verbose ) { syslog( LOG_DEBUG, "Id for new file '%s' is %d", path, id ); } - - fi->fh = id; - + if( pgfuse_files[id % MAX_NOF_OPEN_FILES].id != 0 ) { return -EMFILE; } @@ -338,27 +336,111 @@ static int pgfuse_create( const char *path, mode_t mode, struct fuse_file_info * f->buf = (char *)malloc( f->size ); if( f->buf == NULL ) { f->id = 0; - fi->fh = 0; return -ENOMEM; } f->ref_count = 1; + + fi->fh = id; free( copy_path ); return res; } +static int psql_read_buf( PGconn *conn, const int id, const char *path, char **buf, const size_t len ) +{ + int param1 = htonl( id ); + const char *values[1] = { (char *)¶m1 }; + int lengths[2] = { sizeof( param1 ) }; + int binary[2] = { 1, 1 }; + PGresult *res; + int i_data; + int read; + + res = PQexecParams( conn, "SELECT data FROM DATA WHERE id=$1::int4", + 1, NULL, values, lengths, binary, 1 ); + + if( PQresultStatus( res ) != PGRES_TUPLES_OK ) { + syslog( LOG_ERR, "Error in psql_read_buf for path '%s'", path ); + PQclear( res ); + return -EIO; + } + + if( PQntuples( res ) != 1 ) { + syslog( LOG_ERR, "Expecting exactly one data entry for path '%s' in psql_read_buf, data inconsistent!", path ); + PQclear( res ); + return -EIO; + } + + i_data = PQfnumber( res, "data" ); + read = PQgetlength( res, 0, i_data ); + memcpy( *buf, PQgetvalue( res, 0, i_data ), read ); + + PQclear( res ); + + return read; +} + static int pgfuse_open( const char *path, struct fuse_file_info *fi ) { PgFuseData *data = (PgFuseData *)fuse_get_context( )->private_data; + PgMeta meta; + int id; + PgFuseFile *f; + int res; if( data->verbose ) { char *s = flags_to_string( fi->flags ); syslog( LOG_INFO, "Open '%s' on '%s' with flags '%s'", path, data->mountpoint, s ); if( *s != '<' ) free( s ); } + + id = psql_get_meta( data->conn, path, &meta ); + if( id < 0 ) { + return id; + } + + if( data->verbose ) { + syslog( LOG_DEBUG, "Id for file '%s' to open is %d", path, id ); + } + + if( meta.isdir ) { + /* exists and is a directory, no can do */ + return -EISDIR; + } + + if( meta.size > MAX_FILE_SIZE ) { + return -EFBIG; + } - return -EACCES; + f = &pgfuse_files[id % MAX_NOF_OPEN_FILES]; + + if( f->id != 0 ) { + return -EMFILE; + } + + f->id = id; + f->size = ( ( meta.size / STANDARD_BLOCK_SIZE ) + 1 ) * STANDARD_BLOCK_SIZE; + f->used = meta.size; + f->buf = (char *)malloc( f->size ); + if( f->buf == NULL ) { + f->id = 0; + return -ENOMEM; + } + f->ref_count = 1; + + res = psql_read_buf( data->conn, id, path, &f->buf, f->used ); + if( res != f->used ) { + syslog( LOG_ERR, "Possible data corruption in file '%s', expected '%d' bytes, got '%d', on mountpoint '%s'!", + path, f->used, res, data->mountpoint ); + free( f->buf ); + f->id = 0; + return -EIO; + } + + fi->fh = id; + + return 0; } static int psql_get_parent_id( PGconn *conn, const char *path ) @@ -553,7 +635,7 @@ static int pgfuse_flush( const char *path, struct fuse_file_info *fi ) return 0; } -static int psql_write_buf( PGconn *conn, const int id, const char *path, char *buf, size_t len ) +static int psql_write_buf( PGconn *conn, const int id, const char *path, const char *buf, const size_t len ) { int param1 = htonl( id ); const char *values[2] = { (char *)¶m1, buf }; @@ -637,7 +719,7 @@ static int pgfuse_release( const char *path, struct fuse_file_info *fi ) } static int pgfuse_write( const char *path, const char *buf, size_t size, - off_t offset, struct fuse_file_info *fi ) + off_t offset, struct fuse_file_info *fi ) { PgFuseData *data = (PgFuseData *)fuse_get_context( )->private_data; PgFuseFile *f; @@ -677,6 +759,53 @@ static int pgfuse_write( const char *path, const char *buf, size_t size, return size; } +static int pgfuse_read( const char *path, char *buf, size_t size, + off_t offset, struct fuse_file_info *fi ) +{ + PgFuseData *data = (PgFuseData *)fuse_get_context( )->private_data; + PgFuseFile *f; + + if( data->verbose ) { + syslog( LOG_INFO, "Read to '%s' from offset %d, size %d on '%s'", + path, (unsigned int)offset, (unsigned int)size, data->mountpoint ); + } + + if( fi->fh == 0 ) { + return -EBADF; + } + + f = &pgfuse_files[fi->fh % MAX_NOF_OPEN_FILES]; + + if( offset + size > f->used ) { + size = f->used - offset; + } + + memcpy( buf, f->buf+offset, size ); + + return size; +} + +int pgfuse_ftruncate( const char *path, off_t offset, struct fuse_file_info *fi ) +{ + PgFuseData *data = (PgFuseData *)fuse_get_context( )->private_data; + PgFuseFile *f; + + if( data->verbose ) { + syslog( LOG_INFO, "Truncate of '%s' to size '%d' on '%s'", + path, (unsigned int)offset, data->mountpoint ); + } + + if( fi->fh == 0 ) { + return -EBADF; + } + + f = &pgfuse_files[fi->fh % MAX_NOF_OPEN_FILES]; + + f->used = offset; + + return 0; +} + int pgfuse_utimens( const char *path, const struct timespec tv[2] ) { /* TODO: write tv to 'inode' as atime and mtime */ @@ -697,7 +826,7 @@ static struct fuse_operations pgfuse_oper = { .chown = NULL, .utime = NULL, .open = pgfuse_open, - .read = NULL, + .read = pgfuse_read, .write = pgfuse_write, .statfs = NULL, .flush = pgfuse_flush, @@ -714,7 +843,8 @@ static struct fuse_operations pgfuse_oper = { .destroy = pgfuse_destroy, .access = pgfuse_access, .create = pgfuse_create, - .ftruncate = NULL, + .truncate = NULL, + .ftruncate = pgfuse_ftruncate, .fgetattr = NULL, .lock = NULL, .utimens = pgfuse_utimens, @@ -1,4 +1,5 @@ DROP RULE dir_insert ON dir; +DROP RULE dir_remove ON dir; DROP TABLE data; DROP TABLE dir; @@ -31,7 +32,11 @@ CREATE OR REPLACE RULE "dir_insert" AS ON INSERT TO dir WHERE NEW.isdir = false DO ALSO INSERT INTO data( id ) VALUES ( currval( 'dir_id_seq' ) ); - + +-- garbage collect deleted file entries +CREATE OR REPLACE RULE "dir_remove" AS ON + DELETE TO dir WHERE OLD.isdir = false + DO ALSO DELETE FROM data WHERE id=OLD.id; -- self-referencing anchor for root directory INSERT INTO dir values( 0, 0, '/', '/', true ); |