diff options
| -rw-r--r-- | DEVELOPERS | 42 | ||||
| -rw-r--r-- | Makefile | 99 | ||||
| -rw-r--r-- | inc.mak | 28 | ||||
| -rw-r--r-- | tests/Makefile | 73 | ||||
| -rw-r--r-- | tests/clean.sql (renamed from clean.sql) | 0 | ||||
| -rw-r--r-- | tests/testfsync.c (renamed from testfsync.c) | 0 | ||||
| -rw-r--r-- | tests/testpgsql.c (renamed from testpgsql.c) | 0 |
7 files changed, 132 insertions, 110 deletions
@@ -16,10 +16,14 @@ Coding guidelines 8-character TABs. Standard C89, portable to all platforms supporting FUSE (Linux, - FreeBSD, OpenBSD, NetBSD, Darwin/MacOSX). No C++, no C99. + FreeBSD, OpenBSD, NetBSD, Darwin/MacOSX). No C++, no C99. Some + newer 64-bit things like 'endian.h' are allowed, if there is no + other option. Do not introduce unnecessary 3rd party dependencies in addition to the required 'libfuse' and 'libpq'. + + Use the internal fuse_opts to parse command line options. Use native 'libpq', not abstractions. The database operations are simple enough. If possible avoid string manipulations as @@ -38,7 +42,7 @@ One ByteA field All data in a big bytea column: needs memory of the size of the complete file on client (pgfuse) and server (PostgreSQL) side, - is ok for small files (first proof-of-concept implementation) + is ok for small files (first proof-of-concept implementation). Multiple ByteA of equal size @@ -76,32 +80,36 @@ not really sure if that is good or bad. Directory tree in database -------------------------- -Naive implementation - - Complete path as string. Has high mutation costs for renames, - storage overhead. But is very fast for queries (no joins). - -TODO... +Every inode has a 'parent_id', self-referencing. To avoid the NULL +reference there is a root element point to itself. Pathes are decomposed +programatically by descending the tree. This requires no additional +redudant storage is easy to change in renames and gives acceptable +read performance. Transaction Policies -------------------- -Fundamental querstion: What file operations should form a database transaction? +Fundamental question: What file operations should form a database transaction? -one extreme: isolate threads and all file operations in one transaction +one extreme: isolate threads and all file operations in one transaction. +This is most likely an illution, as we can't assume that FUSE threads +are assigned fix to a specific file (on the contrary!). + other extreme: "autocommit" (every write, every read, etc.), this allows -for parallel usage, as long as we have ref_count and an exclusive -locking, this extreme makes no sense +for parallel usage. We trust in FUSE and the locking there. The database +should help us sequentiallize the operations. -specify transaction strategy as mount options? +Currently the second option was choosen. Self-containment ---------------- React decently to loss of database connections. Try to reestablish the connection, the loss of database connection could be temporary. +Try to reexecute the file system operation. -What should be reported back as temporary error state to FUSE? +What should be reported back as permanent error state to FUSE after +a certain timeout when the database doesn't appear again? EIO seems a good option (as if the disk would have temporary I/O problems). @@ -114,13 +122,17 @@ commands of the shell). bonnie is a good stress and performance tester. Don't despair because of poor performance, that's normal. :-) -Several shells in parallel doing: +Another option is to have many shells open and do some things in +parallel, like: while(true);do mkdir mnt/bla rmdir mnt/bla done +We should actually write a filesystem operation simulator doing all +kind of random weird operations with certain probabilities. + References ---------- @@ -13,93 +13,14 @@ execdir=$(DESTDIR)$(prefix) bindir=$(execdir)/bin datadir=$(execdir)/share -# for debugging -#CFLAGS = -Wall -Werror -g -O0 -pthread -# for releasing -CFLAGS = -Wall -O2 - -# redhat has libpq-fe.h and fuse.h in /usr/include, ok - -# suse has libpq-fe.h in -CFLAGS += -I/usr/include/pgsql - -# debianish systems have libpg-fe.h in -CFLAGS += -I/usr/include/postgresql - -# declare version of FUSE API we want to program against -CFLAGS += -DFUSE_USE_VERSION=26 - -CFLAGS += -D_FILE_OFFSET_BITS=64 - -# debug -#CFLAGS += -I/usr/local/include/fuse -#LDFLAGS = -lpq /usr/local/lib/libfuse.a -pthread -ldl -lrt - -# release -# use pkg-config to detemine compiler/linker flags for libfuse -CFLAGS += `pkg-config fuse --cflags` -LDFLAGS = `pkg-config fuse --libs` -lpq -pthread - -PG_CONNINFO = "" +include inc.mak clean: rm -f pgfuse pgfuse.o pgsql.o pool.o - rm -f testfsync testfsync.o - rm -f testpgsql testpgsql.o + cd tests && $(MAKE) clean -test: pgfuse testfsync testpgsql - psql < clean.sql - psql < schema.sql - test -d mnt || mkdir mnt - ./pgfuse -s -v "$(PG_CONNINFO)" mnt - mount | grep pgfuse - # expect success for making directories - -mkdir mnt/dir - -mkdir mnt/dir/dir2 - -mkdir mnt/dir/dir3 - # expect success on open and file write - -echo "hello" > mnt/dir/dir2/afile - -cp Makefile mnt/dir/dir2/bfile - # expect success on open and file read - -cat mnt/dir/dir2/afile - -ls -al mnt - -ls -al mnt/dir/dir2 - # expect success on rmdir - -rmdir mnt/dir/dir3 - # expect success on chmod - -chmod 777 mnt/dir/dir2/bfile - -ls -al mnt/dir/dir2/bfile - # expect success on symlink creation - -ln -s bfile mnt/dir/dir2/clink - -ls -al mnt/dir/dir2/clink - # expect success on file removal - -rm mnt/dir/dir2/bfile - # expect success on rename - -mkdir mnt/dir/dir3 - -mv mnt/dir/dir3 mnt/dir/dir4 - -mv mnt/dir/dir2/afile mnt/dir/dir4/bfile - # expect fail (directory not empty) - -rmdir mnt/dir - # expect fail (not a directory) - -rmdir mnt/dir/dir2/bfile - # test fdatasync and fsync - ./testfsync - # show times of dirs, files and symlinks - -stat mnt/dir/dir2/afile - -stat mnt/dir/dir3 - -stat mnt/dir/dir2/clink - # show filesystem stats (statvfs) - -stat -f mnt - # expect success, truncate a file (grow and shrink) - -touch mnt/trunc - -ls -al mnt/trunc - -truncate --size 2049 mnt/trunc - -ls -al mnt/trunc - -dd if=/dev/zero of=mnt/trunc bs=512 count=10 - -truncate --size 513 mnt/trunc - -ls -al mnt/trunc - # END: unmount FUSE file system - fusermount -u mnt +test: + cd tests && $(MAKE) test pgfuse: pgfuse.o pgsql.o pool.o $(CC) -o pgfuse pgfuse.o pgsql.o pool.o $(LDFLAGS) @@ -112,18 +33,6 @@ pgsql.o: pgsql.c pgsql.h config.h pool.o: pool.c pool.h $(CC) -c $(CFLAGS) -o pool.o pool.c - -testfsync: testfsync.o - $(CC) -o testfsync testfsync.o - -testfsync.o: testfsync.c - $(CC) -c $(CFLAGS) -o testfsync.o testfsync.c - -testpgsql: testpgsql.o - $(CC) -o testpgsql testpgsql.o $(LDFLAGS) - -testpgsql.o: testpgsql.c - $(CC) -c $(CFLAGS) -o testpgsql.o testpgsql.c install: all test -d "$(bindir)" || mkdir -p "$(bindir)" @@ -0,0 +1,28 @@ +CC=gcc + +# for debugging +CFLAGS = -Wall -Werror -g -O0 -pthread +# for releasing +#CFLAGS = -Wall -O2 + +# redhat has libpq-fe.h and fuse.h in /usr/include, ok + +# suse has libpq-fe.h in +CFLAGS += -I/usr/include/pgsql + +# debianish systems have libpg-fe.h in +CFLAGS += -I/usr/include/postgresql + +# declare version of FUSE API we want to program against +CFLAGS += -DFUSE_USE_VERSION=26 + +CFLAGS += -D_FILE_OFFSET_BITS=64 + +# debug +#CFLAGS += -I/usr/local/include/fuse +#LDFLAGS = -lpq /usr/local/lib/libfuse.a -pthread -ldl -lrt + +# release +# use pkg-config to detemine compiler/linker flags for libfuse +CFLAGS += `pkg-config fuse --cflags` +LDFLAGS = `pkg-config fuse --libs` -lpq -pthread diff --git a/tests/Makefile b/tests/Makefile new file mode 100644 index 0000000..d4aec77 --- /dev/null +++ b/tests/Makefile @@ -0,0 +1,73 @@ +include ../inc.mak + +PG_CONNINFO = "" + +test: testfsync testpgsql + psql < clean.sql + psql < ../schema.sql + test -d mnt || mkdir mnt + ../pgfuse -s -v "$(PG_CONNINFO)" mnt + mount | grep pgfuse + # expect success for making directories + -mkdir mnt/dir + -mkdir mnt/dir/dir2 + -mkdir mnt/dir/dir3 + # expect success on open and file write + -echo "hello" > mnt/dir/dir2/afile + -cp Makefile mnt/dir/dir2/bfile + # expect success on open and file read + -cat mnt/dir/dir2/afile + -ls -al mnt + -ls -al mnt/dir/dir2 + # expect success on rmdir + -rmdir mnt/dir/dir3 + # expect success on chmod + -chmod 777 mnt/dir/dir2/bfile + -ls -al mnt/dir/dir2/bfile + # expect success on symlink creation + -ln -s bfile mnt/dir/dir2/clink + -ls -al mnt/dir/dir2/clink + # expect success on file removal + -rm mnt/dir/dir2/bfile + # expect success on rename + -mkdir mnt/dir/dir3 + -mv mnt/dir/dir3 mnt/dir/dir4 + -mv mnt/dir/dir2/afile mnt/dir/dir4/bfile + # expect fail (directory not empty) + -rmdir mnt/dir + # expect fail (not a directory) + -rmdir mnt/dir/dir2/bfile + # test fdatasync and fsync + ./testfsync + # show times of dirs, files and symlinks + -stat mnt/dir/dir2/afile + -stat mnt/dir/dir3 + -stat mnt/dir/dir2/clink + # show filesystem stats (statvfs) + -stat -f mnt + # expect success, truncate a file (grow and shrink) + -touch mnt/trunc + -ls -al mnt/trunc + -truncate --size 2049 mnt/trunc + -ls -al mnt/trunc + -dd if=/dev/zero of=mnt/trunc bs=512 count=10 + -truncate --size 513 mnt/trunc + -ls -al mnt/trunc + # END: unmount FUSE file system + fusermount -u mnt + +clean: + rm -f testfsync testfsync.o + rm -f testpgsql testpgsql.o + +testfsync: testfsync.o + $(CC) -o testfsync testfsync.o + +testfsync.o: testfsync.c + $(CC) -c $(CFLAGS) -o testfsync.o testfsync.c + +testpgsql: testpgsql.o + $(CC) -o testpgsql testpgsql.o $(LDFLAGS) + +testpgsql.o: testpgsql.c + $(CC) -c $(CFLAGS) -o testpgsql.o testpgsql.c diff --git a/clean.sql b/tests/clean.sql index 6d94089..6d94089 100644 --- a/clean.sql +++ b/tests/clean.sql diff --git a/testfsync.c b/tests/testfsync.c index 571b816..571b816 100644 --- a/testfsync.c +++ b/tests/testfsync.c diff --git a/testpgsql.c b/tests/testpgsql.c index 3159e2d..3159e2d 100644 --- a/testpgsql.c +++ b/tests/testpgsql.c |