[svn r136] MAJOR UNSTABLE UPDATE!!!

Initial commit after moving to Tango instead of Phobos.

Lots of bugfixes...

This build is not suitable for most things.

tango/example/conduits/FileBucket.d

@@ -0,0 +1,319 @@
+module FileBucket;
+
+private import  tango.io.FilePath,
+                tango.io.FileConduit;
+
+private import  tango.core.Exception;
+
+/******************************************************************************
+
+        FileBucket implements a simple mechanism to store and recover a 
+        large quantity of data for the duration of the hosting process.
+        It is intended to act as a local-cache for a remote data-source, 
+        or as a spillover area for large in-memory cache instances. 
+        
+        Note that any and all stored data is rendered invalid the moment
+        a FileBucket object is garbage-collected.
+
+        The implementation follows a fixed-capacity record scheme, where
+        content can be rewritten in-place until said capacity is reached.
+        At such time, the altered content is moved to a larger capacity
+        record at end-of-file, and a hole remains at the prior location.
+        These holes are not collected, since the lifespan of a FileBucket
+        is limited to that of the host process.
+
+        All index keys must be unique. Writing to the FileBucket with an
+        existing key will overwrite any previous content. What follows
+        is a contrived example:
+        
+        ---
+        char[] text = "this is a test";
+
+        auto bucket = new FileBucket (new FilePath("bucket.bin"), FileBucket.HalfK);
+
+        // insert some data, and retrieve it again
+        bucket.put ("a key", text);
+        char[] b = cast(char[]) bucket.get ("a key");
+
+        assert (b == text);
+        bucket.close;
+        ---
+
+******************************************************************************/
+
+class FileBucket
+{
+        /**********************************************************************
+
+                Define the capacity (block-size) of each record
+
+        **********************************************************************/
+
+        struct BlockSize
+        {
+                int capacity;
+        }
+
+        // basic capacity for each record
+        private FilePath                path;
+
+        // basic capacity for each record
+        private BlockSize               block;
+
+        // where content is stored
+        private FileConduit             file;
+
+        // pointers to file records
+        private Record[char[]]          map;
+
+        // current file size
+        private long                    fileSize;
+
+        // current file usage
+        private long                    waterLine;
+
+        // supported block sizes
+        public static const BlockSize   EighthK  = {128-1},
+                                        HalfK    = {512-1},
+                                        OneK     = {1024*1-1},
+                                        TwoK     = {1024*2-1},
+                                        FourK    = {1024*4-1},
+                                        EightK   = {1024*8-1},
+                                        SixteenK = {1024*16-1},
+                                        ThirtyTwoK = {1024*32-1},
+                                        SixtyFourK = {1024*64-1};
+
+
+        /**********************************************************************
+
+                Construct a FileBucket with the provided path and record-
+                size. Selecting a record size that roughly matches the 
+                serialized content will limit 'thrashing'.
+
+        **********************************************************************/
+
+        this (char[] path, BlockSize block)
+        {
+                this (new FilePath(path), block);
+        }
+
+        /**********************************************************************
+
+                Construct a FileBucket with the provided path, record-size,
+                and inital record count. The latter causes records to be 
+                pre-allocated, saving a certain amount of growth activity.
+                Selecting a record size that roughly matches the serialized 
+                content will limit 'thrashing'. 
+
+        **********************************************************************/
+
+        this (FilePath path, BlockSize block, uint initialRecords = 100)
+        {
+                this.path = path;
+                this.block = block;
+
+                // open a storage file
+                file = new FileConduit (path, FileConduit.ReadWriteCreate);
+
+                // set initial file size (can be zero)
+                fileSize = initialRecords * block.capacity;
+                file.seek (fileSize);
+                file.truncate ();
+        }
+
+        /**********************************************************************
+
+                Return the block-size in use for this FileBucket
+
+        **********************************************************************/
+
+        int getBufferSize ()
+        {
+                return block.capacity+1;
+        }
+
+        /**********************************************************************
+        
+                Return where the FileBucket is located
+
+        **********************************************************************/
+
+        FilePath getFilePath ()
+        {
+                return path;
+        }
+
+        /**********************************************************************
+
+                Return the currently populated size of this FileBucket
+
+        **********************************************************************/
+
+        synchronized long length ()
+        {
+                return waterLine;
+        }
+
+        /**********************************************************************
+
+                Return the serialized data for the provided key. Returns
+                null if the key was not found.
+
+        **********************************************************************/
+
+        synchronized void[] get (char[] key)
+        {
+                Record r = null;
+
+                if (key in map)
+                   {
+                   r = map [key];
+                   return r.read (this);
+                   }
+                return null;
+        }
+
+        /**********************************************************************
+
+                Remove the provided key from this FileBucket.
+
+        **********************************************************************/
+
+        synchronized void remove (char[] key)
+        {
+                map.remove(key);
+        }
+
+        /**********************************************************************
+
+                Write a serialized block of data, and associate it with
+                the provided key. All keys must be unique, and it is the
+                responsibility of the programmer to ensure this. Reusing 
+                an existing key will overwrite previous data. 
+
+                Note that data is allowed to grow within the occupied 
+                bucket until it becomes larger than the allocated space.
+                When this happens, the data is moved to a larger bucket
+                at the file tail.
+
+        **********************************************************************/
+
+        synchronized void put (char[] key, void[] data)
+        {
+                Record* r = key in map;
+
+                if (r is null)
+                   {
+                   auto rr = new Record;
+                   map [key] =  rr;
+                   r = &rr;
+                   }
+                r.write (this, data, block);
+        }
+
+        /**********************************************************************
+
+                Close this FileBucket -- all content is lost.
+
+        **********************************************************************/
+
+        synchronized void close ()
+        {
+                if (file)
+                   {
+                   file.detach;
+                   file = null;
+                   map = null;
+                   }
+        }
+
+        /**********************************************************************
+
+                Each Record takes up a number of 'pages' within the file. 
+                The size of these pages is determined by the BlockSize 
+                provided during FileBucket construction. Additional space
+                at the end of each block is potentially wasted, but enables 
+                content to grow in size without creating a myriad of holes.
+
+        **********************************************************************/
+
+        private static class Record
+        {
+                private long            offset;
+                private int             length,
+                                        capacity = -1;
+
+                /**************************************************************
+
+                **************************************************************/
+
+                private static void eof (FileBucket bucket)
+                {
+                        throw new IOException ("Unexpected EOF in FileBucket '"~bucket.path.toString()~"'");
+                }
+
+                /**************************************************************
+
+                        This should be protected from thread-contention at
+                        a higher level.
+
+                **************************************************************/
+
+                void[] read (FileBucket bucket)
+                {
+                        void[] data = new ubyte [length];
+
+                        bucket.file.seek (offset);
+                        if (bucket.file.read (data) != length)
+                            eof (bucket);
+
+                        return data;
+                }
+
+                /**************************************************************
+
+                        This should be protected from thread-contention at
+                        a higher level.
+
+                **************************************************************/
+
+                void write (FileBucket bucket, void[] data, BlockSize block)
+                {
+                        length = data.length;
+
+                        // create new slot if we exceed capacity
+                        if (length > capacity)
+                            createBucket (bucket, length, block);
+
+                        // locate to start of content 
+                        bucket.file.seek (offset);
+        
+                        // write content
+                        if (bucket.file.write (data) != length)
+                            eof (bucket);
+                }
+
+                /**************************************************************
+
+                **************************************************************/
+
+                void createBucket (FileBucket bucket, int bytes, BlockSize block)
+                {
+                        offset = bucket.waterLine;
+                        capacity = (bytes + block.capacity) & ~block.capacity;
+
+                        bucket.waterLine += capacity;
+                        if (bucket.waterLine > bucket.fileSize)
+                           {
+                           // grow the filesize 
+                           bucket.fileSize = bucket.waterLine * 2;
+
+                           // expand the physical file size
+                           bucket.file.seek (bucket.fileSize);
+                           bucket.file.truncate ();
+                           }
+                }
+        }
+}
+
+

tango/example/conduits/composite.d

@@ -0,0 +1,85 @@
+
+private import  tango.io.protocol.Reader,
+                tango.io.protocol.Writer,
+                tango.io.FileConduit;
+
+/*******************************************************************************
+
+        Use cascading reads & writes to handle a composite class. There is
+        just one primary call for output, and just one for input, but the
+        classes propogate the request as appropriate. 
+
+        Note that the class instances don't know how their content will be
+        represented; that is dictated by the caller (via the reader/writer
+        implementation).
+
+        Note also that this only serializes the content. To serialize the
+        classes too, take a look at the Pickle.d example.
+
+*******************************************************************************/
+
+void main()
+{
+        // define a serializable class (via interfaces)
+        class Wumpus : IReadable, IWritable
+        {
+                private int     a = 11,
+                                b = 112,
+                                c = 1024;
+
+                void read (IReader input)
+                {
+                        input (a) (b) (c);
+                }
+
+                void write (IWriter output)
+                {
+                        output (a) (b) (c);
+                }
+        }
+
+
+        // define a serializable class (via interfaces)
+        class Wombat : IReadable, IWritable
+        {
+                private Wumpus  wumpus;
+                private char[]  x = "xyz";
+                private bool    y = true;
+                private float   z = 3.14159;
+
+                this (Wumpus wumpus)
+                {
+                        this.wumpus = wumpus;
+                }
+
+                void read (IReader input)
+                {
+                        input (x) (y) (z) (wumpus);
+                }
+
+                void write (IWriter output)
+                {
+                        output (x) (y) (z) (wumpus);
+                }
+        }
+
+        // construct a Wombat
+        auto wombat = new Wombat (new Wumpus);
+
+        // open a file for IO
+        auto file = new FileConduit ("random.bin", FileConduit.ReadWriteCreate);
+
+        // construct reader & writer upon the file, with binary IO
+        auto output = new Writer (file);
+        auto input = new Reader (file);
+
+        // write both Wombat & Wumpus (and flush them)
+        output (wombat) ();
+
+        // rewind to file start
+        file.seek (0);
+
+        // read both back again
+        input (wombat);
+}
+

tango/example/conduits/filebubbler.d

@@ -0,0 +1,30 @@
+
+private import  tango.io.Console,
+                tango.io.FileScan,
+                tango.io.FileConst;
+
+/*******************************************************************************
+
+        This example sweeps a named sub-directory tree for html files,
+        and moves them to the current directory. The existing directory 
+        hierarchy is flattened into a naming scheme where a '.' is used
+        to replace the traditional path-separator
+
+        Used by the Tango project to help manage renderings of the source 
+        code.
+
+*******************************************************************************/
+
+void main(char[][] args)
+{
+        // sweep all html files in the specified subdir
+        if (args.length is 2)
+            foreach (proxy; (new FileScan).sweep(args[1], ".html").files)
+                    {
+                    auto other = new FilePath (proxy.toString);
+                    proxy.rename (other.replace (FileConst.PathSeparatorChar, '.'));
+                    }
+        else
+           Cout ("usage is filebubbler subdir").newline;
+}
+

tango/example/conduits/filecat.d

@@ -0,0 +1,26 @@
+private import  tango.io.Console,
+                tango.io.FileConduit;
+
+/*******************************************************************************
+
+        Concatenate a number of files onto a single destination
+
+*******************************************************************************/
+
+void main(char[][] args)
+{
+        if (args.length > 2)
+           {
+           // open the file for writing
+           auto dst = new FileConduit (args[1], FileConduit.WriteCreate);
+
+           // copy each file onto dst
+           foreach (char[] arg; args[2..args.length])
+                    dst.copy (new FileConduit(arg));
+
+           // flush output and close
+           dst.close;
+           }
+        else
+           Cout ("usage: filecat target source1 ... sourceN");
+}

tango/example/conduits/filecopy.d

@@ -0,0 +1,23 @@
+
+private import  tango.io.Console,
+                tango.io.FileConduit;
+
+/*******************************************************************************
+
+        open a file, and stream directly to console
+
+*******************************************************************************/
+
+void main (char[][] args)
+{
+        if (args.length is 2)
+           {
+           // open a file for reading
+           auto fc = new FileConduit (args[1]);
+
+           // stream directly to console
+           Cout.stream.copy (fc);
+           }
+        else
+           Cout ("usage is: filecopy filename").newline;
+}

tango/example/conduits/fileops.d

@@ -0,0 +1,26 @@
+/*****************************************************
+
+  Example that shows some simple file operations
+
+  Put into public domain by Lars Ivar Igesund
+
+*****************************************************/
+
+import tango.io.Stdout;
+
+import tango.io.FilePath;
+
+void main (char[][] args)
+{
+    auto src = args[0] ~ ".d";
+    auto dst = new FilePath (args[0] ~ ".d.copy");
+
+    Stdout.formatln ("copy file {} to {}", src, dst);
+    dst.copy (src);
+    assert (dst.exists);
+
+    Stdout.formatln ("removing file {}",  dst);
+    dst.remove;
+
+    assert (dst.exists is false);
+}

tango/example/conduits/filepathname.d

@@ -0,0 +1,11 @@
+import tango.io.Console;
+
+import tango.io.FilePath;
+
+void main(){
+    Cout ((new FilePath(r"d:\path\foo.bat")).name).newline;
+    Cout ((new FilePath(r"d:\path.two\bar")).name).newline;
+    Cout ((new FilePath("/home/user.name/bar.")).name).newline;
+    Cout ((new FilePath(r"d:\path.two\bar")).name).newline;
+    Cout ((new FilePath("/home/user/.resource")).name).newline;
+}

tango/example/conduits/filescan.d

@@ -0,0 +1,32 @@
+private import  tango.io.Stdout,
+                tango.io.FileScan;
+
+/*******************************************************************************
+
+        List ".d" files and enclosing folders visible via a directory given
+        as a command-line argument. In this example we're also postponing a
+        flush on Stdout until output is complete. Stdout is usually flushed
+        on each invocation of newline or formatln, but here we're using '\n'
+        to illustrate how to avoid flushing many individual lines
+        
+*******************************************************************************/
+
+void main(char[][] args)
+{       
+        char[] root = args.length < 2 ? "." : args[1];
+        Stdout.formatln ("Scanning '{}'", root);
+
+        auto scan = (new FileScan)(root, ".d");
+
+        Stdout.format ("\n{} Folders\n", scan.folders.length);
+        foreach (folder; scan.folders)
+                 Stdout.format ("{}\n", folder);
+
+        Stdout.format ("\n{0} Files\n", scan.files.length);
+        foreach (file; scan.files)
+                 Stdout.format ("{}\n", file);
+
+        Stdout.formatln ("\n{} Errors", scan.errors.length);
+        foreach (error; scan.errors)
+                 Stdout (error).newline;
+}

tango/example/conduits/filescanregex.d

@@ -0,0 +1,40 @@
+/**************************************************************
+
+    Example that use FileScan and Regex as a filter.
+
+    Put into public domain by Lars Ivar Igesund
+
+**************************************************************/
+
+import tango.io.File,
+       tango.io.Stdout,
+       tango.io.FileScan,
+       tango.text.Regex;
+
+void main(char[][] args) {
+    uint total;
+
+    if (args.length < 2) {
+        Stdout("Please pass a directory to search").newline;
+        return;
+    }
+
+    scope scan = new FileScan;
+    scope regex =  Regex(r"\.(d|obj)$");
+
+    scan(args[1], delegate bool (FilePath fp, bool isDir) {
+         ++total;
+         return isDir || regex.test(fp.toString);
+    });
+
+
+    foreach (file; scan.files)
+             Stdout(file).newline;
+
+    Stdout.formatln("Found {} matches in {} entries", scan.files.length, total);
+
+    Stdout.formatln ("\n{} Errors", scan.errors.length);
+    foreach (error; scan.errors)
+             Stdout (error).newline;
+}
+

tango/example/conduits/lineio.d

@@ -0,0 +1,29 @@
+
+private import  tango.io.Console,
+                tango.io.FileConduit;
+
+private import  tango.text.stream.LineIterator;
+
+/*******************************************************************************
+
+        Read a file line-by-line, sending each one to the console. This
+        illustrates how to bind a conduit to a stream iterator (iterators
+        also support the binding of a buffer). Note that stream iterators
+        are templated for char, wchar and dchar types.
+
+*******************************************************************************/
+
+void main (char[][] args)
+{
+        if (args.length is 2)
+           {
+           // open a file for reading
+           scope file = new FileConduit (args[1]);
+
+           // process file one line at a time
+           foreach (line; new LineIterator!(char)(file))
+                    Cout (line).newline;
+           }
+        else
+           Cout ("usage: lineio filename").newline;
+}

tango/example/conduits/mmap.d

@@ -0,0 +1,24 @@
+
+private import  tango.io.Console,
+                tango.io.FileConduit,
+                tango.io.MappedBuffer;
+
+/*******************************************************************************
+
+        open a file, map it into memory, and copy to console
+
+*******************************************************************************/
+
+void main (char[][] args)
+{
+        if (args.length is 2)
+           {
+           // open a file for reading
+           auto mmap = new MappedBuffer (new FileConduit (args[1]));
+
+           // copy content to console
+           Cout (cast(char[]) mmap.slice) ();
+           }
+        else
+           Cout ("usage is: mmap filename").newline;
+}

tango/example/conduits/paths.d

@@ -0,0 +1,13 @@
+/*****************************************************
+
+        How to create a path with all ancestors
+
+*****************************************************/
+
+import tango.io.FilePath;
+
+void main (char[][] args)
+{
+        auto path = new FilePath (r"d/tango/foo/bar/wumpus");
+        assert (path.create.exists && path.isFolder);
+}

tango/example/conduits/randomio.d

@@ -0,0 +1,38 @@
+
+private import  tango.io.FileConduit;
+
+private import  tango.io.protocol.Reader,
+                tango.io.protocol.Writer;
+
+/*******************************************************************************
+
+       Create a file for random access. Write some stuff to it, rewind to
+       file start and read back.
+
+*******************************************************************************/
+
+void main()
+{
+        // open a file for reading
+        auto fc = new FileConduit ("random.bin", FileConduit.ReadWriteCreate);
+
+        // construct (binary) reader & writer upon this conduit
+        auto read  = new Reader (fc);
+        auto write = new Writer (fc);
+
+        int x=10, y=20;
+
+        // write some data and flush output since IO is buffered
+        write (x) (y) ();
+
+        // rewind to file start
+        fc.seek (0);
+
+        // read data back again, but swap destinations
+        read (y) (x);
+
+        assert (y is 10);
+        assert (x is 20);
+
+        fc.close();
+}

tango/example/conduits/unifile.d

@@ -0,0 +1,25 @@
+
+private import  tango.io.Console,
+                tango.io.UnicodeFile;
+
+/*******************************************************************************
+
+        Open a unicode file of an unknown encoding, and converts to UTF-8 
+        for console display. UnicodeFile is templated for char/wchar/dchar
+        target encodings
+
+*******************************************************************************/
+
+void main (char[][] args)
+{
+        if (args.length is 2)
+           {
+           // open a file for reading
+           auto file = new UnicodeFile!(char) (args[1], Encoding.Unknown);
+
+           // display on console
+           Cout (file.read).newline;
+           }
+        else
+           Cout ("usage is: unifile filename").newline;
+}