[ { "path": "BuildVersion.cs", "content": "using System.Reflection;\n// build number = 606\n// build version = 4.0.10\n\n[assembly: AssemblyVersion(\"4.0.0.0\")]\n[assembly: AssemblyFileVersion(\"4.0.10.606\")]" }, { "path": "LICENSE", "content": "The MIT License (MIT)\n\nCopyright (c) 2014 Mehdi Gholam\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n\n" }, { "path": "README.md", "content": "# RaptorDB Document Store\n\nNoSql, JSON based, Document store database with compiled .net map functions and automatic hybrid bitmap indexing and LINQ query filters (now with standalone Server mode, Backup and Active Restore, Transactions, Server side queries, MonoDroid support, HQ-Branch Replication)\n\nsee the article here : [http://www.codeproject.com/Articles/375413/RaptorDB-the-Document-Store] (http://www.codeproject.com/Articles/375413/RaptorDB-the-Document-Store)\n\n## Quick Start\n\nFirst compile the source, then you can easily run any c# file like this:\n\n```\n# run any cs file\nc:\\rdb\\test script> ..\\tools\\nscript.exe sample.cs\n\n# or just run the batch file\nc:\\rdb\\test script> run.cmd \n```\n\nThe `sample.cs` file now contains a comment section at the top for specifing references used which will tell `nscript.exe` where to find the dll files:\n\n```\n// ref : ..\\output\\raptordb.dll\n// ref : ..\\output\\raptordb.common.dll\n// ref : ..\\faker.dll\nusing System;\nusing System.Collections.Generic;\n...\n```\n" }, { "path": "RaptorDB/AssemblyInfo.cs", "content": "using System.Reflection;\nusing System.Security;\n\n[assembly: AssemblyTitle(\"RaptorDB Document Store\")]\n[assembly: AssemblyDescription(\"NoSql, JSON based, Document store database with compiled .net map functions and automatic hybrid bitmap indexing and LINQ query filters (now with standalone Server mode, Backup and Active Restore, Transactions, Server side queries, MonoDroid support, HQ-Branch Replication)\")]\n[assembly: AssemblyProduct(\"RaptorDB Document Store\")]\n\n" }, { "path": "RaptorDB/DataTypes/DataTypes.cs", "content": "using System;\nusing RaptorDB.Common;\n\nnamespace RaptorDB\n{\n /// \n /// Used to track ViewDelete usage for view rebuilds\n /// \n internal class View_delete\n {\n public Guid ID = Guid.NewGuid();\n public string Viewname;\n public string Filter;\n }\n\n internal class View_insert\n {\n public Guid ID = Guid.NewGuid();\n public string Viewname;\n public object RowObject;\n }\n\n internal class FullTextString \n {\n\n }\n\n internal class NoIndexing\n {\n\n }\n\n public interface IRowFiller\n {\n object FillRow(object row, object[] data);\n }\n\n internal interface IGetBytes\n {\n byte[] GetBytes(T obj);\n T GetObject(byte[] buffer, int offset, int count);\n }\n\n internal class RDBDataType\n {\n public static IGetBytes ByteHandler()\n {\n Type type = typeof(T);\n\n if (type == typeof(int)) return (IGetBytes)new int_handler();\n else if (type == typeof(uint)) return (IGetBytes)new uint_handler();\n else if (type == typeof(long)) return (IGetBytes)new long_handler();\n else if (type == typeof(Guid)) return (IGetBytes)new guid_handler();\n else if (type == typeof(string)) return (IGetBytes)new string_handler();\n else if (type == typeof(DateTime)) return (IGetBytes)new datetime_handler();\n else if (type == typeof(decimal)) return (IGetBytes)new decimal_handler();\n else if (type == typeof(short)) return (IGetBytes)new short_handler();\n else if (type == typeof(ushort)) return (IGetBytes)new ushort_handler();\n else if (type == typeof(float)) return (IGetBytes)new float_handler();\n else if (type == typeof(byte)) return (IGetBytes)new byte_handler();\n else if (type == typeof(double)) return (IGetBytes)new double_handler();\n\n return null;\n }\n\n public static byte GetByteSize(byte keysize)\n {\n byte size = 4;\n Type t = typeof(T);\n\n if (t == typeof(int)) size = 4;\n if (t == typeof(uint)) size = 4;\n if (t == typeof(long)) size = 8;\n if (t == typeof(Guid)) size = 16;\n if (t == typeof(DateTime)) size = 8;\n if (t == typeof(decimal)) size = 16;\n if (t == typeof(float)) size = 4;\n if (t == typeof(short)) size = 2;\n if (t == typeof(string)) size = keysize;\n if (t == typeof(byte)) size = 1;\n if (t == typeof(double)) size = 8;\n\n return size;\n }\n\n internal static object GetEmpty()\n {\n Type t = typeof(T);\n\n if (t == typeof(string))\n return \"\";\n\n return default(T);\n }\n }\n\n #region [ handlers ]\n\n internal class double_handler : IGetBytes\n {\n public byte[] GetBytes(double obj)\n {\n return BitConverter.GetBytes(obj);\n }\n\n public double GetObject(byte[] buffer, int offset, int count)\n {\n return BitConverter.ToDouble(buffer, offset);\n }\n }\n\n internal class byte_handler : IGetBytes\n {\n public byte[] GetBytes(byte obj)\n {\n return new byte[1] { obj };\n }\n\n public byte GetObject(byte[] buffer, int offset, int count)\n {\n return buffer[offset];\n }\n }\n\n internal class float_handler : IGetBytes\n {\n public byte[] GetBytes(float obj)\n {\n return BitConverter.GetBytes(obj);\n }\n\n public float GetObject(byte[] buffer, int offset, int count)\n {\n return BitConverter.ToSingle(buffer, offset);\n }\n }\n\n internal class decimal_handler : IGetBytes\n {\n public byte[] GetBytes(decimal obj)\n {\n byte[] b = new byte[16];\n var bb = decimal.GetBits(obj);\n int index = 0;\n foreach (var d in bb)\n {\n byte[] db = Helper.GetBytes(d, false);\n Buffer.BlockCopy(db, 0, b, index, 4);\n index += 4;\n }\n\n return b;\n }\n\n public decimal GetObject(byte[] buffer, int offset, int count)\n {\n int[] i = new int[4];\n i[0] = Helper.ToInt32(buffer, offset);\n offset += 4;\n i[1] = Helper.ToInt32(buffer, offset);\n offset += 4;\n i[2] = Helper.ToInt32(buffer, offset);\n offset += 4;\n i[3] = Helper.ToInt32(buffer, offset);\n offset += 4;\n\n return new decimal(i);\n }\n }\n\n internal class ushort_handler : IGetBytes\n {\n public byte[] GetBytes(ushort obj)\n {\n return Helper.GetBytes(obj, false);\n }\n\n public ushort GetObject(byte[] buffer, int offset, int count)\n {\n return (ushort)Helper.ToInt16(buffer, offset);\n }\n }\n\n internal class short_handler : IGetBytes\n {\n public byte[] GetBytes(short obj)\n {\n return Helper.GetBytes(obj, false);\n }\n\n public short GetObject(byte[] buffer, int offset, int count)\n {\n return Helper.ToInt16(buffer, offset);\n }\n }\n\n internal class string_handler : IGetBytes\n {\n public byte[] GetBytes(string obj)\n {\n return Helper.GetBytes(obj);\n }\n\n public string GetObject(byte[] buffer, int offset, int count)\n {\n return Helper.GetString(buffer, offset, (short)count);\n }\n }\n\n internal class int_handler : IGetBytes\n {\n public byte[] GetBytes(int obj)\n {\n return Helper.GetBytes(obj, false);\n }\n\n public int GetObject(byte[] buffer, int offset, int count)\n {\n return Helper.ToInt32(buffer, offset);\n }\n }\n\n internal class uint_handler : IGetBytes\n {\n public byte[] GetBytes(uint obj)\n {\n return Helper.GetBytes(obj, false);\n }\n\n public uint GetObject(byte[] buffer, int offset, int count)\n {\n return (uint)Helper.ToInt32(buffer, offset);\n }\n }\n\n internal class long_handler : IGetBytes\n {\n public byte[] GetBytes(long obj)\n {\n return Helper.GetBytes(obj, false);\n }\n\n public long GetObject(byte[] buffer, int offset, int count)\n {\n return Helper.ToInt64(buffer, offset);\n }\n }\n\n internal class guid_handler : IGetBytes\n {\n public byte[] GetBytes(Guid obj)\n {\n return obj.ToByteArray();\n }\n\n public Guid GetObject(byte[] buffer, int offset, int count)\n {\n byte[] b = new byte[16];\n Buffer.BlockCopy(buffer, offset, b, 0, 16);\n return new Guid(b);\n }\n }\n\n internal class datetime_handler : IGetBytes\n {\n public byte[] GetBytes(DateTime obj)\n {\n return Helper.GetBytes(obj.Ticks, false);\n }\n\n public DateTime GetObject(byte[] buffer, int offset, int count)\n {\n long ticks = Helper.ToInt64(buffer, offset);\n\n return new DateTime(ticks);\n }\n }\n #endregion\n}" }, { "path": "RaptorDB/Global.cs", "content": "namespace RaptorDB\n{\n public class Global\n {\n /// \n /// Store bitmap as int offsets then switch over to bitarray\n /// \n public static int BitmapOffsetSwitchOverCount = 10;\n /// \n /// True = Save to other views in process , False = background save to other views\n /// \n public static bool BackgroundSaveToOtherViews = true;\n /// \n /// Default maximum string key size for indexes\n /// \n public static byte DefaultStringKeySize = 60;\n /// \n /// Free bitmap index memory on save \n /// \n public static bool FreeBitmapMemoryOnSave = false;\n /// \n /// Number of items in each index page (default = 10000) [Expert only, do not change]\n /// \n public static ushort PageItemCount = 10000;\n /// \n /// KeyStore save to disk timer\n /// \n public static int SaveIndexToDiskTimerSeconds = 1800;\n /// \n /// Flush the StorageFile stream immediately\n /// \n public static bool FlushStorageFileImmediately = false;\n /// \n /// Save doc as binary json\n /// \n public static bool SaveAsBinaryJSON = true;\n /// \n /// Remove completed tasks timer\n /// \n public static int TaskCleanupTimerSeconds = 3;\n /// \n /// Save to other views timer seconds if enabled \n /// \n public static int BackgroundSaveViewTimer = 1;\n /// \n /// How many items to process in a background view save event\n /// \n public static int BackgroundViewSaveBatchSize = 1000000;\n ///// \n ///// Check the restore folder for new backup files to restore\n ///// \n //public static int RestoreTimerSeconds = 10; // TODO : implement this\n /// \n /// Timer for full text indexing of original documents (default = 15 sec)\n /// \n public static int FullTextTimerSeconds = 15;\n /// \n /// How many documents to full text index in a batch\n /// \n public static int BackgroundFullTextIndexBatchSize = 10000;\n /// \n /// Free memory checking timer (default = 300 sec ~ 5 min)\n /// \n public static int FreeMemoryTimerSeconds = 5 * 60;// 1800; \n /// \n /// Memory usage limit for internal caching (default = 100 Mb) [using GC.GetTotalMemory()]\n /// \n public static long MemoryLimit = 100;\n /// \n /// Backup cron schedule (default = \"0 * * * *\" [every hour]) \n /// \n public static string BackupCronSchedule = \"0 * * * *\";\n /// \n /// Require primary view to be defined for save, false = key/value store (default = true)\n /// \n public static bool RequirePrimaryView = true;\n /// \n /// Maximum documents in each package for replication\n /// \n public static int PackageSizeItemCountLimit = 10000;\n /// \n /// Process inbox timer (default = 60 sec)\n /// \n public static int ProcessInboxTimerSeconds = 60;\n /// \n /// Split the data storage files in MegaBytes (default 0 = off) [500 = 500mb]\n /// - You can set and unset this value anytime and it will operate from that point on.\n /// - If you unset (0) the value previous split files will remain and all the data will go to the last file.\n /// \n public static ushort SplitStorageFilesMegaBytes = 0;\n /// \n /// Compress the documents in the storage file if it is over this size (default = 100 Kilobytes) \n /// - You will be trading CPU for disk IO\n /// \n public static ushort CompressDocumentOverKiloBytes = 100;\n /// \n /// Disk block size for high frequency KV storage file (default = 2048)\n /// * Do not use anything under 512 with large string keys\n /// \n public static ushort HighFrequencyKVDiskBlockSize = 2048;\n /// \n /// String key MGIndex that stores keys in an external file for smaller index files\n /// \n public static bool EnableOptimizedStringIndex = true;\n /// \n /// Enable the Web Studio interface\n /// \n public static bool EnableWebStudio = false;\n /// \n /// Web Studio port (default = 91)\n /// \n public static short WebStudioPort = 91;\n /// \n /// Local machine access only Web Studio - no network access (default = true)\n /// \n public static bool LocalOnlyWebStudio = true;\n /// \n /// If True -> less memory use SafeSortedList and slower\n /// False -> more memory use SafeDictionary and faster\n /// \n public static bool UseLessMemoryStructures = false;\n\n //public static bool useSortedList = false;\n\n public static bool CompressBitmapBytes = false;\n\n public static bool SkipDocsOnViewInsert = false;\n }\n\n\n}\n" }, { "path": "RaptorDB/Helper/Container.cs", "content": "using System.Collections.Generic;\n\nnamespace RaptorDB\n{\n //----------------------------------------------------------------------------------------------------------------\n //----------------------------------------------------------------------------------------------------------------\n //----------------------------------------------------------------------------------------------------------------\n class BitmapContainer : Container\n {\n public BitmapContainer()\n {\n }\n\n public BitmapContainer(int OneCount)\n {\n if (OneCount > Container.BSize)\n throw new System.Exception(\"OneCount > 65536\");\n\n else if (OneCount == Container.BSize)\n ALLONE = true;\n\n else\n {\n List l = new List();\n while(OneCount>0)\n {\n if (OneCount > 64)\n l.Add(ulong.MaxValue);\n else\n l.Add(ulong.MaxValue< 0)\n return _onecount;\n\n lock (_lock)\n {\n long c = 0;\n\n foreach (var l in _values)\n c += BitCount(l);\n\n _onecount = c;\n\n return c;\n }\n }\n\n public override long CountZeros()\n {\n if (ALLONE)\n return 0;\n\n return BSize - CountOnes();\n }\n\n public override IEnumerable GetBitIndexes()\n {\n lock (_lock)\n {\n if (ALLONE)\n {\n for (int i = 0; i < BSize; i++)\n yield return (ushort)i;\n }\n\n ushort c = 0;\n foreach (var l in _values)\n {\n for (int i = 0; i < 64; i++)\n {\n ulong mask = (ulong)1 << (63 - i); // high order bit get\n if ((l & mask) != 0)\n yield return (ushort)(c + i);\n }\n c += 64;\n }\n }\n }\n\n public override bool Get(long offset)\n {\n lock (_lock)\n {\n if (ALLONE)\n return true;\n\n int pos = (ushort)offset >> 6;\n int off = (int)(offset % 64);\n\n if (pos >= _values.Length) // out of range\n {\n return false;\n }\n\n ulong mask = (ulong)1 << (63 - off); // high order bit get\n\n return (_values[pos] & mask) != 0;\n }\n }\n\n public override void Set(long offset, bool val)\n {\n lock (_lock)\n {\n if (ALLONE)\n {\n if (val == true)\n return;\n // change to bits\n ALLONE = false;\n _values = new ulong[1024];\n for (int i = 0; i < 1024; i++)\n _values[i] = ulong.MaxValue;\n }\n int pos = (ushort)offset >> 6;\n int off = (int)(offset % 64);\n _onecount = -1;\n\n if (_values == null)\n {\n _values = new ulong[0];\n }\n if (pos >= _values.Length) // out of range\n {\n // resize\n var a = new ulong[pos + 1];\n _values.CopyTo(a, 0);\n _values = a;\n }\n ulong mask = (ulong)1 << (63 - off); // high order bit get\n\n if (val)\n _values[pos] |= mask;\n else\n _values[pos] &= ~mask;\n\n Size = _values.Length * 64;\n }\n }\n\n public override bool ChangeRequired()\n {\n if (ALLONE)\n return false;\n\n if (CountOnes() == BSize) // -> all ones container\n return true;\n\n if (CountZeros() < CHGOVER) // -> inverted container\n return true;\n\n var offbytes = CountOnes() << 1; //*2\n var bytes = _values.Length << 3; //*8\n if (bytes > offbytes)\n return true; // -> offset container\n\n return false;\n }\n\n public override Container ToBitmap()\n {\n return Copy();\n }\n\n public override Container Change()\n {\n if (ALLONE)\n return new BitmapContainer(true);\n\n if (CountOnes() == BSize)\n return new BitmapContainer(true);\n\n // create inverted\n if (CountZeros() < CHGOVER)\n return new InvertedContainer(Not().GetBitIndexes());\n\n Container c = null;\n //if (Global.useSortedList)\n // c = new OffsetContainerSL();\n //else\n c = new OffsetContainer();\n\n foreach (var i in GetBitIndexes())\n c.Set(i, true);\n\n return c;\n }\n\n public override Container Copy()\n {\n if (ALLONE)\n return new BitmapContainer(true);\n\n if (_values != null && _values.Length > 0)\n return new BitmapContainer(Values());\n else\n return new BitmapContainer();\n }\n\n public override Container Not()\n {\n lock (_lock)\n {\n if (ALLONE)\n return new BitmapContainer();\n\n var vals = new ulong[1024]; // TODO : upto Size ??\n for (int i = 0; i < 1024; i++)\n vals[i] = ulong.MaxValue;\n\n for (int i = 0; i < _values.Length; i++)\n vals[i] = ~_values[i];\n\n return new BitmapContainer(vals);\n }\n }\n }\n\n //----------------------------------------------------------------------------------------------------------------\n //----------------------------------------------------------------------------------------------------------------\n //----------------------------------------------------------------------------------------------------------------\n class OffsetContainer : Container\n {\n public OffsetContainer()\n {\n\n }\n public OffsetContainer(IEnumerable vals)\n {\n _values = new List(vals);\n Size = _values[_values.Count - 1];\n }\n\n List _values = new List();\n private object _lock = new object();\n\n public ushort[] Values()\n {\n lock (_lock)\n return _values.ToArray();\n }\n\n public override Container ToBitmap()\n {\n lock (_lock)\n {\n var c = new BitmapContainer();\n foreach (var i in _values)\n c.Set(i, true);\n return c;\n }\n }\n\n public override Container Change()\n {\n if (CountOnes() == Container.BSize)\n return new BitmapContainer(true);\n\n if (CountZeros() < CHGOVER)\n return new InvertedContainer(ToBitmap().Not().GetBitIndexes());\n\n return ToBitmap();\n }\n\n public override bool ChangeRequired()\n {\n if (_values.Count > CHGOVER)\n return true;\n\n if (CountZeros() < CHGOVER)\n return true;\n\n return false;\n }\n\n public override Container Copy()\n {\n lock (_lock)\n {\n if (_values != null && _values.Count > 0)\n return new OffsetContainer(_values.ToArray());\n else\n return new OffsetContainer();\n }\n }\n\n public override long CountOnes()\n {\n return _values.Count;\n }\n\n public override long CountZeros()\n {\n return BSize - CountOnes();\n }\n\n public override bool Get(long offset)\n {\n lock (_lock)\n {\n var i = _values.BinarySearch((ushort)offset);\n if (i >= 0)\n return true;\n return false;\n }\n }\n\n public override IEnumerable GetBitIndexes()\n {\n lock (_lock)\n foreach (var i in _values)\n yield return i;\n }\n\n public override void Set(long offset, bool val)\n {\n lock (_lock)\n {\n var i = _values.BinarySearch((ushort)offset);\n\n if (val == true)\n {\n if (i < 0) // not in array -> add\n {\n var c = ~i;\n if (c < _values.Count)\n _values.Insert(c, (ushort)offset);\n else\n _values.Add((ushort)offset);\n }\n }\n else if (i >= 0)\n {\n // remove from array\n _values.RemoveAt(i);\n }\n if (_values.Count > 0)\n Size = _values[_values.Count - 1];\n else\n Size = -1;\n }\n }\n\n public override Container Not()\n {\n return ToBitmap().Not();\n }\n }\n\n //----------------------------------------------------------------------------------------------------------------\n //----------------------------------------------------------------------------------------------------------------\n //----------------------------------------------------------------------------------------------------------------\n class InvertedContainer : Container\n {\n public InvertedContainer()\n {\n\n }\n\n // list of zeros\n public InvertedContainer(IEnumerable vals)\n {\n _values = new List(vals);\n if (_values.Count > 0)\n Size = _values[_values.Count - 1];\n else\n Size = Container.BSize;\n }\n\n List _values = new List();\n private object _lock = new object();\n\n public override Container ToBitmap()\n {\n lock (_lock)\n {\n var b = new BitmapContainer();\n\n foreach (var i in GetBitIndexes())\n {\n b.Set(i, true);\n }\n return b;\n }\n }\n\n public override Container Change()\n {\n if (CountZeros() > CHGOVER)\n return ToBitmap();\n\n return Copy();\n }\n\n public override bool ChangeRequired()\n {\n if (CountZeros() > CHGOVER)\n return true;\n\n return false;\n }\n\n public override Container Copy()\n {\n lock (_lock)\n {\n if (_values != null && _values.Count > 0)\n return new InvertedContainer(_values.ToArray());\n else\n return new InvertedContainer();\n }\n }\n\n public override long CountOnes()\n {\n return BSize - CountZeros();\n }\n\n public override long CountZeros()\n {\n return _values.Count;\n }\n\n public override bool Get(long offset)\n {\n lock (_lock)\n {\n var i = _values.BinarySearch((ushort)offset);\n if (i >= 0) // in the list of zeros\n return false;\n return true;\n }\n }\n\n public override IEnumerable GetBitIndexes()\n {\n lock (_lock)\n {\n for (int i = 0; i < BSize; i++)\n {\n var j = _values.BinarySearch((ushort)i);\n if (j < 0) // not in the list of zeros\n yield return (ushort)i;\n }\n }\n }\n\n public override Container Not()\n {\n return ToBitmap().Not();\n }\n\n public override void Set(long offset, bool val)\n {\n lock (_lock)\n {\n var i = _values.BinarySearch((ushort)offset);\n\n if (val == false)\n {\n if (i < 0) // not in array -> add\n {\n var c = ~i;\n if (c < _values.Count)\n _values.Insert(c, (ushort)offset);\n else\n _values.Add((ushort)offset);\n }\n }\n else if (i >= 0)\n {\n // remove from array\n _values.RemoveAt(i);\n }\n\n // fix : return correct size\n if (_values.Count > 0)\n Size = _values[_values.Count - 1];\n else\n Size = -1;\n }\n }\n\n public ushort[] Values()\n {\n lock (_lock)\n return _values.ToArray();\n }\n }\n\n //----------------------------------------------------------------------------------------------------------------\n //----------------------------------------------------------------------------------------------------------------\n //----------------------------------------------------------------------------------------------------------------\n public abstract class Container\n {\n internal const int BSize = 65536;\n internal const int CHGOVER = 4096;\n\n public abstract void Set(long offset, bool val);\n public abstract bool Get(long offset);\n public abstract long CountOnes();\n public abstract long CountZeros();\n public abstract IEnumerable GetBitIndexes();\n public abstract bool ChangeRequired();\n public abstract Container Change();\n public abstract Container ToBitmap();\n public abstract Container Copy();\n public abstract Container Not();\n\n public int Size = -1;\n\n //[MethodImpl(MethodImplOptions.AggressiveInlining)]\n public static int BitCount(ulong x)\n {\n x -= (x >> 1) & 0x5555555555555555UL; //put count of each 2 bits into those 2 bits\n x = (x & 0x3333333333333333UL) + ((x >> 2) & 0x3333333333333333UL); //put count of each 4 bits into those 4 bits \n x = (x + (x >> 4)) & 0x0F0F0F0F0F0F0F0FUL; //put count of each 8 bits into those 8 bits \n return (int)((x * 0x0101010101010101UL) >> 56); //returns left 8 bits of x + (x<<8) + (x<<16) + (x<<24) + ... \n }\n }\n\n public enum CTYPE\n {\n ALLONES\n , BITMAP\n , OFFSET\n , INV\n //,OFFSETSL\n }\n public class CData\n {\n public ushort i;\n public CTYPE t;\n public byte[] d;\n }\n\n public class MGRBData\n {\n public List c = new List();\n }\n}\n" }, { "path": "RaptorDB/Helper/MGRB.cs", "content": "using RaptorDB.Common;\nusing System;\nusing System.Collections.Generic;\n\nnamespace RaptorDB\n{\n public class MGRB\n {\n public MGRB()\n { }\n\n internal MGRB(SafeSortedList containers) : this(containers, -1)\n { }\n\n internal MGRB(SafeSortedList containers, long size)\n {\n _containers = containers;\n var k = _containers.Keys();\n _size = size;\n if (size <= 0)//== -1)\n {\n _size = 0;\n var l = k.Length - 1;\n if (l >= 0)\n _size = (k[l] << 16) + _containers.GetValue(l).Size;\n }\n }\n\n private SafeSortedList _containers = new SafeSortedList();\n private long _size;\n private ushort _MASK = 0xffff;\n private object _lock = new object();\n public bool isDirty = false;\n\n public long Length { get { return _size; } }\n\n public void Set(long position, bool val)\n {\n lock (_lock)\n {\n isDirty = true;\n if (_size < position && val == true)\n _size = position;\n\n var idx = (int)(position >> 16);\n Container c = null;\n if (_containers.TryGetValue(idx, out c) == false)\n {\n //if (Global.useSortedList)\n // c = new OffsetContainerSL();\n //else\n c = new OffsetContainer();\n // add container\n _containers.Add(idx, c);\n }\n c.Set(position & _MASK, val);\n\n //if (c.ChangeRequired())\n // _containers[idx] = c.Change();\n }\n }\n\n public bool Get(long position)\n {\n lock (_lock)\n {\n var idx = (int)(position >> 16);\n if (_containers.TryGetValue(idx, out Container c))\n return c.Get(position & _MASK);\n\n return false;\n }\n }\n\n public MGRB And(MGRB B)\n {\n var v = new SafeSortedList();\n var len = _size;\n if (B.Length < len)\n len = B.Length;\n var a = LastContainerIdx();\n var b = B.LastContainerIdx();\n var min = a;\n if (b < min)\n min = b;\n min++;\n\n for (int i = 0; i < min; i++)\n {\n Container ca = null;\n Container cb = null;\n\n _containers.TryGetValue(i, out ca);\n B._containers.TryGetValue(i, out cb);\n\n if (ca != null && cb != null)\n v.Add(i, containerAND(ca, cb));\n }\n\n return new MGRB(v, len);\n }\n\n public MGRB Or(MGRB B)\n {\n var v = new SafeSortedList();\n var len = _size;\n if (B.Length > len)\n len = B.Length;\n var a = LastContainerIdx();\n var b = B.LastContainerIdx();\n var max = a;\n if (b > max)\n max = b;\n max++;\n\n for (int i = 0; i < max; i++)\n {\n Container ca = null;\n Container cb = null;\n\n _containers.TryGetValue(i, out ca);\n B._containers.TryGetValue(i, out cb);\n\n if (ca == null && cb != null)\n v.Add(i, cb.Copy());\n else if (cb == null && ca != null)\n v.Add(i, ca.Copy());\n else if (ca != null && cb != null)\n v.Add(i, containerOR(ca, cb));\n }\n\n return new MGRB(v, len);\n }\n\n public MGRB AndNot(MGRB b)\n {\n long c = _size;\n if (b._size > c)\n c = b._size;\n\n return And(b.Not(c));\n }\n\n public MGRB Not()\n {\n var con = new SafeSortedList();\n foreach (var c in _containers)\n {\n con.Add(c.Key, c.Value.Not());\n }\n\n return new MGRB(con, _size);\n }\n\n public MGRB Not(long count)\n {\n var con = new SafeSortedList();\n var c = count >> 16;\n for (int i = 0; i <= c; i++)\n {\n Container a = null;\n _containers.TryGetValue(i, out a);\n if (a == null)\n con.Add(i, new BitmapContainer(true));\n else\n con.Add(i, a.Not());\n }\n\n return new MGRB(con, count);\n }\n\n public static MGRB Fill(long count)\n {\n if (count == 0)\n return new MGRB();\n\n var con = new SafeSortedList();\n int i = 0;\n long c = count;\n while (count > 0)\n {\n if (count > Container.BSize)\n con.Add(i, new BitmapContainer(true));\n else\n con.Add(i, new BitmapContainer((int)count));\n count -= Container.BSize;\n i++;\n }\n\n return new MGRB(con, c);\n }\n\n public long CountOnes()\n {\n long c = 0;\n\n if (_size > 0)\n foreach (var i in _containers)\n c += i.Value.CountOnes();\n\n return c;\n }\n\n public long CountZeros()\n {\n var c = CountOnes();\n\n return _size - c;\n }\n\n public IEnumerable GetBitIndexes()\n {\n foreach (var c in _containers)\n {\n int i = c.Key << 16;\n foreach (var j in c.Value.GetBitIndexes())\n yield return i + j;\n }\n }\n\n public MGRB Optimize()\n {\n lock (_lock)\n {\n var keys = _containers.Keys();\n var remove = new List();\n\n for (int i = 0; i < keys.Length; i++)\n {\n var k = keys[i];\n var c = _containers[k];\n\n if (c.CountOnes() == 0)\n remove.Add(k);\n\n //else if (c.CountZeros() < Container.CHGOVER)\n //{\n // _containers[k] = new ZeroContainer();\n //}\n\n else if (c.ChangeRequired())\n _containers[k] = c.Change();\n }\n\n foreach (var k in remove)\n _containers.Remove(k);\n\n return this;\n }\n }\n\n public MGRBData Serialize()\n {\n var d = new MGRBData();\n\n foreach (var c in _containers)\n {\n var cd = new CData();\n {\n cd.i = (ushort)c.Key;\n if (c.Value is BitmapContainer)\n {\n var bm = c.Value as BitmapContainer;\n cd.t = CTYPE.BITMAP;\n if (bm.ALLONE)\n cd.t = CTYPE.ALLONES;\n else\n {\n // get data\n cd.d = ToByteArray(bm.Values());\n }\n }\n else if (c.Value is OffsetContainer)\n {\n var of = c.Value as OffsetContainer;\n cd.t = CTYPE.OFFSET;\n cd.d = ToByteArray(of.Values());\n }\n else if (c.Value is InvertedContainer)\n {\n var inv = c.Value as InvertedContainer;\n cd.t = CTYPE.INV;\n cd.d = ToByteArray(inv.Values());\n }\n //else\n //{\n // var of = c.Value as OffsetContainerSL;\n // cd.t = CTYPE.OFFSETSL;\n // var b = new byte[cd.d.Length];\n // int k = 0;\n // foreach (var i in of._values)\n // {\n // Buffer.BlockCopy(GetBytes(i.Key, false), 0, b, k, 2);\n // k += 2;\n // }\n // cd.d = b;\n //}\n\n d.c.Add(cd);\n }\n }\n return d;\n }\n\n public void Deserialize(MGRBData input)\n {\n foreach (var c in input.c)\n {\n Container con = null;\n if (c.t == CTYPE.ALLONES)\n {\n con = new BitmapContainer(true);\n }\n else if (c.t == CTYPE.BITMAP)\n {\n List list = new List();\n var dataLen = c.d.Length;\n for (int i = 0; i < dataLen; i += 8)\n {\n list.Add(ToULong(c.d, i));\n }\n con = new BitmapContainer(list.ToArray());\n }\n else if (c.t == CTYPE.OFFSET)\n {\n List list = new List();\n var dataLen = c.d.Length;\n for (int i = 0; i < dataLen; i += 2)\n {\n list.Add(ToUShort(c.d, i));\n }\n con = new OffsetContainer(list);\n }\n else if (c.t == CTYPE.INV)\n {\n List list = new List();\n var dataLen = c.d.Length;\n for (int i = 0; i < dataLen; i += 2)\n {\n list.Add(ToUShort(c.d, i));\n }\n con = new InvertedContainer(list);\n }\n //else\n //{\n // List list = new List();\n // var dataLen = c.d.Length;\n // for (int i = 0; i < dataLen; i += 2)\n // {\n // list.Add(ToUShort(c.d, i));\n // }\n // con = new OffsetContainerSL(list);\n //}\n _containers.Add(c.i, con);\n }\n var k = _containers.Keys();\n var l = k.Length - 1;\n if (l >= 0)\n _size = (k[l] << 16) + _containers.GetValue(l).Size;\n }\n\n public MGRB Copy()\n {\n if (_containers.Count() > 0)\n {\n var o = Serialize();\n var m = new MGRB();\n m.Deserialize(o);\n return m;\n }\n return new MGRB();\n }\n\n public int GetFirst()\n {\n int j = 0;\n foreach (var i in GetBitIndexes())\n {\n j = i;\n break;\n }\n return j;\n }\n\n private int LastContainerIdx()\n {\n if (_containers.Count() > 0)\n return _containers.Keys()[_containers.Count() - 1];\n else\n return 0;\n }\n\n private static Container containerAND(Container ca, Container cb)\n {\n BitmapContainer a = null;\n BitmapContainer b = null;\n\n if (ca is BitmapContainer)\n a = (BitmapContainer)ca;\n else if (ca is OffsetContainer)\n a = (BitmapContainer)ca.ToBitmap();\n else\n a = (BitmapContainer)ca.ToBitmap();\n\n if (cb is BitmapContainer)\n b = (BitmapContainer)cb;\n else if (cb is OffsetContainer)\n b = (BitmapContainer)cb.ToBitmap();\n else\n b = (BitmapContainer)cb.ToBitmap();\n\n var av = a.Values();\n var bv = b.Values();\n var la = av != null ? av.Length : 1024;\n var lb = bv != null ? bv.Length : 1024;\n var min = la;\n if (lb < min)\n min = lb;\n\n List vals = new List();\n for (int i = 0; i < min; i++)\n {\n ulong ua = ulong.MaxValue;\n ulong ub = ulong.MaxValue;\n if (av != null)\n ua = av[i];\n if (bv != null)\n ub = bv[i];\n\n vals.Add(ua & ub);\n }\n\n return new BitmapContainer(vals.ToArray());\n }\n\n private static Container containerOR(Container ca, Container cb)\n {\n BitmapContainer a = null;\n BitmapContainer b = null;\n\n if (ca is BitmapContainer)\n a = (BitmapContainer)ca;\n else if (ca is OffsetContainer)\n a = (BitmapContainer)ca.ToBitmap();\n else\n a = (BitmapContainer)ca.ToBitmap();\n\n if (cb is BitmapContainer)\n b = (BitmapContainer)cb;\n else if (cb is OffsetContainer)\n b = (BitmapContainer)cb.ToBitmap();\n else\n b = (BitmapContainer)cb.ToBitmap();\n\n var av = a.Values();\n var bv = b.Values();\n var la = av != null ? av.Length : 1024;\n var lb = bv != null ? bv.Length : 1024;\n var max = la;\n if (lb > max)\n max = lb;\n\n List vals = new List();\n\n for (int i = 0; i < max; i++)\n {\n ulong ua = 0;\n ulong ub = 0;\n if (av != null && i < la)\n ua = av[i];\n if (bv != null && i < lb)\n ub = bv[i];\n\n vals.Add(ua | ub);\n }\n\n return new BitmapContainer(vals.ToArray());\n }\n\n private static unsafe byte[] GetBytes(ushort num, bool reverse)\n {\n byte[] buffer = new byte[2];\n fixed (byte* numRef = buffer)\n {\n *((ushort*)numRef) = num;\n }\n if (reverse)\n Array.Reverse(buffer);\n return buffer;\n }\n\n private static unsafe ulong ToULong(byte[] value, int startIndex)\n {\n fixed (byte* numRef = &(value[startIndex]))\n {\n return *(((ulong*)numRef));\n }\n }\n\n private static unsafe ushort ToUShort(byte[] value, int startIndex)\n {\n fixed (byte* numRef = &(value[startIndex]))\n {\n return *(((ushort*)numRef));\n }\n }\n\n private static unsafe byte[] ToByteArray(ulong[] data)\n {\n int arrayLength = data.Length;\n byte[] byteArray = new byte[8 * arrayLength];\n fixed (ulong* pointer = data)\n {\n fixed (byte* bytePointer = byteArray)\n {\n ulong* read = pointer;\n ulong* write = (ulong*)bytePointer;\n\n for (int i = 0; i < arrayLength; i++)\n {\n *write++ = *read++;\n }\n }\n // below not working\n //System.Runtime.InteropServices.Marshal.Copy(new IntPtr(pointer), byteArray, 0, arrayLength);\n }\n\n // not working\n //fixed (ulong* src = data)\n //{\n // System.Runtime.InteropServices.Marshal.Copy(new IntPtr(src), byteArray, 0, arrayLength);\n //}\n\n // not working\n //Buffer.BlockCopy(data, 0, byteArray, 0, arrayLength);\n return byteArray;\n }\n\n private static unsafe byte[] ToByteArray(ushort[] data)\n {\n int arrayLength = data.Length;\n byte[] byteArray = new byte[2 * arrayLength];\n fixed (ushort* pointer = data)\n {\n fixed (byte* bytePointer = byteArray)\n {\n ushort* read = pointer;\n ushort* write = (ushort*)bytePointer;\n for (int i = 0; i < arrayLength; i++)\n {\n *write++ = *read++;\n }\n }\n }\n\n // not working\n //fixed (ushort* src = data)\n //{\n // System.Runtime.InteropServices.Marshal.Copy(new IntPtr(src), byteArray, 0, arrayLength);\n //}\n\n // not working\n //Buffer.BlockCopy(data, 0, byteArray, 0, arrayLength);\n\n return byteArray;\n }\n }\n}\n" }, { "path": "RaptorDB/Helper/WAHBitarray2.cs", "content": "using RaptorDB.Common;\nusing System;\nusing System.Collections.Generic;\n\nnamespace RaptorDB\n{\n public class WAHBitArray\n {\n public enum TYPE\n {\n WAH = 1,\n Bitarray = 0,\n Indexes = 2\n }\n\n public WAHBitArray()\n {\n _state = TYPE.Indexes;\n if (Global.UseLessMemoryStructures)\n _offsets = new SafeSortedList();\n else\n _offsets = new SafeDictionary();\n }\n\n public WAHBitArray(TYPE type, uint[] ints)\n {\n _state = type;\n switch (type)\n {\n case TYPE.WAH:\n _compressed = ints;\n Uncompress();\n _state = TYPE.Bitarray;\n _compressed = null;\n break;\n case TYPE.Bitarray:\n _uncompressed = ints;\n break;\n case TYPE.Indexes:\n if (Global.UseLessMemoryStructures)\n _offsets = new SafeSortedList();\n else\n _offsets = new SafeDictionary();\n //new Dictionary();\n foreach (var i in ints)\n _offsets.Add(i, true);\n break;\n }\n }\n\n private uint[] _compressed;\n private uint[] _uncompressed;\n //private Dictionary _offsets = new Dictionary();\n private IKV _offsets = null;// new SafeSortedList();\n private uint _curMax = 0;\n private TYPE _state;\n public bool isDirty = false;\n\n public WAHBitArray Copy()\n {\n lock (_lock)\n {\n uint[] i = GetBitArray();\n return new WAHBitArray(TYPE.Bitarray, i);\n }\n }\n\n public bool Get(int index)\n {\n lock (_lock)\n {\n if (_state == TYPE.Indexes)\n {\n bool b = false;\n var f = _offsets.TryGetValue((uint)index, out b);\n if (f)\n return b;\n else\n return false;\n }\n CheckBitArray();\n\n Resize(index);\n\n return internalGet(index);\n }\n }\n\n private object _lock = new object();\n public void Set(int index, bool val)\n {\n lock (_lock)\n {\n if (_state == TYPE.Indexes)\n {\n isDirty = true;\n\n if (val == true)\n {\n _offsets.Add((uint)index, true);\n // set max\n if (index > _curMax)\n _curMax = (uint)index;\n }\n else\n {\n _offsets.Remove((uint)index);\n }\n\n ChangeTypeIfNeeded();\n return;\n }\n CheckBitArray();\n\n Resize(index);\n\n internalSet(index, val);\n }\n }\n\n public int Length\n {\n set\n {\n lock (_lock)\n {\n if (_state == TYPE.Indexes)\n {\n // ignore\n return;\n }\n CheckBitArray();\n int c = value >> 5;\n c++;\n if (c > _uncompressed.Length)\n {\n uint[] ar = new uint[c];\n _uncompressed.CopyTo(ar, 0);\n _uncompressed = ar;\n }\n }\n }\n get\n {\n if (_state == TYPE.Indexes)\n {\n if (_offsets.Count() == 0) return 0;\n uint[] k = GetOffsets();\n\n uint l = k[k.Length - 1];\n return (int)l;\n }\n CheckBitArray();\n return _uncompressed.Length << 5;\n }\n }\n\n #region [ B I T O P E R T A I O N S ]\n public WAHBitArray And(WAHBitArray op)\n {\n lock (_lock)\n {\n uint[] left;\n uint[] right;\n prelogic(op, out left, out right);\n\n for (int i = 0; i < left.Length; i++)\n left[i] &= right[i];\n\n return new WAHBitArray(TYPE.Bitarray, left);\n }\n }\n\n public WAHBitArray AndNot(WAHBitArray op)\n {\n lock (_lock)\n {\n uint[] left;\n uint[] right;\n prelogic(op, out left, out right);\n\n for (int i = 0; i < left.Length; i++)\n left[i] &= ~right[i];\n\n return new WAHBitArray(TYPE.Bitarray, left);\n }\n }\n\n public WAHBitArray Or(WAHBitArray op)\n {\n lock (_lock)\n {\n uint[] left;\n uint[] right;\n prelogic(op, out left, out right);\n\n for (int i = 0; i < left.Length; i++)\n left[i] |= right[i];\n\n return new WAHBitArray(TYPE.Bitarray, left);\n }\n }\n\n public WAHBitArray Not(int size)\n {\n lock (_lock)\n {\n this.CheckBitArray();\n\n uint[] left = this.GetBitArray();\n int c = left.Length;\n int ms = size >> 5;\n if (size - (ms << 5) > 0)\n ms++; // include remainder\n if (ms > c)\n {\n var a = new uint[ms];\n Array.Copy(left, 0, a, 0, c);\n left = a;\n c = ms;\n }\n\n for (int i = 0; i < c; i++)\n left[i] = ~left[i];\n\n return new WAHBitArray(TYPE.Bitarray, left);\n }\n }\n\n public WAHBitArray Xor(WAHBitArray op)\n {\n lock (_lock)\n {\n uint[] left;\n uint[] right;\n prelogic(op, out left, out right);\n\n for (int i = 0; i < left.Length; i++)\n left[i] ^= right[i];\n\n return new WAHBitArray(TYPE.Bitarray, left);\n }\n }\n #endregion\n\n private static int BitCount(uint n)\n { // 32-bit recursive reduction using SWAR\n n -= ((n >> 1) & 0x55555555);\n n = (((n >> 2) & 0x33333333) + (n & 0x33333333));\n n = (((n >> 4) + n) & 0x0f0f0f0f);\n return (int)((n * 0x01010101) >> 24);\n }\n\n public long CountOnes()\n {\n if (_state == TYPE.Indexes)\n {\n return _offsets.Count();\n }\n\n long c = 0;\n CheckBitArray();\n\n foreach (uint i in _uncompressed)\n c += BitCount(i);\n\n return c;\n }\n\n public long CountZeros()\n {\n if (_state == TYPE.Indexes)\n {\n long ones = _offsets.Count();\n uint[] k = GetOffsets();\n long l = k[k.Length - 1];\n return l - ones;\n }\n\n CheckBitArray();\n int count = _uncompressed.Length << 5;\n long cc = CountOnes();\n\n return count - cc;\n }\n\n public void FreeMemory()\n {\n if (_state == TYPE.Bitarray)\n {\n if (_uncompressed != null)\n {\n lock (_lock)\n {\n _compressed = Compress(_uncompressed);\n _uncompressed = null;\n _state = TYPE.WAH;\n }\n }\n }\n }\n\n public uint[] GetCompressed(out TYPE type)\n {\n type = TYPE.WAH;\n\n ChangeTypeIfNeeded();\n if (_state == TYPE.Indexes)\n {\n type = TYPE.Indexes;\n return GetOffsets();\n }\n else if (_uncompressed == null)\n return new uint[] { 0 };\n\n uint[] d = Compress(_uncompressed);\n return d;\n }\n\n public IEnumerable GetBitIndexes()\n {\n if (_state == TYPE.Indexes)\n {\n foreach (int i in GetOffsets())\n yield return i;\n }\n else\n {\n CheckBitArray();\n int count = _uncompressed.Length;\n\n for (int i = 0; i < count; i++)\n {\n if (_uncompressed[i] > 0)\n {\n for (int j = 0; j < 32; j++)\n {\n bool b = internalGet((i << 5) + j);\n if (b == true)// ones)\n yield return (i << 5) + j;\n }\n }\n }\n }\n }\n\n #region [ P R I V A T E ]\n private uint[] GetOffsets()\n {\n uint[] k;\n lock (_lock)\n {\n k = new uint[_offsets.Count()];\n _offsets.Keys().CopyTo(k, 0);\n }\n Array.Sort(k);\n return k;\n }\n\n private void prelogic(WAHBitArray op, out uint[] left, out uint[] right)\n {\n this.CheckBitArray();\n\n left = this.GetBitArray();\n right = op.GetBitArray();\n int ic = left.Length;\n int uc = right.Length;\n if (ic > uc)\n {\n uint[] ar = new uint[ic];\n right.CopyTo(ar, 0);\n right = ar;\n }\n else if (ic < uc)\n {\n uint[] ar = new uint[uc];\n left.CopyTo(ar, 0);\n left = ar;\n }\n }\n\n internal uint[] GetBitArray()\n {\n lock (_lock)\n {\n if (_state == TYPE.Indexes)\n return UnpackOffsets();\n\n this.CheckBitArray();\n uint[] ui = new uint[_uncompressed.Length];\n _uncompressed.CopyTo(ui, 0);\n\n return ui;\n }\n }\n\n private uint[] UnpackOffsets()\n {\n // return bitmap uints \n uint max = 0;\n if (_offsets.Count() == 0) return new uint[0];\n uint[] k = GetOffsets();\n max = k[k.Length - 1];\n\n uint[] ints = new uint[(max >> 5) + 1];\n\n foreach (int index in k)\n {\n int pointer = ((int)index) >> 5;\n uint mask = (uint)1 << (31 - // high order bit set\n ((int)index % 32));\n\n ints[pointer] |= mask;\n }\n\n return ints;\n }\n\n private void ChangeTypeIfNeeded()\n {\n if (_state != TYPE.Indexes)\n return;\n\n uint T = (_curMax >> 5) + 1;\n int c = _offsets.Count();\n if (c > T && c > Global.BitmapOffsetSwitchOverCount)\n {\n // change type to WAH\n _state = TYPE.Bitarray;\n _uncompressed = new uint[0];\n // create bitmap\n foreach (var i in _offsets.Keys())\n Set((int)i, true);\n // clear list\n if (Global.UseLessMemoryStructures)\n _offsets = new SafeSortedList();\n else\n _offsets = new SafeDictionary();\n //new Dictionary();\n }\n }\n\n private void Resize(int index)\n {\n if (_state == TYPE.Indexes)\n return;\n int c = index >> 5;\n c++;\n if (_uncompressed == null)\n {\n _uncompressed = new uint[c];\n return;\n }\n if (c > _uncompressed.Length)\n {\n uint[] ar = new uint[c];\n _uncompressed.CopyTo(ar, 0);\n _uncompressed = ar;\n }\n }\n\n private static void ResizeAsNeeded(List list, int index)\n {\n int count = index >> 5;\n\n while (list.Count < count)\n list.Add(0);\n }\n\n private void internalSet(int index, bool val)\n {\n isDirty = true;\n int pointer = index >> 5;\n uint mask = (uint)1 << (31 - // high order bit set\n (index % 32));\n\n if (val)\n _uncompressed[pointer] |= mask;\n else\n _uncompressed[pointer] &= ~mask;\n }\n\n private bool internalGet(int index)\n {\n int pointer = index >> 5;\n uint mask = (uint)1 << (31 - // high order bit get\n (index % 32));\n\n if (pointer < _uncompressed.Length)\n return (_uncompressed[pointer] & mask) != 0;\n else\n return false;\n }\n\n private void CheckBitArray()\n {\n if (_state == TYPE.Bitarray)\n return;\n\n if (_state == TYPE.WAH)\n {\n _uncompressed = new uint[0];\n Uncompress();\n _state = TYPE.Bitarray;\n _compressed = null;\n return;\n }\n }\n\n #region compress / uncompress\n private static uint Take31Bits(uint[] data, int index)\n {\n ulong l1 = 0;\n ulong l2 = 0;\n ulong l = 0;\n ulong ret = 0;\n int off = (index % 32);\n int pointer = index >> 5;\n\n l1 = data[pointer];\n pointer++;\n if (pointer < data.Length)\n l2 = data[pointer];\n\n l = (l1 << 32) + l2;\n ret = (l >> (33 - off)) & 0x7fffffff;\n\n return (uint)ret;\n }\n\n private static uint[] Compress(uint[] data)\n {\n List compressed = new List();\n uint zeros = 0;\n uint ones = 0;\n int count = data.Length << 5;\n int i = 0;\n while (i < count)//for (int i = 0; i < count;)\n {\n uint num = Take31Bits(data, i);\n i += 31;\n if (num == 0) // all zero\n {\n zeros += 31;\n FlushOnes(compressed, ref ones);\n }\n else if (num == 0x7fffffff) // all ones\n {\n ones += 31;\n FlushZeros(compressed, ref zeros);\n }\n else // literal\n {\n FlushOnes(compressed, ref ones);\n FlushZeros(compressed, ref zeros);\n compressed.Add(num);\n }\n }\n FlushOnes(compressed, ref ones);\n FlushZeros(compressed, ref zeros);\n return compressed.ToArray();\n }\n\n private static void FlushOnes(List compressed, ref uint ones)\n {\n if (ones > 0)\n {\n uint n = 0xc0000000 + ones;\n ones = 0;\n compressed.Add(n);\n }\n }\n\n private static void FlushZeros(List compressed, ref uint zeros)\n {\n if (zeros > 0)\n {\n uint n = 0x80000000 + zeros;\n zeros = 0;\n compressed.Add(n);\n }\n }\n\n private static void Write31Bits(List list, int index, uint val)\n {\n ResizeAsNeeded(list, index + 32);\n\n int off = (index % 32);\n int pointer = index >> 5;\n\n if (pointer >= list.Count - 1)\n list.Add(0);\n\n ulong l = ((ulong)list[pointer] << 32) + list[pointer + 1];\n l |= (ulong)val << (33 - off);\n\n list[pointer] = (uint)(l >> 32);\n list[pointer + 1] = (uint)(l & 0xffffffff);\n }\n\n private void WriteOnes(List list, int index, uint count)\n {\n ResizeAsNeeded(list, index);\n\n int off = index % 32;\n int pointer = index >> 5;\n int ccount = (int)count;\n int indx = index;\n int x = 32 - off;\n\n if (pointer >= list.Count)\n list.Add(0);\n\n if (ccount > x)//|| x == 32) //current pointer\n {\n list[pointer] |= (uint)((0xffffffff >> off));\n ccount -= x;\n indx += x;\n }\n else\n {\n list[pointer] |= (uint)((0xffffffff << (32 - ccount)) >> off);\n ccount = 0;\n }\n\n bool checklast = true;\n while (ccount >= 32)//full ints\n {\n if (checklast && list[list.Count - 1] == 0)\n {\n list.RemoveAt(list.Count - 1);\n checklast = false;\n }\n\n list.Add(0xffffffff);\n ccount -= 32;\n indx += 32;\n }\n int p = indx >> 5;\n off = indx % 32;\n if (ccount > 0)\n {\n uint i = 0xffffffff << (32 - ccount);\n if (p > (list.Count - 1)) //remaining\n list.Add(i);\n else\n list[p] |= (uint)(i >> off);\n }\n }\n\n private void Uncompress()\n {\n int index = 0;\n List list = new List();\n if (_compressed == null)\n return;\n\n foreach (uint ci in _compressed)\n {\n if ((ci & 0x80000000) == 0) // literal\n {\n Write31Bits(list, index, ci);\n index += 31;\n }\n else\n {\n uint count = ci & 0x3fffffff;\n if ((ci & 0x40000000) != 0) // ones count\n WriteOnes(list, index, count);\n\n index += (int)count;\n }\n }\n ResizeAsNeeded(list, index);\n _uncompressed = list.ToArray();\n }\n #endregion\n\n #endregion\n\n internal static WAHBitArray Fill(int count)\n {\n if (count > 0)\n {\n int c = count >> 5;\n int r = count % 32;\n if (r > 0)\n c++;\n uint[] ints = new uint[c];\n for (int i = 0; i < c; i++)\n ints[i] = 0xffffffff;\n if (r > 0)\n ints[c - 1] = 0xffffffff << (31 - r);\n return new WAHBitArray(TYPE.Bitarray, ints);\n }\n return new WAHBitArray();\n }\n\n internal int GetFirst()\n {\n foreach (var i in GetBitIndexes())\n return i;\n return 0;\n }\n }\n}\n" }, { "path": "RaptorDB/Indexes/BitmapIndex.cs", "content": "using RaptorDB.Common;\nusing System;\nusing System.Collections.Generic;\nusing System.IO;\nusing System.Threading;\n\nnamespace RaptorDB\n{\n internal class BitmapIndex\n {\n public BitmapIndex(string path, string filename)\n {\n if (Global.UseLessMemoryStructures)\n _cache = new SafeSortedList();\n else\n _cache = new SafeDictionary();\n\n _FileName = Path.GetFileNameWithoutExtension(filename);\n _Path = path;\n if (_Path.EndsWith(Path.DirectorySeparatorChar.ToString()) == false)\n _Path += Path.DirectorySeparatorChar.ToString();\n\n Initialize();\n }\n\n class L : IDisposable\n {\n BitmapIndex _sc;\n public L(BitmapIndex sc)\n {\n _sc = sc;\n _sc.CheckInternalOP();\n }\n void IDisposable.Dispose()\n {\n _sc.Done();\n }\n }\n\n private string _recExt = \".mgbmr\";\n private string _bmpExt = \".mgbmp\";\n private string _FileName = \"\";\n private string _Path = \"\";\n private FileStream _bitmapFileWriteOrg;\n private BufferedStream _bitmapFileWrite;\n private FileStream _bitmapFileRead;\n private FileStream _recordFileRead;\n private FileStream _recordFileWriteOrg;\n private BufferedStream _recordFileWrite;\n private long _lastBitmapOffset = 0;\n private int _lastRecordNumber = 0;\n //private SafeDictionary _cache = new SafeDictionary();\n private IKV _cache = null;// new SafeSortedList();\n private ILog log = LogManager.GetLogger(typeof(BitmapIndex));\n private bool _stopOperations = false;\n private bool _shutdownDone = false;\n private int _workingCount = 0;\n private bool _isDirty = false;\n\n #region\n public void Shutdown()\n {\n using (new L(this))\n {\n log.Debug(\"Shutdown BitmapIndex\");\n\n InternalShutdown();\n }\n }\n\n public int GetFreeRecordNumber()\n {\n using (new L(this))\n {\n int i = _lastRecordNumber++;\n\n _cache.Add(i, new MGRB());\n return i;\n }\n }\n\n public void Commit(bool freeMemory)\n {\n if (_isDirty == false)\n return;\n using (new L(this))\n {\n log.Debug(\"writing \" + _FileName);\n int[] keys = _cache.Keys();\n Array.Sort(keys);\n\n foreach (int k in keys)\n {\n MGRB bmp = null;\n if (_cache.TryGetValue(k, out bmp) && bmp.isDirty)\n {\n bmp.Optimize();\n SaveBitmap(k, bmp);\n bmp.isDirty = false;\n }\n }\n Flush();\n if (freeMemory)\n {\n if (Global.UseLessMemoryStructures)\n _cache = new SafeSortedList();\n else\n _cache = new SafeDictionary();\n log.Debug(\" freeing cache\");\n }\n _isDirty = false;\n }\n }\n\n public void SetDuplicate(int bitmaprecno, int record)\n {\n using (new L(this))\n {\n MGRB ba = null;\n\n ba = internalGetBitmap(bitmaprecno); //GetBitmap(bitmaprecno);\n\n ba.Set(record, true);\n _isDirty = true;\n }\n }\n\n public MGRB GetBitmap(int recno)\n {\n using (new L(this))\n {\n return internalGetBitmap(recno);\n }\n }\n\n private object _oplock = new object();\n public void Optimize()\n {\n lock (_oplock)\n lock (_readlock)\n lock (_writelock)\n {\n _stopOperations = true;\n while (_workingCount > 0) Thread.SpinWait(1);\n Flush();\n\n if (File.Exists(_Path + _FileName + \"$\" + _bmpExt))\n File.Delete(_Path + _FileName + \"$\" + _bmpExt);\n\n if (File.Exists(_Path + _FileName + \"$\" + _recExt))\n File.Delete(_Path + _FileName + \"$\" + _recExt);\n\n Stream _newrec = new FileStream(_Path + _FileName + \"$\" + _recExt, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite);\n Stream _newbmp = new FileStream(_Path + _FileName + \"$\" + _bmpExt, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite);\n\n long newoffset = 0;\n int c = (int)(_recordFileRead.Length / 8);\n for (int i = 0; i < c; i++)\n {\n long offset = ReadRecordOffset(i);\n\n byte[] b = ReadBMPDataForOptimize(offset);\n if (b == null)\n {\n _stopOperations = false;\n throw new Exception(\"bitmap index file is corrupted\");\n }\n\n _newrec.Write(Helper.GetBytes(newoffset, false), 0, 8);\n newoffset += b.Length;\n _newbmp.Write(b, 0, b.Length);\n\n }\n _newbmp.Flush();\n _newbmp.Close();\n _newrec.Flush();\n _newrec.Close();\n\n InternalShutdown();\n\n File.Delete(_Path + _FileName + _bmpExt);\n File.Delete(_Path + _FileName + _recExt);\n File.Move(_Path + _FileName + \"$\" + _bmpExt, _Path + _FileName + _bmpExt);\n File.Move(_Path + _FileName + \"$\" + _recExt, _Path + _FileName + _recExt);\n\n Initialize();\n _stopOperations = false;\n }\n }\n\n internal void FreeMemory()\n {\n try\n {\n List free = new List();\n foreach (var k in _cache.Keys())\n {\n var val = _cache.GetValue(k);\n if (val.isDirty == false)\n free.Add(k);\n }\n log.Info(\"releasing bmp count = \" + free.Count + \" out of \" + _cache.Count());\n foreach (int i in free)\n _cache.Remove(i);\n }\n catch (Exception ex)\n {\n log.Error(ex);\n }\n }\n #endregion\n\n #region [ P R I V A T E ]\n private long ReadRecordOffset(int recnum)\n {\n byte[] b = new byte[8];\n long off = ((long)recnum) * 8;\n _recordFileRead.Seek(off, SeekOrigin.Begin);\n _recordFileRead.Read(b, 0, 8);\n return Helper.ToInt64(b, 0);\n }\n\n private void Initialize()\n {\n _recordFileRead = new FileStream(_Path + _FileName + _recExt, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite);\n _recordFileWriteOrg = new FileStream(_Path + _FileName + _recExt, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite);\n _recordFileWrite = new BufferedStream(_recordFileWriteOrg);\n\n _bitmapFileRead = new FileStream(_Path + _FileName + _bmpExt, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite);\n _bitmapFileWriteOrg = new FileStream(_Path + _FileName + _bmpExt, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite);\n _bitmapFileWrite = new BufferedStream(_bitmapFileWriteOrg);\n\n _bitmapFileWrite.Seek(0L, SeekOrigin.End);\n _lastBitmapOffset = _bitmapFileWrite.Length;\n _lastRecordNumber = (int)(_recordFileRead.Length / 8);\n _shutdownDone = false;\n }\n\n private void InternalShutdown()\n {\n bool d1 = false;\n bool d2 = false;\n\n if (_shutdownDone == false)\n {\n Flush();\n if (_recordFileWrite.Length == 0) d1 = true;\n if (_bitmapFileWrite.Length == 0) d2 = true;\n _recordFileRead.Close();\n _bitmapFileRead.Close();\n _bitmapFileWriteOrg.Close();\n _recordFileWriteOrg.Close();\n _recordFileWrite.Close();\n _bitmapFileWrite.Close();\n if (d1)\n File.Delete(_Path + _FileName + _recExt);\n if (d2)\n File.Delete(_Path + _FileName + _bmpExt);\n _recordFileWrite = null;\n _recordFileRead = null;\n _bitmapFileRead = null;\n _bitmapFileWrite = null;\n _recordFileRead = null;\n _recordFileWrite = null;\n _shutdownDone = true;\n }\n }\n\n private void Flush()\n {\n if (_shutdownDone)\n return;\n\n if (_recordFileWrite != null)\n _recordFileWrite.Flush();\n\n if (_bitmapFileWrite != null)\n _bitmapFileWrite.Flush();\n\n if (_recordFileRead != null)\n _recordFileRead.Flush();\n\n if (_bitmapFileRead != null)\n _bitmapFileRead.Flush();\n\n if (_bitmapFileWriteOrg != null)\n _bitmapFileWriteOrg.Flush();\n\n if (_recordFileWriteOrg != null)\n _recordFileWriteOrg.Flush();\n }\n\n private object _readlock = new object();\n private MGRB internalGetBitmap(int recno)\n {\n lock (_readlock)\n {\n MGRB ba = new MGRB();\n if (recno == -1)\n return ba;\n\n if (_cache.TryGetValue(recno, out ba))\n {\n return ba;\n }\n else\n {\n long offset = 0;\n //if (_offsetCache.TryGetValue(recno, out offset) == false)\n {\n offset = ReadRecordOffset(recno);\n // _offsetCache.Add(recno, offset);\n }\n ba = LoadBitmap(offset);\n\n _cache.Add(recno, ba);\n\n return ba;\n }\n }\n }\n\n private object _writelock = new object();\n private void SaveBitmap(int recno, MGRB bmp)\n {\n lock (_writelock)\n {\n long offset = SaveBitmapToFile(bmp);\n //long v;\n //if (_offsetCache.TryGetValue(recno, out v))\n // _offsetCache[recno] = offset;\n //else\n // _offsetCache.Add(recno, offset);\n\n long pointer = ((long)recno) * 8;\n _recordFileWrite.Seek(pointer, SeekOrigin.Begin);\n byte[] b = new byte[8];\n b = Helper.GetBytes(offset, false);\n _recordFileWrite.Write(b, 0, 8);\n }\n }\n\n //-----------------------------------------------------------------\n // new format \n // 0 : b \n // 1 : m\n // 2 : type 0 = uncompressed, 1 = compressed\n // 3 : data size (int)\n // 8 : data bytes\n private byte _hdrlen = 2 + 4 + 1;\n private long SaveBitmapToFile(MGRB bmp)\n {\n long off = _lastBitmapOffset;\n var dat = bmp.Serialize();\n var hdr = new byte[_hdrlen];\n var b = fastBinaryJSON.BJSON.ToBJSON(dat, new fastBinaryJSON.BJSONParameters { UseExtensions = false });\n hdr[0] = (byte)'b';\n hdr[1] = (byte)'m';\n hdr[2] = 0; // uncompressed\n\n if (Global.CompressBitmapBytes)\n {\n hdr[2] = 1;\n b = MiniLZO.Compress(b);\n }\n\n var s = Helper.GetBytes(b.Length, false);\n Buffer.BlockCopy(s, 0, hdr, 3, 4);\n\n _bitmapFileWrite.Write(hdr, 0, hdr.Length);\n _lastBitmapOffset += hdr.Length;\n\n _bitmapFileWrite.Write(b, 0, b.Length);\n _lastBitmapOffset += b.Length;\n\n return off;\n }\n\n private byte[] ReadBMPDataForOptimize(long offset)\n {\n // return data + header\n _bitmapFileRead.Seek(offset, SeekOrigin.Begin);\n\n byte[] hdr = new byte[_hdrlen];\n\n _bitmapFileRead.Read(hdr, 0, _hdrlen);\n if (hdr[0] == (byte)'b' && hdr[1] == (byte)'m')\n {\n int c = Helper.ToInt32(hdr, 3);\n var data = new byte[c + _hdrlen];\n Buffer.BlockCopy(hdr, 0, data, 0, _hdrlen);\n _bitmapFileRead.Read(data, _hdrlen, c);\n return data;\n }\n return null;\n }\n\n private MGRB LoadBitmap(long offset)\n {\n MGRB bc = new MGRB();\n if (offset == -1)\n return bc;\n FileStream bmp = _bitmapFileRead;\n bmp.Seek(offset, SeekOrigin.Begin);\n var hdr = new byte[_hdrlen];\n bmp.Read(hdr, 0, hdr.Length);\n if (hdr[0] == (byte)'b' && hdr[1] == (byte)'m')\n {\n int c = Helper.ToInt32(hdr, 3);\n var b = new byte[c];\n bmp.Read(b, 0, c);\n if (hdr[2] == 1)\n b = MiniLZO.Decompress(b);\n bc.Deserialize(fastBinaryJSON.BJSON.ToObject(b));\n }\n else\n log.Error(\"bitmap not recognized\");\n\n return bc;\n }\n\n#pragma warning disable 642\n private void CheckInternalOP()\n {\n if (_stopOperations)\n lock (_oplock) { } // yes! this is good\n Interlocked.Increment(ref _workingCount);\n }\n#pragma warning restore 642\n\n private void Done()\n {\n Interlocked.Decrement(ref _workingCount);\n }\n #endregion\n }\n}\n" }, { "path": "RaptorDB/Indexes/Cache.cs", "content": "using System;\nusing System.Xml.Serialization;\n\nnamespace RaptorDB\n{\n public enum OPERATION\n {\n AND,\n OR,\n ANDNOT\n }\n\n public class Document\n {\n public string FullName;\n public DateTime Created;\n public DateTime Modified;\n public long Length;\n public string Extension;\n public Document()\n {\n DocNumber = -1;\n }\n public Document(string filename, string text)\n {\n FileName = filename;\n Text = text;\n DocNumber = -1;\n }\n public int DocNumber { get; set; }\n [XmlIgnore]\n public string Text { get; set; }\n public string FileName { get; set; }\n public string Abstract { get; set; }\n\n public override string ToString()\n {\n return FileName;\n }\n }\n}\n" }, { "path": "RaptorDB/Indexes/Hoot.cs", "content": "using System;\nusing System.Collections.Generic;\nusing System.Text;\nusing System.IO;\nusing System.Text.RegularExpressions;\nusing RaptorDB.Common;\n\nnamespace RaptorDB\n{\n public class Hoot\n {\n public Hoot(string IndexPath, string FileName, bool DocMode) : this(IndexPath, FileName, DocMode, new tokenizer())\n {\n }\n public Hoot(string IndexPath, string FileName, bool DocMode, ITokenizer tokenizer)\n {\n if (tokenizer != null)\n _tokenizer = tokenizer;\n else\n _tokenizer = new tokenizer();\n _Path = IndexPath;\n _FileName = FileName;\n _docMode = DocMode;\n if (_Path.EndsWith(Path.DirectorySeparatorChar.ToString()) == false) _Path += Path.DirectorySeparatorChar;\n Directory.CreateDirectory(IndexPath);\n\n _log.Debug(\"Starting hOOt....\");\n _log.Debug(\"Storage Folder = \" + _Path);\n\n if (DocMode)\n {\n _docs = new KeyStoreString(_Path + \"files.docs\", false);\n // read deleted\n _deleted = new BoolIndex(_Path, \"_deleted\", \".hoot\");\n _lastDocNum = (int)_docs.Count();\n }\n _bitmaps = new BitmapIndex(_Path, _FileName + \"_hoot.bmp\");\n // read words\n LoadWords();\n }\n private ITokenizer _tokenizer;\n private SafeDictionary _words = new SafeDictionary();\n //private SafeSortedList _words = new SafeSortedList();\n private BitmapIndex _bitmaps;\n private BoolIndex _deleted;\n private ILog _log = LogManager.GetLogger(typeof(Hoot));\n private int _lastDocNum = 0;\n private string _FileName = \"words\";\n private string _Path = \"\";\n private KeyStoreString _docs;\n private bool _docMode = false;\n private bool _wordschanged = true;\n private bool _shutdowndone = false;\n private object _lock = new object();\n\n public string[] Words\n {\n get { checkloaded(); return _words.Keys(); }\n }\n\n public int WordCount\n {\n get { checkloaded(); return _words.Count(); }\n }\n\n public int DocumentCount\n {\n get { checkloaded(); return _lastDocNum - (int)_deleted.GetBits().CountOnes(); }\n }\n\n public string IndexPath { get { return _Path; } }\n\n public void Save()\n {\n lock (_lock)\n InternalSave();\n }\n\n public void Index(int recordnumber, string text)\n {\n checkloaded();\n AddtoIndex(recordnumber, text);\n }\n\n public MGRB Query(string filter, int maxsize)\n {\n checkloaded();\n return ExecutionPlan(filter, maxsize);\n }\n\n public int Index(Document doc, bool deleteold)\n {\n checkloaded();\n _log.Info(\"indexing doc : \" + doc.FileName);\n DateTime dt = FastDateTime.Now;\n\n if (deleteold && doc.DocNumber > -1)\n _deleted.Set(true, doc.DocNumber);\n\n if (deleteold == true || doc.DocNumber == -1)\n doc.DocNumber = _lastDocNum++;\n\n // save doc to disk\n string dstr = fastJSON.JSON.ToJSON(doc, new fastJSON.JSONParameters { UseExtensions = false });\n _docs.Set(doc.FileName.ToLower(), fastJSON.Reflection.UnicodeGetBytes(dstr));\n\n _log.Info(\"writing doc to disk (ms) = \" + FastDateTime.Now.Subtract(dt).TotalMilliseconds);\n\n dt = FastDateTime.Now;\n // index doc\n AddtoIndex(doc.DocNumber, doc.Text);\n _log.Info(\"indexing time (ms) = \" + FastDateTime.Now.Subtract(dt).TotalMilliseconds);\n\n return _lastDocNum;\n }\n\n public IEnumerable FindRows(string filter)\n {\n checkloaded();\n MGRB bits = ExecutionPlan(filter, _docs.RecordCount());\n // enumerate records\n return bits.GetBitIndexes();\n }\n\n public IEnumerable FindDocuments(string filter)\n {\n checkloaded();\n MGRB bits = ExecutionPlan(filter, _docs.RecordCount());\n // enumerate documents\n foreach (int i in bits.GetBitIndexes())\n {\n if (i > _lastDocNum - 1)\n break;\n string b = _docs.ReadData(i);\n T d = fastJSON.JSON.ToObject(b, new fastJSON.JSONParameters { ParametricConstructorOverride = true });\n\n yield return d;\n }\n }\n\n public IEnumerable FindDocumentFileNames(string filter)\n {\n checkloaded();\n MGRB bits = ExecutionPlan(filter, _docs.RecordCount());\n // enumerate documents\n foreach (int i in bits.GetBitIndexes())\n {\n if (i > _lastDocNum - 1)\n break;\n string b = _docs.ReadData(i);\n var d = (Dictionary)fastJSON.JSON.Parse(b);\n\n yield return d[\"FileName\"].ToString();\n }\n }\n\n public void RemoveDocument(int number)\n {\n // add number to deleted bitmap\n _deleted.Set(true, number);\n }\n\n public bool RemoveDocument(string filename)\n {\n // remove doc based on filename\n byte[] b;\n if (_docs.Get(filename.ToLower(), out b))\n {\n Document d = fastJSON.JSON.ToObject(fastJSON.Reflection.UnicodeGetString(b));\n RemoveDocument(d.DocNumber);\n return true;\n }\n return false;\n }\n\n public bool IsIndexed(string filename)\n {\n byte[] b;\n return _docs.Get(filename.ToLower(), out b);\n }\n\n public void OptimizeIndex()\n {\n lock (_lock)\n {\n InternalSave();\n //_bitmaps.Commit(false);\n _bitmaps.Optimize();\n }\n }\n\n #region [ P R I V A T E M E T H O D S ]\n\n private void checkloaded()\n {\n if (_wordschanged == false)\n {\n LoadWords();\n }\n }\n\n private MGRB ExecutionPlan(string filter, int maxsize)\n {\n //_log.Debug(\"query : \" + filter);\n DateTime dt = FastDateTime.Now;\n // query indexes\n string[] words = filter.Split(' ');\n //bool defaulttoand = true;\n //if (filter.IndexOfAny(new char[] { '+', '-' }, 0) > 0)\n // defaulttoand = false;\n\n MGRB found = null;// MGRB.Fill(maxsize); \n\n foreach (string s in words)\n {\n int c;\n bool not = false;\n string word = s;\n if (s == \"\") continue;\n\n OPERATION op = OPERATION.AND;\n //if (defaulttoand)\n // op = OPERATION.AND;\n\n if (word.StartsWith(\"+\"))\n {\n op = OPERATION.OR;\n word = s.Replace(\"+\", \"\");\n }\n\n if (word.StartsWith(\"-\"))\n {\n op = OPERATION.ANDNOT;\n word = s.Replace(\"-\", \"\");\n not = true;\n if (found == null) // leading with - -> \"-oak hill\"\n {\n found = MGRB.Fill(maxsize);\n }\n }\n\n if (word.Contains(\"*\") || word.Contains(\"?\"))\n {\n MGRB wildbits = new MGRB();\n\n // do wildcard search\n Regex reg = new Regex(\"^\" + word.Replace(\"*\", \".*\").Replace(\"?\", \".\") + \"$\", RegexOptions.IgnoreCase);\n foreach (string key in _words.Keys())\n {\n if (reg.IsMatch(key))\n {\n _words.TryGetValue(key, out c);\n MGRB ba = _bitmaps.GetBitmap(c);\n\n wildbits = DoBitOperation(wildbits, ba, OPERATION.OR, maxsize);\n }\n }\n if (found == null)\n found = wildbits;\n else\n {\n if (not) // \"-oak -*l\"\n found = found.AndNot(wildbits);\n else if (op == OPERATION.AND)\n found = found.And(wildbits);\n else\n found = found.Or(wildbits);\n }\n }\n else if (_words.TryGetValue(word.ToLowerInvariant(), out c))\n {\n // bits logic\n MGRB ba = _bitmaps.GetBitmap(c);\n found = DoBitOperation(found, ba, op, maxsize);\n }\n else if (op == OPERATION.AND)\n found = new MGRB();\n }\n if (found == null)\n return new MGRB();\n\n // remove deleted docs\n MGRB ret;\n if (_docMode)\n ret = found.AndNot(_deleted.GetBits());\n else\n ret = found;\n //_log.Debug(\"query time (ms) = \" + FastDateTime.Now.Subtract(dt).TotalMilliseconds);\n return ret;\n }\n\n private static MGRB DoBitOperation(MGRB bits, MGRB c, OPERATION op, int maxsize)\n {\n if (bits != null)\n {\n switch (op)\n {\n case OPERATION.AND:\n bits = bits.And(c);\n break;\n case OPERATION.OR:\n bits = bits.Or(c);\n break;\n case OPERATION.ANDNOT:\n bits = bits.And(c.Not(maxsize));\n break;\n }\n }\n else\n bits = c;\n return bits;\n }\n\n private void InternalSave()\n {\n _log.Info(\"saving index...\");\n DateTime dt = FastDateTime.Now;\n // save deleted\n if (_deleted != null)\n _deleted.SaveIndex();\n\n // save docs \n if (_docMode)\n _docs.SaveIndex();\n\n if (_bitmaps != null)\n _bitmaps.Commit(true);\n\n if (_words != null && _wordschanged == true)\n {\n // save words and bitmaps\n using (FileStream words = new FileStream(_Path + _FileName + \".words\", FileMode.Create))\n {\n using (BinaryWriter bw = new BinaryWriter(words, Encoding.UTF8))\n {\n foreach (string key in _words.Keys())\n {\n bw.Write(key);\n bw.Write(_words[key]);\n }\n }\n }\n _wordschanged = false;\n }\n _log.Info(\"save time (ms) = \" + FastDateTime.Now.Subtract(dt).TotalMilliseconds);\n }\n\n private void LoadWords()\n {\n lock (_lock)\n {\n if (_words == null)\n _words = new SafeDictionary();\n // new SafeSortedList();\n if (File.Exists(_Path + _FileName + \".words\") == false)\n return;\n // load words\n using (FileStream words = new FileStream(_Path + _FileName + \".words\", FileMode.Open))\n {\n if (words.Length == 0)\n return;\n using (BinaryReader br = new BinaryReader(words, Encoding.UTF8))\n {\n string s = br.ReadString();\n while (s != \"\")\n {\n int off = br.ReadInt32();\n _words.Add(s, off);\n try\n {\n s = br.ReadString();\n }\n catch { s = \"\"; }\n }\n }\n }\n //byte[] b = File.ReadAllBytes(_Path + _FileName + \".words\");\n //if (b.Length == 0)\n // return;\n //MemoryStream ms = new MemoryStream(b);\n //BinaryReader br = new BinaryReader(ms, Encoding.UTF8);\n //string s = br.ReadString();\n //while (s != \"\")\n //{\n // int off = br.ReadInt32();\n // _words.Add(s, off);\n // try\n // {\n // s = br.ReadString();\n // }\n // catch { s = \"\"; }\n //}\n _log.Debug(\"Word Count = \" + _words.Count());\n _wordschanged = true;\n }\n }\n\n private void AddtoIndex(int recnum, string text)\n {\n if (text == \"\" || text == null)\n return;\n text = text.ToLowerInvariant(); // lowercase index \n string[] keys;\n if (_docMode)\n {\n //_log.Debug(\"text size = \" + text.Length);\n Dictionary wordfreq = _tokenizer.GenerateWordFreq(text);\n //_log.Debug(\"word count = \" + wordfreq.Count);\n var kk = wordfreq.Keys;\n keys = new string[kk.Count];\n kk.CopyTo(keys, 0);\n }\n else\n {\n keys = text.Split(' ');\n }\n\n foreach (string key in keys)\n {\n if (key == \"\")\n continue;\n\n int bmp;\n if (_words.TryGetValue(key, out bmp))\n {\n _bitmaps.GetBitmap(bmp).Set(recnum, true);\n }\n else\n {\n bmp = _bitmaps.GetFreeRecordNumber();\n _bitmaps.SetDuplicate(bmp, recnum);\n _words.Add(key, bmp);\n }\n }\n _wordschanged = true;\n }\n\n \n #endregion\n\n public void Shutdown()\n {\n lock (_lock)\n {\n if (_shutdowndone == true)\n return;\n\n InternalSave();\n if (_deleted != null)\n {\n _deleted.SaveIndex();\n _deleted.Shutdown();\n _deleted = null;\n }\n\n if (_bitmaps != null)\n {\n _bitmaps.Commit(Global.FreeBitmapMemoryOnSave);\n _bitmaps.Shutdown();\n _bitmaps = null;\n }\n\n if (_docMode)\n _docs.Shutdown();\n\n _shutdowndone = true;\n }\n }\n\n public void FreeMemory()\n {\n lock (_lock)\n {\n InternalSave();\n\n if (_deleted != null)\n _deleted.FreeMemory();\n\n if (_bitmaps != null)\n _bitmaps.FreeMemory();\n\n if (_docs != null)\n _docs.FreeMemory();\n\n //_words = null;// new SafeSortedList();\n //_loaded = false;\n }\n }\n\n public T Fetch(int docnum)\n {\n string b = _docs.ReadData(docnum);\n return fastJSON.JSON.ToObject(b);\n }\n }\n}\n" }, { "path": "RaptorDB/Indexes/IIndex.cs", "content": "namespace RaptorDB\n{\n internal enum RDBExpression\n {\n Equal,\n Greater,\n GreaterEqual,\n Less,\n LessEqual,\n NotEqual,\n Between,\n Contains\n }\n\n internal interface IIndex\n {\n void Set(object key, int recnum);\n MGRB Query(object fromkey, object tokey, int maxsize);\n MGRB Query(RDBExpression ex, object from , int maxsize);\n void FreeMemory();\n void Shutdown();\n void SaveIndex();\n object[] GetKeys();\n }\n}\n" }, { "path": "RaptorDB/Indexes/ITokenizer.cs", "content": "using System.Collections.Generic;\n\nnamespace RaptorDB\n{\n public interface ITokenizer\n {\n Dictionary GenerateWordFreq(string text);\n }\n}" }, { "path": "RaptorDB/Indexes/IndexFile.cs", "content": "using System;\nusing System.Collections.Generic;\nusing System.IO;\nusing RaptorDB.Common;\nusing System.Threading;\nusing fastBinaryJSON;\n\nnamespace RaptorDB\n{\n internal class IndexFile\n {\n FileStream _file = null;\n private byte[] _FileHeader = new byte[] {\n (byte)'M', (byte)'G', (byte)'I',\n 0, // 3 = [keysize] max 255\n 0,0, // 4 = [node size] max 65536\n 0,0,0,0, // 6 = [root page num]\n 0, // 10 = Index file type : 0=mgindex 1=mgindex+strings (key = firstallocblock)\n 0,0,0,0 // 11 = last record number indexed \n };\n\n private byte[] _BlockHeader = new byte[] {\n (byte)'P',(byte)'A',(byte)'G',(byte)'E',\n 0, // 4 = [Flag] = 0=page 1=page list \n 0,0, // 5 = [item count] \n 0,0,0,0, // 7 = reserved \n 0,0,0,0 // 11 = [right page number] / [next page number]\n };\n\n internal byte _maxKeySize;\n internal ushort _PageNodeCount = 5000;\n private int _LastPageNumber = 1; // 0 = page list\n private int _PageLength;\n private int _rowSize;\n private bool _allowDups = true;\n ILog log = LogManager.GetLogger(typeof(IndexFile));\n private BitmapIndex _bitmap;\n IGetBytes _T = null;\n private object _fileLock = new object();\n\n private StringHF _strings;\n private bool _externalStrings = false;\n //private List _pagelistalllocblock = null;\n private string _FileName = \"\";\n\n public IndexFile(string filename, byte maxKeySize)//, ushort pageNodeCount)\n {\n _T = RDBDataType.ByteHandler();\n if (typeof(T) == typeof(string) && Global.EnableOptimizedStringIndex)\n {\n _externalStrings = true;\n _maxKeySize = 4;// blocknum:int\n }\n else\n _maxKeySize = maxKeySize;\n\n _PageNodeCount = Global.PageItemCount;// pageNodeCount;\n _rowSize = (_maxKeySize + 1 + 4 + 4);\n\n _FileName = filename.Substring(0, filename.LastIndexOf('.'));\n string path = Path.GetDirectoryName(filename);\n\n Directory.CreateDirectory(path);\n if (File.Exists(filename))\n {\n // if file exists open and read header\n _file = File.Open(filename, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite);\n ReadFileHeader();\n if (_externalStrings == false)// if the file says different\n {\n _rowSize = (_maxKeySize + 1 + 4 + 4);\n }\n // compute last page number from file length \n _PageLength = (_BlockHeader.Length + _rowSize * (_PageNodeCount));\n _LastPageNumber = (int)((_file.Length - _FileHeader.Length) / _PageLength);\n }\n else\n {\n // else create new file\n _file = File.Open(filename, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite);\n\n _PageLength = (_BlockHeader.Length + _rowSize * (_PageNodeCount));\n\n CreateFileHeader(0);\n\n _LastPageNumber = (int)((_file.Length - _FileHeader.Length) / _PageLength);\n }\n if (_externalStrings)\n {\n _strings = new StringHF(path, Path.GetFileNameWithoutExtension(filename) + \".strings\");\n }\n if (_LastPageNumber == 0)\n _LastPageNumber = 1;\n // bitmap duplicates \n if (_allowDups)\n _bitmap = new BitmapIndex(Path.GetDirectoryName(filename), Path.GetFileNameWithoutExtension(filename));\n }\n\n #region [ C o m m o n ]\n public void SetBitmapDuplicate(int bitmaprec, int rec)\n {\n _bitmap.SetDuplicate(bitmaprec, rec);\n }\n\n public int GetBitmapDuplaicateFreeRecordNumber()\n {\n return _bitmap.GetFreeRecordNumber();\n }\n\n public IEnumerable GetDuplicatesRecordNumbers(int recno)\n {\n return GetDuplicateBitmap(recno).GetBitIndexes();\n }\n\n public MGRB GetDuplicateBitmap(int recno)\n {\n return _bitmap.GetBitmap(recno);\n }\n\n private byte[] CreateBlockHeader(byte type, ushort itemcount, int rightpagenumber)\n {\n byte[] block = new byte[_BlockHeader.Length];\n Array.Copy(_BlockHeader, block, block.Length);\n block[4] = type;\n byte[] b = Helper.GetBytes(itemcount, false);\n Buffer.BlockCopy(b, 0, block, 5, 2);\n b = Helper.GetBytes(rightpagenumber, false);\n Buffer.BlockCopy(b, 0, block, 11, 4);\n return block;\n }\n\n private void CreateFileHeader(int rowsindexed)\n {\n lock (_fileLock)\n {\n // max key size\n byte[] b = Helper.GetBytes(_maxKeySize, false);\n Buffer.BlockCopy(b, 0, _FileHeader, 3, 1);\n // page node count\n b = Helper.GetBytes(_PageNodeCount, false);\n Buffer.BlockCopy(b, 0, _FileHeader, 4, 2);\n b = Helper.GetBytes(rowsindexed, false);\n Buffer.BlockCopy(b, 0, _FileHeader, 11, 4);\n\n if (_externalStrings)\n _FileHeader[10] = 1;\n\n _file.Seek(0L, SeekOrigin.Begin);\n _file.Write(_FileHeader, 0, _FileHeader.Length);\n if (rowsindexed == 0)\n {\n byte[] pagezero = new byte[_PageLength];\n byte[] block = CreateBlockHeader(1, 0, -1);\n Buffer.BlockCopy(block, 0, pagezero, 0, block.Length);\n _file.Write(pagezero, 0, _PageLength);\n }\n _file.Flush();\n }\n }\n\n private bool ReadFileHeader()\n {\n _file.Seek(0L, SeekOrigin.Begin);\n byte[] b = new byte[_FileHeader.Length];\n _file.Read(b, 0, _FileHeader.Length);\n\n if (b[0] == _FileHeader[0] && b[1] == _FileHeader[1] && b[2] == _FileHeader[2]) // header\n {\n byte maxks = b[3];\n ushort nodes = (ushort)Helper.ToInt16(b, 4);\n int root = Helper.ToInt32(b, 6);\n _maxKeySize = maxks;\n _PageNodeCount = nodes;\n _FileHeader = b;\n if (b[10] == 0)\n _externalStrings = false;\n }\n\n return false;\n }\n\n public int GetNewPageNumber()\n {\n return Interlocked.Increment(ref _LastPageNumber); //_LastPageNumber++;\n }\n\n private void SeekPage(int pnum)\n {\n long offset = _FileHeader.Length;\n offset += (long)pnum * _PageLength;\n if (offset > _file.Length)\n CreateBlankPages(pnum);\n\n _file.Seek(offset, SeekOrigin.Begin);\n }\n\n private void CreateBlankPages(int pnum)\n {\n // create space\n byte[] b = new byte[_PageLength];\n _file.Seek(0L, SeekOrigin.Current);\n for (int i = pnum; i < _LastPageNumber; i++)\n _file.Write(b, 0, b.Length);\n\n _file.Flush();\n }\n\n public void FreeMemory()\n {\n if (_allowDups)\n _bitmap.FreeMemory();\n }\n\n public void Shutdown()\n {\n log.Debug(\"Shutdown IndexFile\");\n if (_externalStrings)\n _strings.Shutdown();\n\n if (_file != null)\n {\n _file.Flush();\n _file.Close();\n }\n _file = null;\n if (_allowDups)\n {\n _bitmap.Commit(Global.FreeBitmapMemoryOnSave);\n _bitmap.Shutdown();\n }\n }\n\n #endregion\n\n #region [ P a g e s ]\n\n public void GetPageList(List PageListDiskPages, SafeSortedList PageList, out int lastIndexedRow)\n {\n lastIndexedRow = Helper.ToInt32(_FileHeader, 11);\n // load page list\n PageListDiskPages.Add(0); // first page list\n int nextpage = LoadPageListData(0, PageList);\n while (nextpage != -1)\n {\n nextpage = LoadPageListData(nextpage, PageList);\n\n if (nextpage != -1)\n PageListDiskPages.Add(nextpage);\n }\n }\n\n private int LoadPageListData(int page, SafeSortedList PageList)\n {\n lock (_fileLock)\n {\n // load page list data\n int nextpage = -1;\n SeekPage(page);\n byte[] b = new byte[_PageLength];\n _file.Read(b, 0, _PageLength);\n\n if (b[0] == _BlockHeader[0] && b[1] == _BlockHeader[1] && b[2] == _BlockHeader[2] && b[3] == _BlockHeader[3])\n {\n short count = Helper.ToInt16(b, 5);\n if (count > _PageNodeCount)\n throw new Exception(\"Count > node size\");\n nextpage = Helper.ToInt32(b, 11);\n int index = _BlockHeader.Length;\n object[] keys = null;\n // TODO : needed??\n //if (File.Exists(_FileName + \".pagelist\"))\n //{\n // var bn = File.ReadAllBytes(_FileName + \".pagelist\");\n // int blknum = Helper.ToInt32(bn, 0);\n // byte[] bb = _strings.GetData(blknum, out _pagelistalllocblock);\n // keys = (object[])BJSON.ToObject(bb);\n //}\n for (int i = 0; i < count; i++)\n {\n int idx = index + _rowSize * i;\n byte ks = b[idx];\n T key;\n if (_externalStrings == false)\n key = _T.GetObject(b, idx + 1, ks);\n else\n {\n if (keys == null)\n key = _T.GetObject(b, idx + 1, ks); // do old way until better way\n else\n key = (T)keys[i];\n }\n int pagenum = Helper.ToInt32(b, idx + 1 + _maxKeySize);\n // add counts\n int unique = Helper.ToInt32(b, idx + 1 + _maxKeySize + 4);\n // FEATURE : add dup count\n PageList.Add(key, new PageInfo(pagenum, unique, 0));\n }\n }\n else\n throw new Exception(\"Page List header is invalid\");\n\n return nextpage;\n }\n }\n\n internal void SavePage(Page page)\n {\n lock (_fileLock)\n {\n int pnum = page.DiskPageNumber;\n if (pnum > _LastPageNumber)\n throw new Exception(\"should not be here: page out of bounds\");\n\n SeekPage(pnum);\n byte[] pagebytes = new byte[_PageLength];\n byte[] blockheader = CreateBlockHeader(0, (ushort)page.tree.Count(), page.RightPageNumber);\n Buffer.BlockCopy(blockheader, 0, pagebytes, 0, blockheader.Length);\n\n int index = blockheader.Length;\n int i = 0;\n byte[] b = null;\n T[] keys = page.tree.Keys();\n Array.Sort(keys); // sort keys on save for read performance\n int blocknum = 0;\n if (_externalStrings)\n {\n // free old blocks\n if (page.allocblocks != null)\n _strings.FreeBlocks(page.allocblocks);\n List blocks = new List();\n blocknum = _strings.SaveData(page.DiskPageNumber.ToString(), BJSON.ToBJSON(keys,\n new BJSONParameters { UseUnicodeStrings = false, UseTypedArrays = false }), out blocks);\n page.allocblocks = blocks;\n }\n // node children\n foreach (var kp in keys)\n {\n var val = page.tree[kp];\n int idx = index + _rowSize * i;\n // key bytes\n byte[] kk;\n byte size;\n if (_externalStrings == false)\n {\n kk = _T.GetBytes(kp);\n size = (byte)kk.Length;\n if (size > _maxKeySize)\n size = _maxKeySize;\n }\n else\n {\n kk = new byte[4];\n Buffer.BlockCopy(Helper.GetBytes(blocknum, false), 0, kk, 0, 4);\n size = 4;\n }\n // key size = 1 byte\n pagebytes[idx] = size;\n Buffer.BlockCopy(kk, 0, pagebytes, idx + 1, pagebytes[idx]);\n // offset = 4 bytes\n b = Helper.GetBytes(val.RecordNumber, false);\n Buffer.BlockCopy(b, 0, pagebytes, idx + 1 + _maxKeySize, b.Length);\n // duplicatepage = 4 bytes\n b = Helper.GetBytes(val.DuplicateBitmapNumber, false);\n Buffer.BlockCopy(b, 0, pagebytes, idx + 1 + _maxKeySize + 4, b.Length);\n i++;\n }\n _file.Write(pagebytes, 0, pagebytes.Length);\n }\n }\n\n public Page LoadPageFromPageNumber(int number)\n {\n lock (_fileLock)\n {\n SeekPage(number);\n byte[] b = new byte[_PageLength];\n _file.Read(b, 0, _PageLength);\n\n if (b[0] == _BlockHeader[0] && b[1] == _BlockHeader[1] && b[2] == _BlockHeader[2] && b[3] == _BlockHeader[3])\n {\n // create node here\n Page page = new Page();\n\n short count = Helper.ToInt16(b, 5);\n if (count > _PageNodeCount)\n throw new Exception(\"Count > node size\");\n page.DiskPageNumber = number;\n page.RightPageNumber = Helper.ToInt32(b, 11);\n int index = _BlockHeader.Length;\n object[] keys = null;\n\n for (int i = 0; i < count; i++)\n {\n int idx = index + _rowSize * i;\n byte ks = b[idx];\n T key = default(T);\n if (_externalStrings == false)\n key = _T.GetObject(b, idx + 1, ks);\n else\n {\n if (keys == null)\n {\n int blknum = Helper.ToInt32(b, idx + 1, false);\n List ablocks = new List();\n byte[] bb = _strings.GetData(blknum, out ablocks);\n page.allocblocks = ablocks;\n keys = (object[])BJSON.ToObject(bb);\n }\n key = (T)keys[i];\n }\n int offset = Helper.ToInt32(b, idx + 1 + _maxKeySize);\n int duppage = Helper.ToInt32(b, idx + 1 + _maxKeySize + 4);\n page.tree.Add(key, new KeyInfo(offset, duppage));\n }\n return page;\n }\n else\n throw new Exception(\"Page read error header invalid, number = \" + number);\n }\n }\n #endregion\n\n internal void SavePageList(SafeSortedList _pages, List diskpages)\n {\n lock (_fileLock)\n {\n T[] keys = _pages.Keys();\n int blocknum = 0;\n // TODO : needed??\n //if (_externalStrings)\n //{\n // if (_pagelistalllocblock != null)\n // _strings.FreeBlocks(_pagelistalllocblock);\n // blocknum = _strings.SaveData(\"pagelist\", BJSON.ToBJSON(keys,\n // new BJSONParameters { UseUnicodeStrings = false, UseTypedArrays = false }));\n // File.WriteAllBytes(_FileName + \".pagelist\", Helper.GetBytes(blocknum, false));\n //}\n // save page list\n int c = (_pages.Count() / Global.PageItemCount) + 1;\n // allocate pages needed \n while (c > diskpages.Count)\n diskpages.Add(GetNewPageNumber());\n\n byte[] page = new byte[_PageLength];\n\n for (int i = 0; i < (diskpages.Count - 1); i++)\n {\n byte[] block = CreateBlockHeader(1, Global.PageItemCount, diskpages[i + 1]);\n Buffer.BlockCopy(block, 0, page, 0, block.Length);\n\n for (int j = 0; j < Global.PageItemCount; j++)\n CreatePageListData(_pages, i * Global.PageItemCount, block.Length, j, page, blocknum);\n\n SeekPage(diskpages[i]);\n _file.Write(page, 0, page.Length);\n }\n\n c = _pages.Count() % Global.PageItemCount;\n byte[] lastblock = CreateBlockHeader(1, (ushort)c, -1);\n Buffer.BlockCopy(lastblock, 0, page, 0, lastblock.Length);\n int lastoffset = (_pages.Count() / Global.PageItemCount) * Global.PageItemCount;\n\n for (int j = 0; j < c; j++)\n CreatePageListData(_pages, lastoffset, lastblock.Length, j, page, blocknum);\n\n SeekPage(diskpages[diskpages.Count - 1]);\n _file.Write(page, 0, page.Length);\n }\n }\n\n private void CreatePageListData(SafeSortedList _pages, int offset, int rowindex, int counter, byte[] page, int blocknum)\n {\n int idx = rowindex + _rowSize * counter;\n // key bytes\n byte[] kk;\n byte size;\n if (_externalStrings == false)\n {\n kk = _T.GetBytes(_pages.GetKey(counter + offset));\n size = (byte)kk.Length;\n if (size > _maxKeySize)\n size = _maxKeySize;\n }\n else\n {\n kk = new byte[4];\n Buffer.BlockCopy(Helper.GetBytes(counter + offset, false), 0, kk, 0, 4);\n size = 4;\n }\n // key size = 1 byte\n page[idx] = size;\n Buffer.BlockCopy(kk, 0, page, idx + 1, page[idx]);\n // offset = 4 bytes\n byte[] b = Helper.GetBytes(_pages.GetValue(offset + counter).PageNumber, false);\n Buffer.BlockCopy(b, 0, page, idx + 1 + _maxKeySize, b.Length);\n // add counts \n b = Helper.GetBytes(_pages.GetValue(offset + counter).UniqueCount, false);\n Buffer.BlockCopy(b, 0, page, idx + 1 + _maxKeySize + 4, b.Length);\n // FEATURE : add dup counts\n }\n\n internal void SaveLastRecordNumber(int recnum)\n {\n // save the last record number indexed to the header\n CreateFileHeader(recnum);\n }\n\n internal void BitmapFlush()\n {\n if (_allowDups)\n _bitmap.Commit(Global.FreeBitmapMemoryOnSave);\n }\n }\n}" }, { "path": "RaptorDB/Indexes/Indexes.cs", "content": "using System;\nusing System.Collections.Generic;\nusing System.IO;\n\nnamespace RaptorDB\n{\n #region [ TypeIndexes ]\n internal class TypeIndexes : MGIndex, IIndex where T : IComparable\n {\n public TypeIndexes(string path, string filename, byte keysize)\n : base(path, filename + \".mgidx\", keysize, true)\n {\n\n }\n\n public void Set(object key, int recnum)\n {\n if (key == null) return; // FEATURE : index null values ??\n\n base.Set((T)key, recnum);\n }\n\n public MGRB Query(RDBExpression ex, object from, int maxsize)\n {\n T f = default(T);\n if (typeof(T).Equals(from.GetType()) == false)\n f = Converter(from);\n else\n f = (T)from;\n\n return base.Query(ex, f, maxsize);\n }\n\n private T Converter(object from)\n {\n if (typeof(T) == typeof(Guid))\n {\n object o = new Guid(from.ToString());\n return (T)o;\n }\n else\n return (T)Convert.ChangeType(from, typeof(T));\n }\n\n void IIndex.FreeMemory()\n {\n base.SaveIndex();\n base.FreeMemory();\n }\n\n void IIndex.Shutdown()\n {\n base.Shutdown();\n }\n\n object[] IIndex.GetKeys()\n {\n return base.GetKeys();\n }\n\n public MGRB Query(object fromkey, object tokey, int maxsize)\n {\n T f = default(T);\n if (typeof(T).Equals(fromkey.GetType()) == false)\n f = (T)Convert.ChangeType(fromkey, typeof(T));\n else\n f = (T)fromkey;\n\n T t = default(T);\n if (typeof(T).Equals(tokey.GetType()) == false)\n t = (T)Convert.ChangeType(tokey, typeof(T));\n else\n t = (T)tokey;\n\n return base.Query(f, t, maxsize);\n }\n }\n #endregion\n\n #region [ BoolIndex ]\n internal class BoolIndex : IIndex\n {\n public BoolIndex(string path, string filename, string extension)\n {\n // create file\n _filename = filename + extension;\n _path = path;\n if (_path.EndsWith(Path.DirectorySeparatorChar.ToString()) == false)\n _path += Path.DirectorySeparatorChar.ToString();\n\n if (File.Exists(_path + _filename))\n ReadFile();\n }\n\n private MGRB _bits = new MGRB();\n private string _filename;\n private string _path;\n private object _lock = new object();\n\n public MGRB GetBits()\n {\n return _bits.Copy();\n }\n\n public void Set(object key, int recnum)\n {\n lock (_lock)\n if (key != null)\n _bits.Set(recnum, (bool)key);\n }\n\n public MGRB Query(RDBExpression ex, object from, int maxsize)\n {\n lock (_lock)\n {\n bool b = (bool)from;\n if (b)\n return _bits;\n else\n return _bits.Not(maxsize);\n }\n }\n\n public void FreeMemory()\n {\n lock (_lock)\n {\n _bits.Optimize();\n SaveIndex();\n }\n }\n\n public void Shutdown()\n {\n // shutdown\n WriteFile();\n }\n\n public void SaveIndex()\n {\n WriteFile();\n }\n\n public void InPlaceOR(MGRB left)\n {\n lock (_lock)\n _bits = _bits.Or(left);\n }\n\n private void WriteFile()\n {\n lock (_lock)\n {\n _bits.Optimize();\n var o = _bits.Serialize();\n var b = fastBinaryJSON.BJSON.ToBJSON(o, new fastBinaryJSON.BJSONParameters { UseExtensions = false });\n File.WriteAllBytes(_path + _filename, b);\n }\n }\n\n private void ReadFile()\n {\n byte[] b = File.ReadAllBytes(_path + _filename);\n var o = fastBinaryJSON.BJSON.ToObject(b);\n _bits = new MGRB();\n _bits.Deserialize(o);\n }\n\n public MGRB Query(object fromkey, object tokey, int maxsize)\n {\n return Query(RDBExpression.Greater, fromkey, maxsize); // range doesn't make sense here just do from \n }\n\n public object[] GetKeys()\n {\n return new object[] { true, false };\n }\n }\n #endregion\n\n #region [ FullTextIndex ]\n internal class FullTextIndex : Hoot, IIndex\n {\n public FullTextIndex(string IndexPath, string FileName, bool docmode, bool sortable, ITokenizer tokenizer)\n : base(IndexPath, FileName, docmode, tokenizer)\n {\n if (sortable)\n {\n _idx = new TypeIndexes(IndexPath, FileName, Global.DefaultStringKeySize);\n _sortable = true;\n }\n }\n private bool _sortable = false;\n private IIndex _idx;\n\n public void Set(object key, int recnum)\n {\n base.Index(recnum, (string)key);\n if (_sortable)\n _idx.Set(key, recnum);\n }\n\n public MGRB Query(RDBExpression ex, object from, int maxsize)\n {\n return base.Query(\"\" + from, maxsize);\n }\n\n public void SaveIndex()\n {\n base.Save();\n if (_sortable)\n _idx.SaveIndex();\n }\n\n public MGRB Query(object fromkey, object tokey, int maxsize)\n {\n return base.Query(\"\" + fromkey, maxsize); // range doesn't make sense here just do from \n }\n\n public object[] GetKeys()\n {\n if (_sortable)\n return _idx.GetKeys(); // support get keys \n else\n return new object[] { };\n }\n void IIndex.FreeMemory()\n {\n base.FreeMemory();\n\n this.SaveIndex();\n }\n\n void IIndex.Shutdown()\n {\n this.SaveIndex();\n base.Shutdown();\n if (_sortable)\n _idx.Shutdown();\n }\n\n }\n #endregion\n\n #region [ EnumIndex ]\n internal class EnumIndex : MGIndex, IIndex //where T : IComparable\n {\n public EnumIndex(string path, string filename)\n : base(path, filename + \".mgidx\", 30, /*Global.PageItemCount,*/ true)\n {\n\n }\n\n public void Set(object key, int recnum)\n {\n if (key == null) return; // FEATURE : index null values ??\n\n base.Set(key.ToString(), recnum);\n }\n\n public MGRB Query(RDBExpression ex, object from, int maxsize)\n {\n T f = default(T);\n if (typeof(T).Equals(from.GetType()) == false)\n f = Converter(from);\n else\n f = (T)from;\n\n return base.Query(ex, f.ToString(), maxsize);\n }\n\n private T Converter(object from)\n {\n if (typeof(T) == typeof(Guid))\n {\n object o = new Guid(from.ToString());\n return (T)o;\n }\n else\n return (T)Convert.ChangeType(from, typeof(T));\n }\n\n void IIndex.FreeMemory()\n {\n base.SaveIndex();\n base.FreeMemory();\n }\n\n void IIndex.Shutdown()\n {\n base.SaveIndex();\n base.Shutdown();\n }\n\n public MGRB Query(object fromkey, object tokey, int maxsize)\n {\n T f = default(T);\n if (typeof(T).Equals(fromkey.GetType()) == false)\n f = (T)Convert.ChangeType(fromkey, typeof(T));\n else\n f = (T)fromkey;\n\n T t = default(T);\n if (typeof(T).Equals(tokey.GetType()) == false)\n t = (T)Convert.ChangeType(tokey, typeof(T));\n else\n t = (T)tokey;\n\n return base.Query(f.ToString(), t.ToString(), maxsize);\n }\n\n object[] IIndex.GetKeys()\n {\n return base.GetKeys();\n }\n }\n #endregion\n\n #region [ NoIndex ]\n internal class NoIndex : IIndex\n {\n public void Set(object key, int recnum)\n {\n // ignore set\n }\n\n public MGRB Query(RDBExpression ex, object from, int maxsize)\n {\n // always return everything\n return MGRB.Fill(maxsize);\n }\n\n public void FreeMemory()\n {\n\n }\n\n public void Shutdown()\n {\n\n }\n\n public void SaveIndex()\n {\n\n }\n\n public object[] GetKeys()\n {\n return new object[] { };\n }\n\n public MGRB Query(object fromkey, object tokey, int maxsize)\n {\n return MGRB.Fill(maxsize); // TODO : all or none??\n }\n }\n #endregion\n}\n" }, { "path": "RaptorDB/Indexes/MGIndex.cs", "content": "using System;\nusing System.Collections.Generic;\nusing System.IO;\nusing RaptorDB.Common;\n\nnamespace RaptorDB\n{\n #region [ internal classes ]\n\n internal struct PageInfo // FEATURE : change back to class for count access for query caching\n {\n public PageInfo(int pagenum, int uniquecount, int duplicatecount)\n {\n PageNumber = pagenum;\n UniqueCount = uniquecount;\n }\n public int PageNumber;\n public int UniqueCount;\n }\n\n internal struct KeyInfo\n {\n public KeyInfo(int recnum)\n {\n RecordNumber = recnum;\n DuplicateBitmapNumber = -1;\n }\n public KeyInfo(int recnum, int bitmaprec)\n {\n RecordNumber = recnum;\n DuplicateBitmapNumber = bitmaprec;\n }\n public int RecordNumber;\n public int DuplicateBitmapNumber;\n }\n\n internal class Page\n {\n public Page() // kludge so the compiler doesn't complain\n {\n DiskPageNumber = -1;\n RightPageNumber = -1;\n tree = new SafeDictionary(Global.PageItemCount);\n isDirty = false;\n FirstKey = default(T);\n }\n public int DiskPageNumber;\n public int RightPageNumber;\n public T FirstKey;\n public bool isDirty;\n public SafeDictionary tree;\n public List allocblocks = null; // for string keys in HF key store\n }\n\n #endregion\n\n internal class MGIndex where T : IComparable\n {\n ILog _log = LogManager.GetLogger(typeof(MGIndex));\n private SafeSortedList _pageList = new SafeSortedList();\n\n //private SafeDictionary> _cache = new SafeDictionary>();\n private IKV> _cache = null;//new SafeSortedList>();\n\n private List _pageListDiskPages = new List();\n private IndexFile _index;\n private bool _AllowDuplicates = true;\n private int _LastIndexedRecordNumber = 0;\n\n public MGIndex(string path, string filename, byte keysize, bool allowdups)\n {\n if (Global.UseLessMemoryStructures)\n _cache = new SafeSortedList>();\n else\n _cache = new SafeDictionary>();\n\n _AllowDuplicates = allowdups;\n if (path.EndsWith(Path.DirectorySeparatorChar.ToString()) == false)\n path += Path.DirectorySeparatorChar;\n\n _index = new IndexFile(path + filename, keysize);\n // load page list\n _index.GetPageList(_pageListDiskPages, _pageList, out _LastIndexedRecordNumber);\n if (_pageList.Count() == 0)\n {\n Page page = new Page();\n page.FirstKey = (T)RDBDataType.GetEmpty();\n page.DiskPageNumber = _index.GetNewPageNumber();\n page.isDirty = true;\n _pageList.Add(page.FirstKey, new PageInfo(page.DiskPageNumber, 0, 0));\n _cache.Add(page.DiskPageNumber, page);\n }\n }\n\n public int GetLastIndexedRecordNumber()\n {\n return _LastIndexedRecordNumber;\n }\n\n public MGRB Query(T from, T to, int maxsize)\n {\n MGRB bits = new MGRB();\n T temp = default(T);\n if (from.CompareTo(to) > 0) // check values order\n {\n temp = from;\n from = to;\n to = temp;\n }\n // find first page and do > than\n bool found = false;\n int startpos = FindPageOrLowerPosition(from, ref found);\n // find last page and do < than\n int endpos = FindPageOrLowerPosition(to, ref found);\n bool samepage = startpos == endpos;\n\n // from key page\n Page page = LoadPage(_pageList.GetValue(startpos).PageNumber);\n T[] keys = page.tree.Keys();\n Array.Sort(keys);\n\n // find better start position rather than 0\n int pos = Array.BinarySearch(keys, from); // FEATURE : rewrite??\n if (pos < 0) pos = ~pos;\n\n for (int i = pos; i < keys.Length; i++)\n {\n T k = keys[i];\n int bn = page.tree[k].DuplicateBitmapNumber;\n\n if (samepage)\n {\n if (k.CompareTo(from) >= 0 && k.CompareTo(to) <= 0) // if from,to same page\n bits = bits.Or(_index.GetDuplicateBitmap(bn));\n }\n else\n {\n if (k.CompareTo(from) >= 0)\n bits = bits.Or(_index.GetDuplicateBitmap(bn));\n }\n }\n if (!samepage)\n {\n // to key page\n page = LoadPage(_pageList.GetValue(endpos).PageNumber);\n keys = page.tree.Keys();\n Array.Sort(keys);\n // find better end position rather than last key\n pos = Array.BinarySearch(keys, to);\n if (pos < 0) pos = ~pos;\n\n for (int i = 0; i <= pos; i++)\n {\n T k = keys[i];\n int bn = page.tree[k].DuplicateBitmapNumber;\n\n if (k.CompareTo(to) <= 0)\n bits = bits.Or(_index.GetDuplicateBitmap(bn));\n }\n // do all pages in between\n for (int i = startpos + 1; i < endpos; i++)\n {\n doPageOperation(ref bits, i);\n }\n }\n return bits;\n }\n\n public MGRB Query(RDBExpression exp, T from, int maxsize)\n {\n T key = from;\n if (exp == RDBExpression.Equal || exp == RDBExpression.NotEqual)\n return doEqualOp(exp, key, maxsize);\n\n // FEATURE : optimize complement search if page count less for the complement pages\n\n if (exp == RDBExpression.Less || exp == RDBExpression.LessEqual)\n {\n return doLessOp(exp, key);\n }\n else if (exp == RDBExpression.Greater || exp == RDBExpression.GreaterEqual)\n {\n return doMoreOp(exp, key);\n }\n\n return new MGRB(); // blank results \n }\n\n private object _setlock = new object();\n public void Set(T key, int val)\n {\n lock (_setlock)\n {\n PageInfo pi;\n Page page = LoadPage(key, out pi);\n\n KeyInfo ki;\n if (page.tree.TryGetValue(key, out ki))\n {\n // item exists\n if (_AllowDuplicates)\n {\n SaveDuplicate(key, ref ki);\n // set current record in the bitmap also\n _index.SetBitmapDuplicate(ki.DuplicateBitmapNumber, val);\n }\n ki.RecordNumber = val;\n page.tree[key] = ki; // structs need resetting\n }\n else\n {\n // new item \n ki = new KeyInfo(val);\n if (_AllowDuplicates)\n SaveDuplicate(key, ref ki);\n pi.UniqueCount++;\n page.tree.Add(key, ki);\n }\n\n if (page.tree.Count() > Global.PageItemCount)\n SplitPage(page);\n\n _LastIndexedRecordNumber = val;\n page.isDirty = true;\n }\n }\n\n public bool Get(T key, out int val)\n {\n val = -1;\n PageInfo pi;\n Page page = LoadPage(key, out pi);\n KeyInfo ki;\n bool ret = page.tree.TryGetValue(key, out ki);\n if (ret)\n val = ki.RecordNumber;\n return ret;\n }\n\n public void SaveIndex()\n {\n //_log.Debug(\"Total split time (s) = \" + _totalsplits);\n //_log.Debug(\"Total pages = \" + _pageList.Count);\n int[] keys = _cache.Keys();\n Array.Sort(keys);\n // save index to disk\n foreach (var i in keys)\n {\n var p = _cache.GetValue(i);\n if (p.isDirty)\n {\n _index.SavePage(p);\n p.isDirty = false;\n }\n }\n _index.SavePageList(_pageList, _pageListDiskPages);\n _index.BitmapFlush();\n }\n\n public void Shutdown()\n {\n SaveIndex();\n // save page list\n //_index.SavePageList(_pageList, _pageListDiskPages);\n // shutdown\n _index.Shutdown();\n }\n\n public void FreeMemory()\n {\n _index.FreeMemory();\n try\n {\n List free = new List();\n foreach (var k in _cache.Keys())\n {\n var val = _cache.GetValue(k);\n if (val.isDirty == false)\n free.Add(k);\n }\n _log.Info(\"releasing page count = \" + free.Count + \" out of \" + _cache.Count());\n foreach (var i in free)\n _cache.Remove(i);\n }\n catch { }\n }\n\n public IEnumerable GetDuplicates(T key)\n {\n PageInfo pi;\n Page page = LoadPage(key, out pi);\n KeyInfo ki;\n bool ret = page.tree.TryGetValue(key, out ki);\n if (ret)\n // get duplicates\n if (ki.DuplicateBitmapNumber != -1)\n return _index.GetDuplicatesRecordNumbers(ki.DuplicateBitmapNumber);\n\n return new List();\n }\n\n public void SaveLastRecordNumber(int recnum)\n {\n _index.SaveLastRecordNumber(recnum);\n }\n\n public bool RemoveKey(T key)\n {\n PageInfo pi;\n Page page = LoadPage(key, out pi);\n bool b = page.tree.Remove(key);\n // TODO : reset the first key for page ??\n if (b)\n {\n pi.UniqueCount--;\n // FEATURE : decrease dup count\n }\n page.isDirty = true;\n return b;\n }\n\n #region [ P R I V A T E ]\n private MGRB doMoreOp(RDBExpression exp, T key)\n {\n bool found = false;\n int pos = FindPageOrLowerPosition(key, ref found);\n MGRB result = new MGRB();\n if (pos < _pageList.Count())\n {\n // all the pages after\n for (int i = pos + 1; i < _pageList.Count(); i++)\n doPageOperation(ref result, i);\n }\n // key page\n Page page = LoadPage(_pageList.GetValue(pos).PageNumber);\n T[] keys = page.tree.Keys();\n Array.Sort(keys);\n\n // find better start position rather than 0\n pos = Array.BinarySearch(keys, key);\n if (pos < 0) pos = ~pos;\n\n for (int i = pos; i < keys.Length; i++)\n {\n T k = keys[i];\n int bn = page.tree[k].DuplicateBitmapNumber;\n\n if (k.CompareTo(key) > 0)\n result = result.Or(_index.GetDuplicateBitmap(bn));\n\n if (exp == RDBExpression.GreaterEqual && k.CompareTo(key) == 0)\n result = result.Or(_index.GetDuplicateBitmap(bn));\n }\n return result;\n }\n\n private MGRB doLessOp(RDBExpression exp, T key)\n {\n bool found = false;\n int pos = FindPageOrLowerPosition(key, ref found);\n MGRB result = new MGRB();\n if (pos > 0)\n {\n // all the pages before\n for (int i = 0; i < pos - 1; i++)\n doPageOperation(ref result, i);\n }\n // key page\n Page page = LoadPage(_pageList.GetValue(pos).PageNumber);\n T[] keys = page.tree.Keys();\n Array.Sort(keys);\n // find better end position rather than last key\n pos = Array.BinarySearch(keys, key);\n if (pos < 0) pos = ~pos;\n for (int i = 0; i < pos; i++)\n {\n T k = keys[i];\n if (k.CompareTo(key) > 0)\n break;\n int bn = page.tree[k].DuplicateBitmapNumber;\n\n if (k.CompareTo(key) < 0)\n result = result.Or(_index.GetDuplicateBitmap(bn));\n\n if (exp == RDBExpression.LessEqual && k.CompareTo(key) == 0)\n result = result.Or(_index.GetDuplicateBitmap(bn));\n }\n return result;\n }\n\n private MGRB doEqualOp(RDBExpression exp, T key, int maxsize)\n {\n PageInfo pi;\n Page page = LoadPage(key, out pi);\n KeyInfo k;\n if (page.tree.TryGetValue(key, out k))\n {\n int bn = k.DuplicateBitmapNumber;\n\n if (exp == RDBExpression.Equal)\n return _index.GetDuplicateBitmap(bn);\n else\n return _index.GetDuplicateBitmap(bn).Not(maxsize);\n }\n else\n {\n if (exp == RDBExpression.NotEqual)\n return new MGRB().Not(maxsize);\n else\n return new MGRB();\n }\n }\n\n private void doPageOperation(ref MGRB res, int pageidx)\n {\n Page page = LoadPage(_pageList.GetValue(pageidx).PageNumber);\n T[] keys = page.tree.Keys(); // avoid sync issues\n foreach (var k in keys)\n {\n int bn = page.tree[k].DuplicateBitmapNumber;\n\n res = res.Or(_index.GetDuplicateBitmap(bn));\n }\n }\n\n private double _totalsplits = 0;\n private void SplitPage(Page page)\n {\n // split the page\n DateTime dt = FastDateTime.Now;\n\n Page newpage = new Page();\n newpage.DiskPageNumber = _index.GetNewPageNumber();\n newpage.RightPageNumber = page.RightPageNumber;\n newpage.isDirty = true;\n page.RightPageNumber = newpage.DiskPageNumber;\n _pageList.Remove(page.FirstKey);\n // get and sort keys\n T[] keys = page.tree.Keys();\n Array.Sort(keys);\n // copy data to new \n for (int i = keys.Length / 2; i < keys.Length; i++)\n {\n newpage.tree.Add(keys[i], page.tree[keys[i]]);\n // remove from old page\n page.tree.Remove(keys[i]);\n }\n // set the first key\n newpage.FirstKey = keys[keys.Length / 2]; // new key\n // remove keys from page list\n _pageList.Remove(newpage.FirstKey);\n page.FirstKey = keys[0]; // new key\n // re add to page list\n _pageList.Add(page.FirstKey, new PageInfo(page.DiskPageNumber, page.tree.Count(), 0));\n _pageList.Add(newpage.FirstKey, new PageInfo(newpage.DiskPageNumber, newpage.tree.Count(), 0));\n _cache.Add(newpage.DiskPageNumber, newpage);\n\n _totalsplits += FastDateTime.Now.Subtract(dt).TotalSeconds;\n }\n\n private Page LoadPage(T key, out PageInfo pageinfo)\n {\n int pagenum = -1;\n // find page in list of pages\n\n bool found = false;\n int pos = 0;\n if (key != null)\n pos = FindPageOrLowerPosition(key, ref found);\n pageinfo = _pageList.GetValue(pos);\n pagenum = pageinfo.PageNumber;\n\n Page page;\n if (_cache.TryGetValue(pagenum, out page) == false)\n {\n //load page from disk\n page = _index.LoadPageFromPageNumber(pagenum);\n _cache.Add(pagenum, page);\n }\n return page;\n }\n\n private Page LoadPage(int pagenum)\n {\n Page page;\n if (_cache.TryGetValue(pagenum, out page) == false)\n {\n //load page from disk\n page = _index.LoadPageFromPageNumber(pagenum);\n _cache.Add(pagenum, page);\n }\n return page;\n }\n\n private void SaveDuplicate(T key, ref KeyInfo ki)\n {\n if (ki.DuplicateBitmapNumber == -1)\n ki.DuplicateBitmapNumber = _index.GetBitmapDuplaicateFreeRecordNumber();\n\n _index.SetBitmapDuplicate(ki.DuplicateBitmapNumber, ki.RecordNumber);\n }\n\n private int FindPageOrLowerPosition(T key, ref bool found)\n {\n if (_pageList.Count() == 0)\n return 0;\n // binary search\n int lastlower = 0;\n int first = 0;\n int last = _pageList.Count() - 1;\n int mid = 0;\n while (first <= last)\n {\n mid = (first + last) >> 1;\n T k = _pageList.GetKey(mid);\n int compare = k.CompareTo(key);\n if (compare < 0)\n {\n lastlower = mid;\n first = mid + 1;\n }\n if (compare == 0)\n {\n found = true;\n return mid;\n }\n if (compare > 0)\n {\n last = mid - 1;\n }\n }\n\n return lastlower;\n }\n #endregion\n\n internal object[] GetKeys()\n {\n List keys = new List();\n for (int i = 0; i < _pageList.Count(); i++)\n {\n Page page = LoadPage(_pageList.GetValue(i).PageNumber);\n foreach (var k in page.tree.Keys())\n keys.Add(k);\n }\n return keys.ToArray();\n }\n\n internal int Count()\n {\n int count = 0;\n for (int i = 0; i < _pageList.Count(); i++)\n {\n Page page = LoadPage(_pageList.GetValue(i).PageNumber);\n //foreach (var k in page.tree.Keys())\n // count++;\n count += page.tree.Count();\n }\n return count;\n }\n }\n}\n" }, { "path": "RaptorDB/Indexes/tokenizer.cs", "content": "using System.Collections.Generic;\n\nnamespace RaptorDB\n{\n class tokenizer : ITokenizer\n {\n public Dictionary GenerateWordFreq(string text)\n {\n Dictionary dic = new Dictionary(500);\n\n char[] chars = text.ToCharArray();\n int index = 0;\n int look = 0;\n int count = chars.Length;\n int lastlang = langtype(chars[0]);\n while (index < count)\n {\n int lang = -1;\n while (look < count)\n {\n char c = chars[look];\n lang = langtype(c);\n if (lang == lastlang)\n look++;\n else\n break;\n }\n if (lastlang > -1)\n ParseString(dic, chars, look, index);\n index = look;\n lastlang = lang;\n }\n return dic;\n }\n\n private static int langtype(char c)\n {\n if (char.IsDigit(c))\n return 0;\n\n else if (char.IsWhiteSpace(c))\n return -1;\n\n else if (char.IsPunctuation(c))\n return -1;\n\n else if (char.IsLetter(c)) // FEATURE : language checking here\n return 1;\n\n else\n return -1;\n }\n\n private static void ParseString(Dictionary dic, char[] chars, int end, int start)\n {\n // check if upper lower case mix -> extract words\n int uppers = 0;\n bool found = false;\n for (int i = start; i < end; i++)\n {\n if (char.IsUpper(chars[i]))\n uppers++;\n }\n // not all uppercase\n if (uppers != end - start - 1)\n {\n int lastUpper = start;\n\n string word = \"\";\n for (int i = start + 1; i < end; i++)\n {\n char c = chars[i];\n if (char.IsUpper(c))\n {\n found = true;\n word = new string(chars, lastUpper, i - lastUpper).ToLowerInvariant().Trim();\n AddDictionary(dic, word);\n lastUpper = i;\n }\n }\n if (lastUpper > start)\n {\n string last = new string(chars, lastUpper, end - lastUpper).ToLowerInvariant().Trim();\n if (word != last)\n AddDictionary(dic, last);\n }\n }\n if (found == false)\n {\n string s = new string(chars, start, end - start).ToLowerInvariant().Trim();\n AddDictionary(dic, s);\n }\n }\n\n private static void AddDictionary(Dictionary dic, string word)\n {\n if (word == null)\n return;\n\n int l = word.Length;\n // too long\n if (l > Global.DefaultStringKeySize)\n return;\n\n // too short\n if (l < 2)\n return;\n\n addword(dic, word);\n }\n\n private static void addword(Dictionary dic, string word)\n {\n int cc = 0;\n if (dic.TryGetValue(word, out cc))\n dic[word] = ++cc;\n else\n dic.Add(word, 1);\n }\n }\n}\n" }, { "path": "RaptorDB/Properties/Resources.Designer.cs", "content": "//------------------------------------------------------------------------------\n// \n// This code was generated by a tool.\n// Runtime Version:4.0.30319.18213\n//\n// Changes to this file may cause incorrect behavior and will be lost if\n// the code is regenerated.\n// \n//------------------------------------------------------------------------------\n\nnamespace RaptorDB.Properties {\n using System;\n \n \n /// \n /// A strongly-typed resource class, for looking up localized strings, etc.\n /// \n // This class was auto-generated by the StronglyTypedResourceBuilder\n // class via a tool like ResGen or Visual Studio.\n // To add or remove a member, edit your .ResX file then rerun ResGen\n // with the /str option, or rebuild your VS project.\n [global::System.CodeDom.Compiler.GeneratedCodeAttribute(\"System.Resources.Tools.StronglyTypedResourceBuilder\", \"4.0.0.0\")]\n [global::System.Diagnostics.DebuggerNonUserCodeAttribute()]\n [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()]\n internal class Resources {\n \n private static global::System.Resources.ResourceManager resourceMan;\n \n private static global::System.Globalization.CultureInfo resourceCulture;\n \n [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute(\"Microsoft.Performance\", \"CA1811:AvoidUncalledPrivateCode\")]\n internal Resources() {\n }\n \n /// \n /// Returns the cached ResourceManager instance used by this class.\n /// \n [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)]\n internal static global::System.Resources.ResourceManager ResourceManager {\n get {\n if (object.ReferenceEquals(resourceMan, null)) {\n global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager(\"RaptorDB.Properties.Resources\", typeof(Resources).Assembly);\n resourceMan = temp;\n }\n return resourceMan;\n }\n }\n \n /// \n /// Overrides the current thread's CurrentUICulture property for all\n /// resource lookups using this strongly typed resource class.\n /// \n [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)]\n internal static global::System.Globalization.CultureInfo Culture {\n get {\n return resourceCulture;\n }\n set {\n resourceCulture = value;\n }\n }\n \n /// \n /// Looks up a localized string similar to The following error occurred and the json document is below, you can skip this \n ///document if you wish by incrementing the %c% file :\n ///\n ///%ex%\n ///------------------------------------------------------------------------------\n ///The json document is :\n ///\n ///%js%\n ///\n ///.\n /// \n internal static string msg {\n get {\n return ResourceManager.GetString(\"msg\", resourceCulture);\n }\n }\n }\n}\n" }, { "path": "RaptorDB/Properties/Resources.resx", "content": "\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n text/microsoft-resx\n \n \n 2.0\n \n \n System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089\n \n \n System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089\n \n \n \n ..\\replication\\msg.txt;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;utf-8\n \n" }, { "path": "RaptorDB/REST/aWebServer.cs", "content": "using System;\nusing System.Net;\nusing System.Text;\nusing System.IO;\nusing System.Threading.Tasks;\nusing System.Reflection;\nusing System.Collections.Generic;\nusing System.IO.Compression;\n\nnamespace RaptorDB\n{\n public abstract class aWebServer\n {\n public aWebServer(int HttpPort, bool localonly, AuthenticationSchemes authenticationType, string apiPrefix)\n {\n _apiPrefix = apiPrefix;\n _authenticationType = authenticationType;\n _localonly = localonly;\n _port = HttpPort;\n Console.WriteLine(\"web port = \" + _port);\n Task.Factory.StartNew(() => Start(), TaskCreationOptions.LongRunning);\n }\n\n public delegate void Handler(HttpListenerContext ctx);\n public abstract void InitializeCommandHandler(Dictionary handler);\n\n #region [ properties ]\n private string _S = Path.DirectorySeparatorChar.ToString();\n internal ILog _log = LogManager.GetLogger(typeof(aWebServer));\n private bool _run = true;\n private int _port;\n private HttpListener _server;\n private bool _localonly = false;\n private Dictionary _handler = new Dictionary();\n internal Dictionary _WebCache = new Dictionary();\n private AuthenticationSchemes _authenticationType = AuthenticationSchemes.None;\n private string _apiPrefix = \"myapi\";\n #endregion\n\n #region [ web call back handlers ]\n private void ListenerCallback(IAsyncResult ar)\n {\n var listener = ar.AsyncState as HttpListener;\n\n var ctx = listener.EndGetContext(ar);\n _log.Debug(\"Remote Address = \" + ctx.Request.RemoteEndPoint.Address);\n\n try\n {\n //do some stuff\n string path = ctx.Request.Url.GetComponents(UriComponents.Path, UriFormat.Unescaped).ToLower();\n if (ctx.User != null)\n _log.Debug(\"user : \" + ctx.User.Identity.Name);\n else\n {\n //ctx.Response.Redirect(\"login\");\n //return;\n }\n\n string webpath = \"WEB\\\\\";\n webpath = webpath.Replace(\"\\\\\", \"/\");\n bool handled = false;\n if (path.StartsWith(_apiPrefix))\n {\n string command = path.Replace(_apiPrefix + \"/\", \"\");\n if (command.Contains(\"?\"))\n command = command.Substring(0, command.IndexOf('?') - 1);\n if (command.Contains(\"/\"))\n command = command.Substring(0, command.IndexOf('/'));\n\n Handler handler = null;\n\n if (_handler.TryGetValue(command, out handler))\n {\n handled = true;\n handler(ctx);\n }\n }\n\n if (!handled)\n {\n if (path == \"\")\n {\n ctx.Response.ContentType = \"text/html\";\n WriteResponse(ctx, 200, ReadFromStream(_WebCache[(webpath + \"index.html\").ToLower()]), false);\n }\n else\n {\n if (path.EndsWith(_apiPrefix + \".png\") && File.Exists(\"logo.png\"))\n {\n OutPutContentType(ctx, path);\n WriteResponse(ctx, 200, File.ReadAllBytes(\"logo.png\"), false);\n }\n else if (_WebCache.ContainsKey((webpath + path).ToLower()))\n {\n bool compress = OutPutContentType(ctx, path);\n var o = _WebCache[(webpath + path).ToLower()];\n WriteResponse(ctx, 200, ReadFromStream(o), compress);\n }\n else\n WriteResponse(ctx, 404, \"route path not found : \" + ctx.Request.Url.GetComponents(UriComponents.Path, UriFormat.Unescaped));\n }\n }\n ctx.Response.OutputStream.Close();\n }\n catch (Exception ex)\n {\n _log.Error(ex);\n }\n }\n\n\n #endregion\n\n #region [ internal ]\n internal void Stop()\n {\n _run = false;\n }\n\n internal static bool OutPutContentType(HttpListenerContext ctx, string path)\n {\n bool compress = false;\n switch (Path.GetExtension(path).ToLower())\n {\n case \".jpg\":\n case \".jpeg\":\n ctx.Response.ContentType = \"image/jpeg\"; break;\n case \".gif\":\n ctx.Response.ContentType = \"image/gif\"; break;\n case \".mht\":\n compress = true;\n ctx.Response.ContentType = \"multipart/related\"; break;\n case \".eml\":\n compress = true;\n ctx.Response.ContentType = \"message/rfc822\"; break;//\"application/x-mimearchive\"; break;\n case \".json\":\n ctx.Response.ContentEncoding = UTF8Encoding.UTF8;\n ctx.Response.ContentType = \"application/json\"; break;\n case \".js\":\n ctx.Response.ContentEncoding = UTF8Encoding.UTF8;\n ctx.Response.ContentType = \"application/javascript\"; break;\n case \".css\":\n ctx.Response.ContentType = \"text/css\"; break;\n case \".html\":\n case \".htm\":\n compress = true;\n ctx.Response.ContentEncoding = UTF8Encoding.UTF8;\n ctx.Response.ContentType = \"text/html\"; break;\n case \".pdf\":\n ctx.Response.ContentType = \"application/pdf\"; break;\n case \".doc\":\n case \".docx\":\n ctx.Response.ContentType = \"application/msword\"; break;\n case \".xls\":\n case \".xlsx\":\n ctx.Response.ContentType = \"application/vnd.ms-excel\"; break;\n default:\n ctx.Response.ContentType = \"application/octet-stream\"; break;\n\n }\n return compress;\n }\n\n internal void WriteResponse(HttpListenerContext ctx, int code, string msg)\n {\n WriteResponse(ctx, code, Encoding.UTF8.GetBytes(msg), false);\n }\n\n internal void WriteResponse(HttpListenerContext ctx, int code, string msg, bool compress)\n {\n WriteResponse(ctx, code, Encoding.UTF8.GetBytes(msg), compress);\n }\n\n internal void WriteResponse(HttpListenerContext ctx, int code, byte[] data, bool compress)\n {\n ctx.Response.StatusCode = code;\n ctx.Response.AppendHeader(\"Access-Control-Allow-Origin\", \"*\");\n byte[] b = data;\n if (compress == true && b.Length > 100 * 1024)\n {\n _log.Debug(\"original data size : \" + b.Length.ToString(\"#,0\"));\n using (var ms = new MemoryStream())\n {\n using (var zip = new GZipStream(ms, CompressionMode.Compress, true))\n zip.Write(b, 0, b.Length);\n b = ms.ToArray();\n }\n _log.Debug(\"compressed size : \" + b.Length.ToString(\"#,0\"));\n ctx.Response.AppendHeader(\"Content-Encoding\", \"gzip\");\n }\n ctx.Response.ContentLength64 = b.LongLength;\n ctx.Response.OutputStream.Write(b, 0, b.Length);\n }\n #endregion\n\n #region [ private ]\n private void Start()\n {\n try\n {\n InitializeCommandHandler(_handler);\n\n ReadResources();\n\n _server = new HttpListener();\n if (_authenticationType != AuthenticationSchemes.None)\n _server.AuthenticationSchemes = _authenticationType;\n\n if (_localonly)\n {\n _server.Prefixes.Add(\"http://localhost:\" + _port + \"/\");\n _server.Prefixes.Add(\"http://127.0.0.1:\" + _port + \"/\");\n _server.Prefixes.Add(\"http://\" + Environment.MachineName + \":\" + _port + \"/\");\n }\n else\n _server.Prefixes.Add(\"http://*:\" + _port + \"/\");\n\n\n _server.Start();\n while (_run)\n {\n var context = _server.BeginGetContext(new AsyncCallback(ListenerCallback), _server);\n context.AsyncWaitHandle.WaitOne();\n }\n }\n catch (Exception ex)\n {\n _log.Error(ex);\n }\n }\n\n private byte[] ReadFromStream(string name)\n {\n using (MemoryStream ms = new MemoryStream())\n {\n Assembly.GetExecutingAssembly().GetManifestResourceStream(name).CopyTo(ms);\n return ms.ToArray();\n }\n }\n\n private void ReadResources()\n {\n string name = Assembly.GetExecutingAssembly().GetName().Name + \".\";\n foreach (var r in Assembly.GetExecutingAssembly().GetManifestResourceNames())\n {\n string s = r.Replace(name, \"\");\n if (s.StartsWith(\"WEB\"))\n {\n var ext = Path.GetExtension(s);\n s = s.Replace(ext, \"\").Replace(\".\", \"/\");\n var p = s + ext;\n _WebCache.Add(p.ToLower(), r);\n }\n }\n }\n\n #endregion\n }\n}" }, { "path": "RaptorDB/REST/rdbRest.cs", "content": "using RaptorDB.Common;\nusing System;\nusing System.Collections.Generic;\nusing System.Diagnostics;\nusing System.IO;\nusing System.Net;\nusing System.Reflection;\nusing System.Text;\nusing System.Text.RegularExpressions;\n\nnamespace RaptorDB\n{\n #region [ rdb rest helper classes ]\n public class RDBRoute\n {\n public string URL;\n public Type EntityType;\n public string Viewname;\n public ServerSideFunc function;\n public override string ToString()\n {\n if (EntityType != null)\n return \"POST-able type : \" + EntityType.Name;\n else if (function != null)\n return \"Server side function\";\n else if (Viewname != \"\")\n return \"View name : \" + Viewname;\n\n return \"Undefined\";\n }\n }\n\n public interface IRouteAPI\n {\n void AddRoute(RDBRoute route);\n void RegisterView(View view);\n }\n\n public interface IRDBRouting\n {\n void Initialize(IRouteAPI api);\n }\n\n public class RDBJsonContainer\n {\n public string URL;\n public DateTime date;\n public string json;\n public string useragent;\n public Guid docid;\n }\n #endregion\n\n class rdbRest : aWebServer, IRouteAPI\n {\n public rdbRest(int HttpPort, RaptorDB rdb, string routingpath, bool localonly) : base(HttpPort, localonly, System.Net.AuthenticationSchemes.None, \"raptordb\")\n {\n _rdb = rdb;\n //save = _rdb.GetType().GetMethod(\"Save\", BindingFlags.Instance | BindingFlags.Public);\n //register = _rdb.GetType().GetMethod(\"RegisterView\", BindingFlags.Instance | BindingFlags.Public);\n if (_S == \"/\")\n _path = routingpath.Replace(\"\\\\\", \"/\");\n else\n _path = routingpath;\n }\n private string _S = Path.DirectorySeparatorChar.ToString();\n private RaptorDB _rdb;\n private string _path;\n private SafeDictionary _routing = new SafeDictionary();\n //private KeyStore _jsonstore;\n\n\n public new void Stop()\n {\n base.Stop();\n //_jsonstore.Shutdown();\n //_rdb.Shutdown();\n }\n\n public void AddRoute(RDBRoute route)\n {\n _log.Debug(\"adding route : \" + route.URL);\n _routing.Add(route.URL.ToLower(), route);\n }\n\n public void RegisterView(View view)\n {\n _log.Debug(\"registering view : \" + view.Name);\n AddRoute(new RDBRoute { URL = \"RaptorDB/Views/\" + view.Name, Viewname = view.Name });\n _rdb.RegisterView(view);\n }\n\n public override void InitializeCommandHandler(Dictionary _handler)\n {\n _handler.Add(\"getroutes\",\n (ctx) =>\n {\n List o = new List();\n foreach (var rr in _routing)\n o.Add(new { URL = rr.Value.URL, Description = rr.Value.ToString() });\n OutputJsonData(ctx, o);\n });\n\n _handler.Add(\"getviews\",\n (ctx) =>\n {\n List o = new List();\n foreach (var v in _rdb.GetViews())\n o.Add(new { Name = v.Name, Description = v.Description, BackgroundIndexing = v.BackgroundIndexing, Version = v.Version, isPrimaryList = v.isPrimaryList });\n OutputJsonData(ctx, o);\n });\n\n _handler.Add(\"getschema\",\n (ctx) =>\n {\n string qry = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped);\n if (qry == \"\")\n {\n WriteResponse(ctx, 404, \"GetSchema requires a viewname to be defined e.g. ?view=customerview\");\n }\n else\n {\n List o = new List();\n string view = qry.Split('=')[1];\n var sc = _rdb.GetSchema(view);\n foreach (var i in sc.Columns)\n o.Add(new { ColumnName = i.Key, Type = i.Value.Name });\n OutputJsonData(ctx, o);\n }\n });\n\n _handler.Add(\"systeminfo\",\n (ctx) =>\n {\n var oo = GetInfo();\n var s = fastJSON.JSON.ToJSON(oo, new fastJSON.JSONParameters { UseExtensions = false, UseFastGuid = false, EnableAnonymousTypes = true });\n ctx.Response.ContentType = \"application/json\";\n WriteResponse(ctx, 200, s);\n });\n\n _handler.Add(\"action\",\n (ctx) =>\n {\n string action = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped);\n switch (action)\n {\n case \"backup\":\n _rdb.Backup();\n WriteResponse(ctx, 200, \"\\\"Done\\\"\");\n break;\n case \"compact\":\n _rdb.GetKVHF().CompactStorageHF();\n WriteResponse(ctx, 200, \"\\\"Done\\\"\");\n break;\n case \"getconfigs\":\n WriteResponse(ctx, 200, File.ReadAllText(_path + \"raptordb.config\"));\n break;\n }\n });\n\n _handler.Add(\"docget\", // takes : guid \n (ctx) =>\n {\n string qry = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped);\n var g = Guid.Parse(qry);\n _log.Debug(\"docid = \" + qry);\n var s = fastJSON.JSON.ToNiceJSON(_rdb.Fetch(g), new fastJSON.JSONParameters { UseExtensions = true, UseFastGuid = false, UseEscapedUnicode = false });\n ctx.Response.ContentType = \"application/json\";\n WriteResponse(ctx, 200, s);\n });\n\n _handler.Add(\"dochistory\", // takes : guid \n (ctx) =>\n {\n string qry = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped);\n var g = Guid.Parse(qry);\n var h = _rdb.FetchHistoryInfo(g);\n _log.Debug(\"docid = \" + qry);\n var s = fastJSON.JSON.ToJSON(h, new fastJSON.JSONParameters { UseExtensions = false, UseFastGuid = false, UseEscapedUnicode = false });\n ctx.Response.ContentType = \"application/json\";\n WriteResponse(ctx, 200, s);\n });\n\n _handler.Add(\"docversion\", // takes : version\n (ctx) =>\n {\n string qry = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped);\n var v = int.Parse(qry);\n var oo = _rdb.FetchVersion(v);\n var s = fastJSON.JSON.ToNiceJSON(oo, new fastJSON.JSONParameters { UseExtensions = true, UseFastGuid = false, UseEscapedUnicode = false });\n ctx.Response.ContentType = \"application/json\";\n WriteResponse(ctx, 200, s);\n });\n\n _handler.Add(\"fileget\", // takes : guid \n (ctx) =>\n {\n string qry = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped);\n var g = Guid.Parse(qry);\n _log.Debug(\"fileid = \" + qry);\n var s = fastJSON.JSON.ToNiceJSON(_rdb.FetchBytes(g), new fastJSON.JSONParameters { UseExtensions = true, UseFastGuid = false, UseEscapedUnicode = false });\n ctx.Response.ContentType = \"application/json\";\n WriteResponse(ctx, 200, s);\n });\n\n _handler.Add(\"filehistory\", // takes : guid \n (ctx) =>\n {\n string qry = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped);\n var g = Guid.Parse(qry);\n var h = _rdb.FetchBytesHistoryInfo(g);\n _log.Debug(\"fileid = \" + qry);\n var s = fastJSON.JSON.ToJSON(h, new fastJSON.JSONParameters { UseExtensions = false, UseFastGuid = false, UseEscapedUnicode = false });\n ctx.Response.ContentType = \"application/json\";\n WriteResponse(ctx, 200, s);\n });\n\n _handler.Add(\"fileversion\", // takes : version\n (ctx) =>\n {\n string qry = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped);\n var v = int.Parse(qry);\n var oo = _rdb.FetchBytesVersion(v);\n var s = fastJSON.JSON.ToNiceJSON(oo, new fastJSON.JSONParameters { UseExtensions = true, UseFastGuid = false, UseEscapedUnicode = false });\n ctx.Response.ContentType = \"application/json\";\n WriteResponse(ctx, 200, s);\n });\n\n _handler.Add(\"docsearch\", // takes : string & count =x &start=y\n (ctx) =>\n {\n string qry = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped);\n int start = 0;\n int count = -1;\n\n var m = _start_regex.Match(qry);\n if (m.Success)\n {\n start = int.Parse(m.Groups[\"start\"].Value);\n qry = qry.Replace(m.Value, \"\");\n }\n m = _count_regex.Match(qry);\n if (m.Success)\n {\n count = int.Parse(m.Groups[\"count\"].Value);\n qry = qry.Replace(m.Value, \"\");\n }\n var h = _rdb.FullTextSearch(qry);\n List list = new List();\n _log.Debug(\"search = \" + qry);\n if (count > -1 && h.Length > 0)\n {\n int c = h.Length;\n for (int i = start; i < start + count && i < c; i++)\n list.Add(h[i]);\n }\n var obj = new\n {\n Items = list,\n Count = count,\n TotalCount = h.Length\n };\n var s = fastJSON.JSON.ToJSON(obj, new fastJSON.JSONParameters { UseExtensions = false, UseFastGuid = false, UseEscapedUnicode = false, EnableAnonymousTypes = true });\n ctx.Response.ContentType = \"application/json\";\n WriteResponse(ctx, 200, s);\n });\n\n _handler.Add(\"hfkeys\", // takes : count =x &start=y\n (ctx) =>\n {\n string qry = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped);\n int start = 0;\n int count = -1;\n\n var m = _start_regex.Match(qry);\n if (m.Success)\n {\n start = int.Parse(m.Groups[\"start\"].Value);\n qry = qry.Replace(m.Value, \"\");\n }\n m = _count_regex.Match(qry);\n if (m.Success)\n {\n count = int.Parse(m.Groups[\"count\"].Value);\n qry = qry.Replace(m.Value, \"\");\n }\n var h = _rdb.GetKVHF().GetKeysHF();\n List list = new List();\n if (count > -1 && h.Length > 0)\n {\n int c = h.Length;\n for (int i = start; i < start + count && i < c; i++)\n list.Add(h[i]);\n }\n var obj = new\n {\n Items = list,\n Count = count,\n TotalCount = h.Length\n };\n var s = fastJSON.JSON.ToJSON(obj, new fastJSON.JSONParameters { UseExtensions = false, UseFastGuid = false, UseEscapedUnicode = false, EnableAnonymousTypes = true });\n ctx.Response.ContentType = \"application/json\";\n WriteResponse(ctx, 200, s);\n });\n\n _handler.Add(\"hfget\", // takes : string \n (ctx) =>\n {\n string qry = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped);\n var h = _rdb.GetKVHF().GetObjectHF(qry);\n var s = fastJSON.JSON.ToNiceJSON(h, new fastJSON.JSONParameters { UseExtensions = false, UseFastGuid = false, UseEscapedUnicode = false, EnableAnonymousTypes = true });\n ctx.Response.ContentType = \"application/json\";\n WriteResponse(ctx, 200, s);\n });\n\n _handler.Add(\"viewinfo\", // takes : viewname\n (ctx) =>\n {\n string qry = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped);\n if (qry == \"\")\n {\n WriteResponse(ctx, 404, \"ViewInfo requires a viewname to be defined e.g. ?customerview\");\n }\n else\n {\n var vi = GetViewInfo(qry);\n if (vi == \"\")\n WriteResponse(ctx, 500, \"View not found.\");\n else\n {\n ctx.Response.ContentType = \"application/json\";\n WriteResponse(ctx, 200, vi);\n }\n }\n });\n\n _handler.Add(\"excelexport\",\n (ctx) =>\n {\n string path = ctx.Request.Url.GetComponents(UriComponents.Path, UriFormat.Unescaped).ToLower();\n\n var data = DoQuery(_rdb, ctx, path.Replace(\"raptordb/excelexport/\", \"\"), null);\n ctx.Response.AddHeader(\"content-disposition\", \"attachment;filename=\" + data.Title + \".csv\");\n ctx.Response.AddHeader(\"Content-Type\", \"application/vnd.ms-excel\");\n _log.Debug(\"exporting to excel rows : \" + data.Rows.Count);\n WriteResponse(ctx, 200, WriteCsv(data.Rows), true);\n });\n\n _handler.Add(\"views\",\n (ctx) =>\n {\n string path = ctx.Request.Url.GetComponents(UriComponents.Path, UriFormat.Unescaped).ToLower();\n ProcessGET(_rdb, ctx, path.Replace(\"raptordb/views/\", \"\"), null);\n });\n }\n\n\n #region [ private ]\n private string GetViewInfo(string name)\n {\n var v = _rdb.GetViews().Find(x => x.Name.ToLower() == name.ToLower());\n if (v == null)\n return \"\";\n\n var p = new Dictionary();\n\n foreach (var pr in v.Schema.GetProperties())\n p.Add(pr.Name, pr.PropertyType.ToString());\n\n foreach (var pr in v.Schema.GetFields())\n p.Add(pr.Name, pr.FieldType.ToString());\n\n var obj = new\n {\n View = v,\n Schema = p\n };\n var s = fastJSON.JSON.ToJSON(obj, new fastJSON.JSONParameters { UseFastGuid = false, UseEscapedUnicode = false, EnableAnonymousTypes = true });\n return s;\n }\n\n private object GetInfo()\n {\n var ts = _rdb.Uptime();\n var s = \"\" + ts.Days + \" days, \" + ts.Hours + \" hours, \" + ts.Minutes + \" mins, \" + ts.Seconds + \" secs\";\n // get info here\n return new\n {\n DocumentCount = _rdb.DocumentCount(),\n FileCount = _rdb.FileCount(),\n OSVersion = Environment.OSVersion.ToString(),\n NumberOfViews = _rdb.GetViews().Count,\n HighFrequncyItems = _rdb.GetKVHF().CountHF(),\n RaptorDBVersion = FileVersionInfo.GetVersionInfo(this.GetType().Assembly.Location).ProductVersion.ToString(),\n DataFolderSize = _rdb.GetDataFolderSize(),\n Uptime = s,\n MemoryUsage = GetMemoryUsage(),\n LogItems = LogManager.GetLastLogs()\n };\n }\n\n private string GetMemoryUsage() // KLUDGE but works\n {\n try\n {\n string fname = Path.GetFileNameWithoutExtension(Assembly.GetEntryAssembly().Location);\n\n ProcessStartInfo ps = new ProcessStartInfo(\"tasklist\");\n ps.Arguments = \"/fi \\\"IMAGENAME eq \" + fname + \".*\\\" /FO CSV /NH\";\n ps.RedirectStandardOutput = true;\n ps.CreateNoWindow = true;\n ps.UseShellExecute = false;\n var p = Process.Start(ps);\n if (p.WaitForExit(1000))\n {\n var s = p.StandardOutput.ReadToEnd().Split('\\\"');\n return s[9].Replace(\"\\\"\", \"\");\n }\n }\n catch { }\n return \"Unable to get memory usage\";\n }\n\n private string WriteCsv(List data)//, Stream stream)\n {\n StringBuilder output = new StringBuilder();\n var o = data[0];\n\n foreach (var prop in o.GetType().GetProperties())\n {\n output.Append(prop.Name); // header\n output.Append(\",\");\n }\n foreach (var prop in o.GetType().GetFields())\n {\n output.Append(prop.Name); // header\n output.Append(\",\");\n }\n\n output.AppendLine();\n foreach (var item in data)\n {\n foreach (var prop in o.GetType().GetProperties())\n {\n output.Append(\"\\\"\" + prop.GetValue(item, null));\n output.Append(\"\\\",\");\n }\n foreach (var prop in o.GetType().GetFields())\n {\n output.Append(\"\\\"\" + prop.GetValue(item));\n output.Append(\"\\\",\");\n }\n output.AppendLine();\n }\n return output.ToString();\n }\n\n //private MethodInfo register = null;\n //private void CompileAndRegisterScriptRoutes(string routefolder)\n //{\n // // compile & register views\n // string[] files = Directory.GetFiles(routefolder, \"*.route\");\n\n // foreach (var fn in files)\n // {\n // Assembly a = CompileScript(fn);\n // if (a != null)\n // {\n // foreach (var t in a.GetTypes())\n // {\n // if (typeof(IRDBRouting).IsAssignableFrom(t))\n // {\n // IRDBRouting r = (IRDBRouting)Activator.CreateInstance(t);\n // r.Initialize(this);\n // }\n // // load views if exists\n // foreach (var att in t.GetCustomAttributes(typeof(RegisterViewAttribute), false))\n // {\n // try\n // {\n // object o = Activator.CreateInstance(t);\n // // handle types when view also\n // Type[] args = t.GetGenericArguments();\n // if (args.Length == 0)\n // args = t.BaseType.GetGenericArguments();\n // Type tt = args[0];\n // var m = register.MakeGenericMethod(new Type[] { tt });\n // m.Invoke(_rdb, new object[] { o });\n // }\n // catch (Exception ex)\n // {\n // _log.Error(ex);\n // }\n // }\n // }\n // }\n // }\n //}\n\n //private Assembly CompileScript(string file)\n //{\n // try\n // {\n // _log.Debug(\"Compiling route script : \" + file);\n // CodeDomProvider compiler = CodeDomProvider.CreateProvider(\"CSharp\");\n\n // CompilerParameters compilerparams = new CompilerParameters();\n // compilerparams.GenerateInMemory = false;\n // compilerparams.GenerateExecutable = false;\n // compilerparams.OutputAssembly = file.Replace(\".route\", \".dll\");\n // compilerparams.CompilerOptions = \"/optimize\";\n\n // Regex regex = new Regex(\n // @\"\\/\\/\\s*ref\\s*\\:\\s*(?.*)\",\n // System.Text.RegularExpressions.RegexOptions.IgnoreCase);\n\n // compilerparams.ReferencedAssemblies.Add(typeof(View<>).Assembly.Location); //raprotdb.common.dll\n // compilerparams.ReferencedAssemblies.Add(typeof(object).Assembly.Location); //mscorlib.dll\n // compilerparams.ReferencedAssemblies.Add(typeof(System.Uri).Assembly.Location); //system.dll\n // compilerparams.ReferencedAssemblies.Add(typeof(System.Linq.Enumerable).Assembly.Location);//system.core.dll\n // compilerparams.ReferencedAssemblies.Add(typeof(IRDBRouting).Assembly.Location); //raptordb.rest.dll\n\n // foreach (Match m in regex.Matches(File.ReadAllText(file)))\n // {\n // string str = m.Groups[\"refs\"].Value.Trim();\n // Assembly a = Assembly.LoadWithPartialName(Path.GetFileNameWithoutExtension(str));//load from GAC if possible\n // if (a != null)\n // compilerparams.ReferencedAssemblies.Add(a.Location);\n // else\n // {\n // string assm = Path.GetDirectoryName(this.GetType().Assembly.Location) + _S + str;\n // a = Assembly.LoadFrom(assm);\n // if (a != null)\n // compilerparams.ReferencedAssemblies.Add(a.Location);\n // else\n // _log.Error(\"unable to find referenced file for view compiling : \" + str);\n // }\n // }\n\n // CompilerResults results = compiler.CompileAssemblyFromFile(compilerparams, file);\n\n // if (results.Errors.HasErrors == true)\n // {\n // _log.Error(\"Error compiling route definition : \" + file);\n // foreach (var e in results.Errors)\n // _log.Error(e.ToString());\n // return null;\n // }\n\n // return results.CompiledAssembly;\n // }\n // catch (Exception ex)\n // {\n // _log.Error(\"Error compiling route definition : \" + file);\n // _log.Error(ex);\n // return null;\n // }\n //}\n\n private void OutputJsonData(HttpListenerContext ctx, List o)\n {\n Result resf = new Result(true);\n resf.Rows = o;\n resf.TotalCount = o.Count;\n resf.Count = o.Count;\n var s = fastJSON.JSON.ToJSON(resf, new fastJSON.JSONParameters { UseExtensions = false, UseFastGuid = false, EnableAnonymousTypes = true });\n ctx.Response.ContentType = \"application/json\";\n WriteResponse(ctx, 200, s);\n }\n\n private Regex _start_regex = new Regex(@\"[\\?\\&]?\\s*start\\s*\\=\\s*[-+]?(?\\d*)\", RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace | RegexOptions.Compiled);\n private Regex _count_regex = new Regex(@\"[\\?\\&]?\\s*count\\s*\\=\\s*[-+]?(?\\d*)\", RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace | RegexOptions.Compiled);\n private Regex _order_regex = new Regex(@\"[\\?\\&]?\\s*orderby\\s*\\=\\s*[-+]?(?.*)\", RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace | RegexOptions.Compiled);\n\n private Result DoQuery(IRaptorDB rdb, HttpListenerContext ctx, string path, RDBRoute route)\n {\n string qry = ctx.Request.Url.GetComponents(UriComponents.Query, UriFormat.Unescaped);\n string viewname = path;\n if (route != null)\n {\n //if (route.EntityType != null)\n //{\n // if (qry != \"\")\n // {\n // // fetch the json document\n // string[] s = qry.Split('=');\n // object obj = null;\n // if (_jsonstore.GetObject(Guid.Parse(s[1].Replace(\"\\\"\", \"\")), out obj))\n // {\n // RDBJsonContainer d = (RDBJsonContainer)obj;\n // WriteResponse(ctx, 200, d.json);\n // return;\n // }\n // }\n\n // WriteResponse(ctx, 404, \"GUID not found :\" + qry);\n // return;\n //}\n if (route.Viewname == null && route.function != null)\n {\n viewname = route.Viewname;\n var o = route.function(_rdb, qry);\n Result resf = new Result(true);\n resf.Rows = o;\n resf.TotalCount = o.Count;\n resf.Count = o.Count;\n resf.Title = route.Viewname;\n return resf;\n }\n }\n\n // parse \"start\" and \"count\" from qry if exists\n int start = 0;\n int count = -1;\n string orderby = \"\";\n\n var m = _start_regex.Match(qry);\n if (m.Success)\n {\n start = int.Parse(m.Groups[\"start\"].Value);\n qry = qry.Replace(m.Value, \"\");\n }\n m = _count_regex.Match(qry);\n if (m.Success)\n {\n count = int.Parse(m.Groups[\"count\"].Value);\n qry = qry.Replace(m.Value, \"\");\n }\n m = _order_regex.Match(qry);\n if (m.Success)\n {\n orderby = m.Groups[\"orderby\"].Value;\n qry = qry.Replace(m.Value, \"\");\n }\n\n var res = rdb.Query(viewname, qry, start, count, orderby);\n res.Title = viewname;\n return res;\n }\n\n private void ProcessGET(IRaptorDB rdb, HttpListenerContext ctx, string path, RDBRoute route)\n {\n try\n {\n var result = DoQuery(rdb, ctx, path, route);\n var s = fastJSON.JSON.ToJSON(result, new fastJSON.JSONParameters { UseExtensions = false, UseFastGuid = false, EnableAnonymousTypes = true });\n ctx.Response.ContentType = \"application/json\";\n WriteResponse(ctx, 200, s);\n return;\n }\n catch (Exception ex)\n {\n WriteResponse(ctx, 500, \"\" + ex);\n }\n }\n #endregion\n }\n}\n" }, { "path": "RaptorDB/RaptorDB.cs", "content": "using RaptorDB.Common;\nusing RaptorDB.Views;\nusing System;\nusing System.CodeDom.Compiler;\nusing System.Collections.Generic;\nusing System.ComponentModel;\nusing System.IO;\nusing System.IO.Compression;\nusing System.Linq;\nusing System.Linq.Expressions;\nusing System.Reflection;\nusing System.Text;\nusing System.Text.RegularExpressions;\nusing System.Threading;\n\n// ----- Feature list -------\n// TODO : enum in row schema support\n// TODO : validate view schema with mapper on startup ??\n// TODO : HFKV transaction mode set and rollback handling\n// TODO : put page list string key alloc block num in index file header \n\nnamespace RaptorDB\n{\n public class RaptorDB : IRaptorDB\n {\n private RaptorDB(string FolderPath, ITokenizer tokenizer)\n {\n if (tokenizer != null)\n _tokenizer = tokenizer;\n else\n _tokenizer = new tokenizer();\n // speed settings\n fastJSON.JSON.Parameters.ParametricConstructorOverride = true;\n fastBinaryJSON.BJSON.Parameters.ParametricConstructorOverride = true;\n fastJSON.JSON.Parameters.UseEscapedUnicode = false;\n // backward compatibility \n //fastBinaryJSON.BJSON.Parameters.v1_4TypedArray = true;\n fastJSON.Reflection.RDBMode = true;\n\n if (_S == \"/\")\n FolderPath = FolderPath.Replace(\"\\\\\", \"/\");\n // create folders \n Directory.CreateDirectory(FolderPath);\n string foldername = Path.GetFullPath(FolderPath);\n if (foldername.EndsWith(_S) == false)\n foldername += _S;\n _Path = foldername;\n\n // if configs !exists create template config files\n CreateTemplateConfigFiles();\n\n Initialize();\n }\n\n public static RaptorDB Open(string FolderPath)\n {\n return new RaptorDB(FolderPath, new tokenizer());\n }\n\n public static RaptorDB Open(string FolderPath, ITokenizer tokenizer)\n {\n return new RaptorDB(FolderPath, tokenizer);\n }\n\n private string _S = Path.DirectorySeparatorChar.ToString();\n private ILog _log = LogManager.GetLogger(typeof(RaptorDB));\n private ViewManager _viewManager;\n private KeyStore _objStore;\n private KeyStore _fileStore;\n private KeyStoreHF _objHF;\n private string _Path = \"\";\n private int _LastRecordNumberProcessed = -1; // used by background saver\n private int _LastFulltextIndexed = -1; // used by the fulltext indexer\n private int _LastBackupRecordNumber = -1;\n private int _CurrentRecordNumber = -1;\n private System.Timers.Timer _saveTimer;\n private System.Timers.Timer _fulltextTimer;\n private System.Timers.Timer _freeMemTimer;\n private System.Timers.Timer _processinboxTimer;\n private bool _shuttingdown = false;\n private bool _pauseindexer = false;\n private MethodInfo otherviews = null;\n private MethodInfo save = null;\n private MethodInfo saverep = null;\n private SafeDictionary _savecache = new SafeDictionary();\n private SafeDictionary _saverepcache = new SafeDictionary();\n private FullTextIndex _fulltextindex;\n private CronDaemon _cron;\n private Replication.ReplicationServer _repserver;\n private Replication.ReplicationClient _repclient;\n private DateTime _startTime = DateTime.Now;\n private ITokenizer _tokenizer = new tokenizer();\n private int _RaptorDBVersion = 4;\n\n\n #region [ P U B L I C I N T E R F A C E ]\n /// \n /// Save files to RaptorDB\n /// \n /// \n /// \n public bool SaveBytes(Guid docID, byte[] bytes)\n {\n // save files in storage\n _fileStore.SetBytes(docID, bytes);\n return true;\n }\n /// \n /// Delete a document (note data is not lost just flagged as deleted)\n /// \n /// \n /// \n public bool Delete(Guid docid)\n {\n bool b = _objStore.Delete(docid);\n _viewManager.Delete(docid);\n return b;\n }\n\n /// \n /// Delete a file (note data is not lost just flagged as deleted)\n /// \n /// \n /// \n public bool DeleteBytes(Guid bytesid)\n {\n return _fileStore.Delete(bytesid);\n }\n\n /// \n /// Save a document\n /// \n /// \n /// \n /// \n /// \n public bool Save(Guid docid, T data)\n {\n string viewname = _viewManager.GetPrimaryViewForType(data.GetType());\n if (viewname == \"\" && Global.RequirePrimaryView == true)\n {\n _log.Warn(\"Primary View not defined for object : \" + data.GetType());\n return false;\n }\n _pauseindexer = true;\n if (viewname != \"\" && _viewManager.isTransaction(viewname))\n {\n _log.Debug(\"TRANSACTION started for docid : \" + docid);\n // add code here\n _viewManager.StartTransaction();\n\n bool b = SaveInPrimaryViewTransaction(viewname, docid, data);\n if (b == true)\n {\n b = SaveToConsistentViewsTransaction(docid, data);\n if (b == true)\n {\n b = SaveInOtherViewsTransaction(docid, data);\n if (b == true)\n {\n _viewManager.Commit(Thread.CurrentThread.ManagedThreadId);\n int recnum = _objStore.SetObject(docid, data);\n _CurrentRecordNumber = recnum;\n _pauseindexer = false;\n return true;\n }\n }\n }\n _viewManager.Rollback(Thread.CurrentThread.ManagedThreadId);\n _pauseindexer = false;\n return false;\n }\n else\n {\n int recnum = _objStore.SetObject(docid, data);\n _CurrentRecordNumber = recnum;\n\n if (viewname != \"\")\n {\n SaveInPrimaryView(viewname, docid, data);\n\n SaveToConsistentViews(docid, data);\n\n if (Global.BackgroundSaveToOtherViews == false)\n {\n SaveInOtherViews(docid, data);\n _LastRecordNumberProcessed = recnum;\n }\n }\n _pauseindexer = false;\n return true;\n }\n }\n\n internal TimeSpan Uptime()\n {\n return DateTime.Now.Subtract(_startTime);\n }\n\n internal object FileCount()\n {\n return _fileStore.Count();\n }\n\n /// \n /// Query any view -> get all rows\n /// \n /// \n /// \n /// \n public Result Query(string viewname)\n {\n return _viewManager.Query(viewname, 0, -1);\n }\n\n /// \n /// Query a view using a string filter\n /// \n /// \n /// \n /// \n public Result Query(string viewname, string filter)\n {\n if (filter == \"\")\n return _viewManager.Query(viewname, 0, -1);\n\n return _viewManager.Query(viewname, filter, 0, -1);\n }\n\n /// \n /// Fetch a document by it's ID\n /// \n /// \n /// \n public object Fetch(Guid docID)\n {\n object b = null;\n _objStore.GetObject(docID, out b);\n return b;\n }\n\n /// \n /// Fetch file data by it's ID\n /// \n /// \n /// \n public byte[] FetchBytes(Guid fileID)\n {\n byte[] b = null;\n if (_fileStore.GetBytes(fileID, out b))\n return b;\n else\n return null;\n }\n\n /// \n /// Register a view\n /// \n /// \n /// \n public void RegisterView(View view)\n {\n _viewManager.RegisterView(view);\n }\n\n /// \n /// Shutdown the database engine and flush memory to disk\n /// \n public void Shutdown()\n {\n if (_shuttingdown == true)\n return;\n\n _shuttingdown = true;\n\n if (_restServer != null)\n _restServer.Stop();\n\n _processinboxTimer.Enabled = false;\n _saveTimer.Enabled = false;\n _freeMemTimer.Enabled = false;\n _fulltextTimer.Enabled = false;\n\n if (_repserver != null)\n _repserver.Shutdown();\n\n if (_repclient != null)\n _repclient.Shutdown();\n\n // feature : write global or something else?\n File.WriteAllText(_Path + \"RaptorDB.config\", fastJSON.JSON.ToNiceJSON(new Global(), new fastJSON.JSONParameters { UseExtensions = false }));\n if (_cron != null)\n _cron.Stop();\n _fulltextTimer.Stop();\n _fulltextindex.Shutdown();\n\n _log.Debug(\"Shutting down\");\n _saveTimer.Stop();\n _viewManager.ShutDown();\n _objStore.Shutdown();\n _fileStore.Shutdown();\n _objHF.Shutdown();\n\n // save records \n _log.Debug(\"last full text record = \" + _LastFulltextIndexed);\n File.WriteAllBytes(_Path + \"Data\" + _S + \"Fulltext\" + _S + \"_fulltext.rec\", Helper.GetBytes(_LastFulltextIndexed, false));\n _log.Debug(\"last record = \" + _LastRecordNumberProcessed);\n File.WriteAllBytes(_Path + \"Data\" + _S + \"_lastrecord.rec\", Helper.GetBytes(_LastRecordNumberProcessed, false));\n _log.Debug(\"last backup record = \" + _LastBackupRecordNumber);\n File.WriteAllBytes(_Path + \"Backup\" + _S + \"LastBackupRecord.rec\", Helper.GetBytes(_LastBackupRecordNumber, false));\n\n _log.Debug(\"Shutting down log.\");\n _log.Debug(\"RaptorDB done.\");\n LogManager.Shutdown();\n }\n\n\n #region [ BACKUP/RESTORE and REPLICATION ]\n private object _backuplock = new object();\n /// \n /// Backup the document storage file incrementally to \"Backup\" folder\n /// \n /// True = done\n public bool Backup()\n {\n if (_LastBackupRecordNumber >= _CurrentRecordNumber)\n return false;\n lock (_backuplock)\n {\n _log.Debug(\"Backup Started...\");\n string tempp = _Path + \"Temp\" + _S + DateTime.Now.ToString(\"yyyy-MM-dd-HH-mm\");\n Directory.CreateDirectory(tempp);\n StorageFile backup = new StorageFile(tempp + _S + \"backup.mgdat\", SF_FORMAT.BSON, true);\n _log.Debug(\"Copying data to backup\");\n if (_LastBackupRecordNumber == -1)\n _LastBackupRecordNumber = 0;\n int rec = _objStore.CopyTo(backup, _LastBackupRecordNumber);\n backup.Shutdown();\n\n _log.Debug(\"Last backup rec# = \" + rec);\n\n // compress the file\n using (FileStream read = File.OpenRead(tempp + _S + \"backup.mgdat\"))\n using (FileStream outp = File.Create(tempp + _S + \"backup.mgdat.gz\"))\n CompressForBackup(read, outp);\n\n _log.Debug(\"Backup compressed and done\");\n File.Move(tempp + _S + \"backup.mgdat.gz\", _Path + \"Backup\" + _S + DateTime.Now.ToString(\"yyyy-MM-dd-HH-mm\") + \".mgdat.gz\");\n\n _log.Debug(\"last backup record = \" + _LastBackupRecordNumber);\n File.WriteAllBytes(_Path + \"Backup\" + _S + \"LastBackupRecord.rec\", Helper.GetBytes(_LastBackupRecordNumber, false));\n // cleanup temp folder\n Directory.Delete(tempp, true);\n _log.Debug(\"Backup done.\");\n _LastBackupRecordNumber = rec;\n return true;\n }\n }\n\n private DateTime _lastFailedTime = DateTime.Now;\n private object _replock = new object();\n private void ProcessReplicationInbox(string inboxfolder)\n {\n lock (_replock)\n {\n if (Directory.Exists(inboxfolder) == false)\n return;\n\n string[] files = Directory.GetFiles(inboxfolder, \"*.counter\");\n\n // check if \".counter\" file exists\n if (files.Length > 0)\n {\n // FEATURE: if lastfailedtime < 15 -> wait 15 min and retry (avoid extra cpu burning)\n // recovery mode\n string fn = files[0];\n int start = -1;\n if (int.TryParse(File.ReadAllText(fn).Trim(), out start))\n {\n if (DoRepProcessing(fn.Replace(\".counter\", \".mgdat\"), start) == false)\n return;\n }\n else\n {\n _log.Error(\"Unable to parse counter value in : \" + fn);\n return;\n }\n }\n\n files = Directory.GetFiles(inboxfolder, \"*.gz\");\n\n Array.Sort(files);\n foreach (var filename in files)\n {\n\n string tmp = filename.Replace(\".gz\", \"\");// FEATURE : to temp folder ??\n if (File.Exists(tmp))\n File.Delete(tmp);\n using (FileStream read = File.OpenRead(filename))\n using (FileStream outp = File.Create(tmp))\n DecompressForRestore(read, outp);\n _log.Debug(\"Uncompress done : \" + Path.GetFileName(tmp));\n if (DoRepProcessing(tmp, 0) == false)\n return;\n if (_shuttingdown)\n return;\n }\n }\n }\n\n private bool DoRepProcessing(string filename, int start)\n {\n string fn = Path.GetFileNameWithoutExtension(filename);\n string path = Path.GetDirectoryName(filename);\n StorageFile sf = StorageFile.ReadForward(filename);\n int counter = 0;\n if (start > 0)\n _log.Debug(\"skipping replication items : \" + start);\n foreach (var i in sf.ReadOnlyEnumerate())\n {\n if (start > 0) // skip already done\n {\n start--;\n counter++;\n }\n else\n {\n if (i.meta.isDeleted)\n DeleteReplicate(i.meta.key);\n else\n {\n try\n {\n object obj = CreateObject(i.data);\n var m = GetSaveReplicate(obj.GetType());\n m.Invoke(this, new object[] { i.meta.key, obj });\n }\n catch (Exception ex)\n {\n _log.Error(ex);\n sf.Shutdown();\n string err = Properties.Resources.msg.Replace(\"%js%\", fastJSON.JSON.Beautify(Helper.GetString(i.data, 0, (short)i.data.Length)))\n .Replace(\"%ex%\", \"\" + ex)\n .Replace(\"%c%\", path + _S + fn + \".counter\");\n\n File.WriteAllText(path + _S + fn + \".error.txt\", err);\n _lastFailedTime = DateTime.Now;\n return false;\n }\n }\n counter++;\n File.WriteAllText(path + _S + fn + \".counter\", \"\" + counter);\n if (_shuttingdown)\n {\n _log.Debug(\"shutting down before replicate data completed...\");\n sf.Shutdown();\n return false;\n }\n }\n }\n sf.Shutdown();\n _log.Debug(\"File replicate complete : \" + Path.GetFileName(filename));\n foreach (var f in Directory.GetFiles(path, fn + \".*\"))\n File.Delete(f);\n return true;\n }\n\n private void DeleteReplicate(Guid docid)\n {\n bool b = _objStore.DeleteReplicated(docid);\n _viewManager.Delete(docid);\n }\n\n private object _restoreLock = new object();\n /// \n /// Start background restore of backups in the \"Restore\" folder\n /// \n public void Restore()\n {\n lock (_restoreLock)\n {\n try\n {\n string[] files = Directory.GetFiles(_Path + \"Restore\", \"*.counter\");\n // check if \".counter\" file exists\n if (files.Length > 0)\n {\n // resume mode\n string fn = files[0];\n int start = -1;\n if (int.TryParse(File.ReadAllText(fn).Trim(), out start))\n {\n if (DoRestoreProcessinng(fn.Replace(\".counter\", \".mgdat\"), start) == false)\n return;\n }\n else\n {\n _log.Error(\"Unable to parse counter value in : \" + fn);\n return;\n }\n }\n // do restore \n files = Directory.GetFiles(_Path + \"Restore\", \"*.gz\");\n Array.Sort(files);\n _log.Debug(\"Restoring file count = \" + files.Length);\n\n foreach (string file in files)\n {\n string tmp = file.Replace(\".gz\", \"\");// FEATURE : to temp folder ??\n if (File.Exists(tmp))\n File.Delete(tmp);\n using (FileStream read = File.OpenRead(file))\n using (FileStream outp = File.Create(tmp))\n DecompressForRestore(read, outp);\n _log.Debug(\"Uncompress done : \" + Path.GetFileName(tmp));\n\n if (DoRestoreProcessinng(tmp, 0))\n File.Move(file, _Path + \"Restore\" + _S + \"Done\" + _S + Path.GetFileName(file));\n }\n }\n catch (Exception ex)\n {\n _log.Error(ex);\n }\n }\n }\n\n private bool DoRestoreProcessinng(string filename, int start)\n {\n string fn = Path.GetFileNameWithoutExtension(filename);\n string path = Path.GetDirectoryName(filename);\n int counter = 0;\n StorageFile sf = StorageFile.ReadForward(filename);\n foreach (var i in sf.ReadOnlyEnumerate())\n {\n if (start > 0)\n {\n start--;\n counter++;\n }\n else\n {\n if (i.meta.isDeleted)\n Delete(i.meta.key);\n else\n {\n object obj = CreateObject(i.data);\n var m = GetSave(obj.GetType());\n m.Invoke(this, new object[] { i.meta.key, obj });\n }\n counter++;\n File.WriteAllText(path + _S + fn + \".counter\", \"\" + counter);\n if (_shuttingdown)\n {\n _log.Debug(\"shutting down before restore completed...\");\n sf.Shutdown();\n return false;\n }\n }\n }\n sf.Shutdown();\n _log.Debug(\"File restore complete : \" + Path.GetFileName(filename));\n foreach (var f in Directory.GetFiles(path, fn + \".*\"))\n File.Delete(f);\n\n return true;\n }\n\n private bool SaveReplicationObject(Guid docid, T data)\n {\n string viewname = _viewManager.GetPrimaryViewForType(data.GetType());\n if (viewname == \"\")\n {\n _log.Debug(\"Primary View not defined for object : \" + data.GetType());\n return false;\n }\n _pauseindexer = true;\n int recnum = _objStore.SetReplicationObject(docid, data);\n _CurrentRecordNumber = recnum;\n\n SaveInPrimaryView(viewname, docid, data);\n\n SaveToConsistentViews(docid, data);\n\n if (Global.BackgroundSaveToOtherViews == false)\n {\n SaveInOtherViews(docid, data);\n _LastRecordNumberProcessed = recnum;\n }\n _pauseindexer = false;\n return true;\n }\n #endregion\n\n /// \n /// Add a user (only supported in server mode)\n /// \n /// \n /// \n /// \n /// \n public bool AddUser(string username, string oldpassword, string newpassword)\n {\n return false;\n }\n\n /// \n /// Execute a server side string filter query\n /// \n /// \n /// \n /// \n public object[] ServerSide(ServerSideFunc func, string filter)\n {\n return func(this, filter).ToArray();\n }\n\n /// \n /// Execute a server side LINQ query\n /// \n /// \n /// \n /// \n /// \n public object[] ServerSide(ServerSideFunc func, Expression> filter)\n {\n LINQString ls = new LINQString();\n ls.Visit(filter);\n return func(this, ls.sb.ToString()).ToArray();\n }\n\n /// \n /// Full text search the entire original document\n /// \n /// \n /// \n public int[] FullTextSearch(string filter)\n {\n var wbmp = _fulltextindex.Query(filter, _objStore.RecordCount());\n List a = new List();\n a.AddRange(wbmp.GetBitIndexes());\n\n return a.ToArray();\n }\n\n /// \n /// Query a view\n /// \n /// \n /// \n /// \n public Result Query(Expression> filter)\n {\n return _viewManager.Query(filter, 0, -1);\n }\n\n /// \n /// Query a view with paging\n /// \n /// \n /// \n /// \n /// \n /// \n public Result Query(Expression> filter, int start, int count)\n {\n return _viewManager.Query(filter, start, count, \"\");\n }\n\n /// \n /// Query a view with paging and order by\n /// \n /// \n /// \n /// \n /// \n /// \n /// \n public Result Query(Expression> filter, int start, int count, string orderby)\n {\n return _viewManager.Query(filter, start, count, orderby);\n }\n\n /// \n /// Query a view \n /// \n /// \n /// \n /// \n public Result Query(string filter)\n {\n return _viewManager.Query(filter, 0, -1);\n }\n\n /// \n /// Query a view with paging\n /// \n /// \n /// \n /// \n /// \n /// \n public Result Query(string filter, int start, int count)\n {\n return _viewManager.Query(filter, start, count);\n }\n\n /// \n /// Count with filter\n /// \n /// \n /// \n /// \n public int Count(Expression> filter)\n {\n return _viewManager.Count(filter);\n }\n\n /// \n /// \n /// \n /// \n /// \n /// \n /// \n public Result Query(string viewname, int start, int count)\n {\n return _viewManager.Query(viewname, start, count);\n }\n\n /// \n /// \n /// \n /// \n /// \n /// \n /// \n /// \n public Result Query(string viewname, string filter, int start, int count)\n {\n return _viewManager.Query(viewname, filter, start, count);\n }\n\n /// \n /// Count all data associated with View name\n /// \n /// \n /// \n public int Count(string viewname)\n {\n return _viewManager.Count(viewname, \"\");\n }\n\n /// \n /// Count all data associated with View name and string filter\n /// \n /// \n /// \n /// \n public int Count(string viewname, string filter)\n {\n return _viewManager.Count(viewname, filter);\n }\n\n /// \n /// Fetch the change history for a document\n /// \n /// \n /// \n public int[] FetchHistory(Guid docid)\n {\n return _objStore.GetHistory(docid);\n }\n\n /// \n /// Fetch a change history for a file\n /// \n /// \n /// \n public int[] FetchBytesHistory(Guid fileid)\n {\n return _fileStore.GetHistory(fileid);\n }\n\n /// \n /// Fetch the specific document version \n /// \n /// \n /// \n public object FetchVersion(int versionNumber)\n {\n StorageItem meta = null;\n return _objStore.GetObject(versionNumber, out meta);\n }\n\n /// \n /// Fetch the specific file version\n /// \n /// \n /// \n public byte[] FetchBytesVersion(int versionNumber)\n {\n StorageItem meta = null;\n return _fileStore.GetBytes(versionNumber, out meta);\n }\n /// \n /// Get the current registered views\n /// \n /// \n public List GetViews()\n {\n return _viewManager.GetViews();\n }\n\n /// \n /// Get the schema for a view\n /// \n /// \n /// \n public ViewRowDefinition GetSchema(string view)\n {\n return _viewManager.GetSchema(view);\n }\n\n /// \n /// Query a view with paging and ordering\n /// \n /// \n /// \n /// \n /// \n /// \n /// \n public Result Query(string viewname, string filter, int start, int count, string orderby)\n {\n return _viewManager.Query(viewname, filter, start, count, orderby);\n }\n\n /// \n /// Query a view with paging and ordering\n /// \n /// \n /// \n /// \n /// \n /// \n /// \n public Result Query(string filter, int start, int count, string orderby)\n {\n return _viewManager.Query(filter, start, count, orderby);\n }\n\n /// \n /// Get the history information for a document\n /// \n /// \n /// \n public HistoryInfo[] FetchHistoryInfo(Guid docid)\n {\n List h = new List();\n\n foreach (int i in FetchHistory(docid))\n {\n HistoryInfo hi = new HistoryInfo();\n hi.Version = i;\n var o = _objStore.GetMeta(i);\n hi.ChangeDate = o.date;\n if (o.isDeleted == false)\n h.Add(hi);\n }\n return h.ToArray();\n }\n\n /// \n /// Get the history information for a file\n /// \n /// \n /// \n public HistoryInfo[] FetchBytesHistoryInfo(Guid docid)\n {\n List h = new List();\n\n foreach (int i in FetchBytesHistory(docid))\n {\n HistoryInfo hi = new HistoryInfo();\n hi.Version = i;\n var o = _fileStore.GetMeta(i);\n hi.ChangeDate = o.date;\n if (o.isDeleted == false)\n h.Add(hi);\n }\n return h.ToArray();\n }\n\n /// \n /// Direct delete from a view\n /// \n /// \n /// \n /// \n public int ViewDelete(Expression> filter)\n {\n // do the delete\n int c = _viewManager.ViewDelete(filter);\n if (c > 0)\n {\n if (Global.SkipDocsOnViewInsert)\n {\n // save this filter to docs\n View_delete vd = new View_delete();\n LINQString lq = new LINQString();\n lq.Visit(filter);\n vd.Filter = lq.sb.ToString();\n vd.Viewname = _viewManager.GetViewName(typeof(TRowSchema));\n _objStore.SetObject(vd.ID, vd);\n }\n }\n return c;\n }\n\n /// \n /// Direct delete from a view\n /// \n /// \n /// \n /// \n public int ViewDelete(string viewname, string filter)\n {\n // do the delete\n int c = _viewManager.ViewDelete(viewname, filter);\n if (c > 0)\n {\n if (Global.SkipDocsOnViewInsert)\n {\n // save this filter to docs\n View_delete vd = new View_delete();\n vd.Filter = filter;\n vd.Viewname = viewname;\n _objStore.SetObject(vd.ID, vd);\n }\n }\n return c;\n }\n\n /// \n /// Direct insert into a view\n /// \n /// \n /// \n /// \n /// \n public bool ViewInsert(Guid id, TRowSchema row)\n {\n string vn = _viewManager.GetViewName(typeof(TRowSchema));\n if (vn != \"\")\n {\n if (_viewManager.ViewInsert(id, row))\n {\n if (Global.SkipDocsOnViewInsert == false)\n {\n View_insert vi = new View_insert();\n vi.ID = id;\n vi.Viewname = vn;\n vi.RowObject = row;\n _objStore.SetObject(vi.ID, vi);\n }\n return true;\n }\n }\n return false;\n }\n\n /// \n /// Direct insert into a view\n /// \n /// \n /// \n /// \n /// \n public bool ViewInsert(string viewname, Guid id, object row)\n {\n if (_viewManager.ViewInsert(viewname, id, row))\n {\n if (Global.SkipDocsOnViewInsert == false)\n {\n View_insert vi = new View_insert();\n vi.ID = id;\n vi.Viewname = viewname;\n vi.RowObject = row;\n _objStore.SetObject(vi.ID, vi);\n }\n return true;\n }\n return false;\n }\n\n /// \n /// Total number of documents in the storage file including duplicates\n /// \n /// \n public long DocumentCount()\n {\n return _objStore.Count();\n }\n\n public IKeyStoreHF GetKVHF()\n {\n return _objHF;\n }\n\n public object[] ServerSide(ServerSideFuncWithArgs func, string filter, params object[] args)\n {\n return func(this, filter, args).ToArray();\n }\n\n public object[] ServerSide(ServerSideFuncWithArgs func, Expression> filter, params object[] args)\n {\n LINQString ls = new LINQString();\n ls.Visit(filter);\n return func(this, ls.sb.ToString(), args).ToArray();\n }\n #endregion\n\n #region [ P R I V A T E M E T H O D S ]\n\n internal string GetViewName(Type type)\n {\n return _viewManager.GetViewName(type);\n }\n\n private bool SaveToView(Guid docid, T data, List list)\n {\n if (list != null)\n foreach (string name in list)\n {\n bool ret = _viewManager.InsertTransaction(name, docid, data);\n if (ret == false)\n return false;\n }\n return true;\n }\n\n private bool SaveInOtherViewsTransaction(Guid docid, T data)\n {\n List list = _viewManager.GetOtherViewsList(data.GetType());\n return SaveToView(docid, data, list);\n }\n\n private bool SaveToConsistentViewsTransaction(Guid docid, T data)\n {\n List list = _viewManager.GetConsistentViews(data.GetType());\n return SaveToView(docid, data, list);\n }\n\n private bool SaveInPrimaryViewTransaction(string viewname, Guid docid, T data)\n {\n return _viewManager.InsertTransaction(viewname, docid, data);\n }\n\n private static void PumpDataForBackup(Stream input, Stream output)\n {\n byte[] bytes = new byte[4096 * 2];\n int n;\n while ((n = input.Read(bytes, 0, bytes.Length)) != 0)\n output.Write(bytes, 0, n);\n }\n\n private static void CompressForBackup(Stream source, Stream destination)\n {\n using (GZipStream gz = new GZipStream(destination, CompressionMode.Compress))\n PumpDataForBackup(source, gz);\n }\n\n private static void DecompressForRestore(Stream source, Stream destination)\n {\n using (GZipStream gz = new GZipStream(source, CompressionMode.Decompress))\n PumpDataForBackup(gz, destination);\n }\n\n private void SaveToConsistentViews(Guid docid, T data)\n {\n List list = _viewManager.GetConsistentViews(data.GetType());\n if (list != null)\n foreach (string name in list)\n {\n //_log.Info(\"Saving to consistent view : \" + name);\n _viewManager.Insert(name, docid, data);\n }\n }\n\n private object CreateObject(byte[] b)\n {\n if (b[0] < 32)\n return fastBinaryJSON.BJSON.ToObject(b);\n else\n return fastJSON.JSON.ToObject(Encoding.ASCII.GetString(b));\n }\n\n private void SaveInOtherViews(Guid docid, T data)\n {\n List list = _viewManager.GetOtherViewsList(data.GetType());\n if (list != null)\n foreach (string name in list)\n _viewManager.Insert(name, docid, data);\n }\n\n private void SaveInPrimaryView(string viewname, Guid docid, T data)\n {\n _viewManager.Insert(viewname, docid, data);\n }\n\n private void Initialize()\n {\n // read raptordb.config here (running parameters)\n if (File.Exists(_Path + \"RaptorDB.config\"))\n fastJSON.JSON.FillObject(new Global(), File.ReadAllText(_Path + \"RaptorDB.config\"));\n\n Directory.CreateDirectory(_Path + \"Data\");\n Directory.CreateDirectory(_Path + \"Data\" + _S + \"Fulltext\");\n Directory.CreateDirectory(_Path + \"Views\");\n Directory.CreateDirectory(_Path + \"Logs\");\n Directory.CreateDirectory(_Path + \"Temp\");\n Directory.CreateDirectory(_Path + \"Backup\");\n Directory.CreateDirectory(_Path + \"Restore\");\n Directory.CreateDirectory(_Path + \"Restore\" + _S + \"Done\");\n // load logger\n LogManager.Configure(_Path + \"Logs\" + _S + \"log.txt\", 500, false);\n\n _log.Debug(\"\\r\\n\\r\\nRaptorDB starting...\");\n _log.Debug(\"RaptorDB data folder = \" + _Path);\n\n // check doc & file storage file version and upgrade if needed here\n int v = StorageFile.GetStorageFileHeaderVersion(_Path + \"Data\" + _S + \"data\");\n if (v < StorageFile._CurrentVersion)\n UpgradeStorageFile(_Path + \"Data\" + _S + \"data\", v);\n\n v = StorageFile.GetStorageFileHeaderVersion(_Path + \"Data\" + _S + \"files\");\n if (v < StorageFile._CurrentVersion)\n UpgradeStorageFile(_Path + \"Data\" + _S + \"files\", v);\n\n // upgrade old mgidx files \n if (File.Exists(_Path + \"Data\" + _S + \"RaptorDB.version\") == false)\n {\n RebuildDataFiles();\n }\n\n File.WriteAllText(_Path + \"data\" + _S + \"RaptorDB.version\", \"\" + _RaptorDBVersion);\n\n _objStore = new KeyStore(_Path + \"Data\" + _S + \"data\", true);\n _fileStore = new KeyStore(_Path + \"Data\" + _S + \"files\", true);\n\n _objHF = new KeyStoreHF(_Path + \"DataHF\");\n\n _viewManager = new Views.ViewManager(_Path + \"Views\", _objStore, _objHF, _tokenizer);\n\n // load _LastFulltextIndexed \n if (File.Exists(_Path + \"Data\" + _S + \"Fulltext\" + _S + \"_fulltext.rec\"))\n {\n byte[] b = File.ReadAllBytes(_Path + \"Data\" + _S + \"Fulltext\" + _S + \"_fulltext.rec\");\n _LastFulltextIndexed = Helper.ToInt32(b, 0, false);\n }\n // load _LastRecordNumberProcessed \n if (File.Exists(_Path + \"Data\" + _S + \"_lastrecord.rec\"))\n {\n byte[] b = File.ReadAllBytes(_Path + \"Data\" + _S + \"_lastrecord.rec\");\n _LastRecordNumberProcessed = Helper.ToInt32(b, 0, false);\n }\n // load _LastBackupRecordNumber \n if (File.Exists(_Path + \"Backup\" + _S + \"LastBackupRecord.rec\"))\n {\n byte[] b = File.ReadAllBytes(_Path + \"Backup\" + _S + \"LastBackupRecord.rec\");\n _LastBackupRecordNumber = Helper.ToInt32(b, 0, false);\n }\n _CurrentRecordNumber = _objStore.RecordCount();\n\n otherviews = this.GetType().GetMethod(\"SaveInOtherViews\", BindingFlags.Instance | BindingFlags.NonPublic);\n save = this.GetType().GetMethod(\"Save\", BindingFlags.Instance | BindingFlags.Public);\n saverep = this.GetType().GetMethod(\"SaveReplicationObject\", BindingFlags.Instance | BindingFlags.NonPublic);\n\n _fulltextindex = new FullTextIndex(_Path + \"Data\" + _S + \"Fulltext\", \"fulltext\", true, false, _tokenizer);\n\n // start backround save to views\n _saveTimer = new System.Timers.Timer(Global.BackgroundSaveViewTimer * 1000);\n _saveTimer.Elapsed += new System.Timers.ElapsedEventHandler(_saveTimer_Elapsed);\n _saveTimer.Enabled = true;\n _saveTimer.AutoReset = true;\n _saveTimer.Start();\n\n // start full text timer \n _fulltextTimer = new System.Timers.Timer(Global.FullTextTimerSeconds * 1000);\n _fulltextTimer.Elapsed += new System.Timers.ElapsedEventHandler(_fulltextTimer_Elapsed);\n _fulltextTimer.Enabled = true;\n _fulltextTimer.AutoReset = true;\n _fulltextTimer.Start();\n\n // start free memory timer \n _freeMemTimer = new System.Timers.Timer(Global.FreeMemoryTimerSeconds * 1000);\n _freeMemTimer.Elapsed += new System.Timers.ElapsedEventHandler(_freeMemTimer_Elapsed);\n _freeMemTimer.Enabled = true;\n _freeMemTimer.AutoReset = true;\n _freeMemTimer.Start();\n\n // start inbox procesor timer \n _processinboxTimer = new System.Timers.Timer(Global.ProcessInboxTimerSeconds * 1000);\n _processinboxTimer.Elapsed += new System.Timers.ElapsedEventHandler(_processinboxTimer_Elapsed);\n _processinboxTimer.Enabled = true;\n _processinboxTimer.AutoReset = true;\n _processinboxTimer.Start();\n\n // start cron daemon\n _cron = new CronDaemon();\n _cron.AddJob(Global.BackupCronSchedule, () => this.Backup());\n\n // compile & register view files\n CompileAndRegisterScriptViews(_Path + \"Views\");\n\n\n if (File.Exists(_Path + \"RaptorDB-Replication.config\"))\n {\n // if replication.config exists -> start replication server\n _repserver = new Replication.ReplicationServer(_Path, File.ReadAllText(_Path + \"RaptorDB-Replication.config\"), _objStore);\n }\n else if (File.Exists(_Path + \"RaptorDB-Branch.config\"))\n {\n // if branch.config exists -> start replication client\n _repclient = new Replication.ReplicationClient(_Path, File.ReadAllText(_Path + \"RaptorDB-Branch.config\"), _objStore);\n }\n if (Global.EnableWebStudio)\n {\n _log.Debug(\"Enabling WEBSTUDIO on port : \" + Global.WebStudioPort);\n _restServer = new rdbRest(Global.WebStudioPort, this, _Path, Global.LocalOnlyWebStudio);\n }\n }\n\n private void RebuildDataFiles()\n {\n try\n {\n var data = _Path + \"data\" + _S;\n var tmp = data + \"temp\" + _S;\n _log.Debug(\"Rebuilding MGDAT files...\");\n\n _log.Debug(\" deleting Fulltext folder.\");\n // delete \"fulltext\" folder -> will rebuild automatically\n Directory.Delete(data + \"Fulltext\", true);\n\n _log.Debug(\" moving mgdat files to temp folder.\");\n Directory.CreateDirectory(tmp);\n File.Move(data + \"data.mgdat\", tmp + \"data.mgdat\");\n File.Move(data + \"files.mgdat\", tmp + \"files.mgdat\");\n\n // delete all index files -> keep data.mgdat & files.mgdat\n Directory.GetFiles(data, \"*.*\").ToList().ForEach(x => File.Delete(x));\n\n // upgrade by rebuilding indexes \n var sf = StorageFile.ReadForward(tmp + \"data.mgdat\");\n var obstore = new KeyStore(data + \"data\", true);\n _log.Debug(\" rebuilding index for data.mgdat\");\n _log.Debug(\" docs count = \" + sf.Count());\n foreach (var i in sf.ReadOnlyEnumerate())\n {\n if (i.meta.isDeleted)\n obstore.Delete(i.meta.key);\n else\n {\n var o = CreateObject(i.data);\n obstore.SetObject(i.meta.key, o);\n }\n }\n obstore.Shutdown();\n sf.Shutdown();\n sf = StorageFile.ReadForward(tmp + \"files.mgdat\");\n obstore = new KeyStore(data + \"files\", true);\n _log.Debug(\" rebuilding index for files.mgdat\");\n _log.Debug(\" files count = \" + sf.Count());\n foreach (var i in sf.ReadOnlyEnumerate())\n {\n if (i.meta.isDeleted)\n obstore.Delete(i.meta.key);\n else\n {\n var o = CreateObject(i.data);\n obstore.SetObject(i.meta.key, o);\n }\n }\n obstore.Shutdown();\n sf.Shutdown();\n File.WriteAllText(data + \"RaptorDB.version\", \"\" + _RaptorDBVersion);\n Directory.Delete(tmp, true);\n }\n catch (Exception ex)\n {\n _log.Error(ex);\n }\n }\n\n object _inboxlock = new object();\n void _processinboxTimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)\n {\n lock (_inboxlock)\n {\n string d = _Path + \"Replication\" + _S + \"Inbox\";\n if (Directory.Exists(d) == false)\n return;\n\n // start inbox processing timer\n ProcessReplicationInbox(d);\n\n foreach (var f in Directory.GetDirectories(d))\n ProcessReplicationInbox(f);\n }\n }\n\n private void CompileAndRegisterScriptViews(string viewfolder)\n {\n // compile & register views\n string[] files = Directory.GetFiles(viewfolder, \"*.view\");\n MethodInfo register = this.GetType().GetMethod(\"RegisterView\", BindingFlags.Instance | BindingFlags.Public);\n foreach (var fn in files)\n {\n Assembly a = CompileScript(fn);\n if (a != null)\n {\n foreach (var t in a.GetTypes())\n {\n foreach (var att in t.GetCustomAttributes(typeof(RegisterViewAttribute), false))\n {\n try\n {\n object o = Activator.CreateInstance(t);\n // handle types when view also\n Type[] args = t.GetGenericArguments();\n if (args.Length == 0)\n args = t.BaseType.GetGenericArguments();\n Type tt = args[0];\n var m = register.MakeGenericMethod(new Type[] { tt });\n m.Invoke(this, new object[] { o });\n }\n catch (Exception ex)\n {\n _log.Error(ex);\n }\n }\n }\n }\n }\n }\n\n private Assembly CompileScript(string file)\n {\n try\n {\n _log.Debug(\"Compiling script view : \" + file);\n CodeDomProvider compiler = CodeDomProvider.CreateProvider(\"CSharp\");\n\n CompilerParameters compilerparams = new CompilerParameters();\n compilerparams.GenerateInMemory = false;\n compilerparams.GenerateExecutable = false;\n compilerparams.OutputAssembly = file.Replace(\".view\", \".dll\");\n compilerparams.CompilerOptions = \"/optimize\";\n\n Regex regex = new Regex(\n @\"\\/\\/\\s*ref\\s*\\:\\s*(?.*)\",\n System.Text.RegularExpressions.RegexOptions.IgnoreCase);\n\n compilerparams.ReferencedAssemblies.Add(typeof(View<>).Assembly.Location);\n compilerparams.ReferencedAssemblies.Add(typeof(object).Assembly.Location);\n compilerparams.ReferencedAssemblies.Add(typeof(ICustomTypeDescriptor).Assembly.Location);\n\n foreach (Match m in regex.Matches(File.ReadAllText(file)))\n {\n string str = m.Groups[\"refs\"].Value.Trim();\n#pragma warning disable 618\n Assembly a = Assembly.LoadWithPartialName(Path.GetFileNameWithoutExtension(str));//load from GAC if possible\n#pragma warning restore 618\n if (a != null)\n compilerparams.ReferencedAssemblies.Add(a.Location);\n else\n {\n string assm = Path.GetDirectoryName(this.GetType().Assembly.Location) + _S + str;\n a = Assembly.LoadFrom(assm);\n if (a != null)\n compilerparams.ReferencedAssemblies.Add(a.Location);\n else\n _log.Error(\"unable to find referenced file for view compiling : \" + str);\n }\n }\n\n CompilerResults results = compiler.CompileAssemblyFromFile(compilerparams, file);\n\n if (results.Errors.HasErrors == true)\n {\n _log.Error(\"Error compiling view definition : \" + file);\n foreach (var e in results.Errors)\n _log.Error(e.ToString());\n return null;\n }\n\n return results.CompiledAssembly;\n }\n catch (Exception ex)\n {\n _log.Error(\"Error compiling view definition : \" + file);\n _log.Error(ex);\n return null;\n }\n }\n\n void _freeMemTimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)\n {\n FreeMemory();\n }\n\n private void UpgradeStorageFile(string filename, int ver)\n {\n _log.Debug(\"Upgrading storage file version from \" + ver + \" to \" + StorageFile._CurrentVersion + \" on file : \" + filename);\n throw new Exception(\"not implemented yet - contact the author if you need this functionality\");\n // read from one file and write to the other \n }\n\n private object _slock = new object();\n private void _saveTimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)\n {\n if (_shuttingdown)\n return;\n\n if (Global.BackgroundSaveToOtherViews == false)\n return;\n\n if (_CurrentRecordNumber == 0)\n return;\n\n if (_CurrentRecordNumber == _LastRecordNumberProcessed)\n return;\n\n lock (_slock)\n {\n int batch = Global.BackgroundViewSaveBatchSize;\n while (batch > 0)\n {\n if (_shuttingdown)\n return;\n while (_pauseindexer) Thread.Sleep(0);\n if (_CurrentRecordNumber == _LastRecordNumberProcessed)\n return;\n _LastRecordNumberProcessed++;\n StorageItem meta = null;\n object obj = _objStore.GetObject(_LastRecordNumberProcessed, out meta);\n if (meta != null && meta.isDeleted)\n _viewManager.Delete(meta.key);\n else\n {\n if (obj == null)\n {\n _log.Debug(\"byte[] is null\");\n _log.Debug(\"curr rec = \" + _CurrentRecordNumber);\n _log.Debug(\"last rec = \" + _LastRecordNumberProcessed);\n continue;\n }\n\n var m = otherviews.MakeGenericMethod(new Type[] { obj.GetType() });\n m.Invoke(this, new object[] { meta.key, obj });\n }\n\n batch--;\n }\n }\n }\n\n private object _flock = new object();\n private Regex _jsonfilter = new Regex(\"[\\\\[\\\\]\\\"{}:,]\", RegexOptions.Compiled);\n private rdbRest _restServer = null;\n\n private void _fulltextTimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)\n {\n if (_shuttingdown)\n return;\n\n if (_CurrentRecordNumber == 0)\n return;\n\n if (_CurrentRecordNumber == _LastFulltextIndexed)\n return;\n\n lock (_flock)\n {\n int batch = Global.BackgroundFullTextIndexBatchSize;\n while (batch > 0)\n {\n if (_shuttingdown)\n return;\n //_log.Debug(\"batch full text indexing...\");\n while (_pauseindexer) Thread.Sleep(0);\n if (_CurrentRecordNumber == _LastFulltextIndexed)\n return;\n _LastFulltextIndexed++;\n StorageItem meta = null;\n object obj = _objStore.GetObject(_LastFulltextIndexed, out meta);\n if (meta != null && meta.isDeleted == false)\n {\n if (obj != null)\n {\n // normal string and normal guid \n string json = fastJSON.JSON.ToJSON(obj, new fastJSON.JSONParameters { UseEscapedUnicode = false, UseFastGuid = false });\n json = _jsonfilter.Replace(json, \" \"); // filter out json characters\n _fulltextindex.Set(json, _LastFulltextIndexed);\n }\n }\n batch--;\n }\n\n return;\n }\n }\n\n private MethodInfo GetSave(Type type)\n {\n MethodInfo m = null;\n if (_savecache.TryGetValue(type, out m))\n return m;\n\n m = save.MakeGenericMethod(new Type[] { type });\n _savecache.Add(type, m);\n return m;\n }\n\n private MethodInfo GetSaveReplicate(Type type)\n {\n MethodInfo m = null;\n if (_saverepcache.TryGetValue(type, out m))\n return m;\n\n m = saverep.MakeGenericMethod(new Type[] { type });\n _saverepcache.Add(type, m);\n return m;\n }\n\n private void CreateTemplateConfigFiles()\n {\n if (File.Exists(_Path + \"RaptorDB.config\") == false)\n File.WriteAllText(_Path + \"-RaptorDB.config\", fastJSON.JSON.ToNiceJSON(new Global(), new fastJSON.JSONParameters { UseExtensions = false }));\n\n if (File.Exists(_Path + \"RaptorDB-Branch.config\") == false)\n File.WriteAllText(_Path + \"-RaptorDB-Branch.config\", fastJSON.JSON.ToNiceJSON(new Replication.ClientConfiguration(), new fastJSON.JSONParameters { UseExtensions = false }));\n\n if (File.Exists(_Path + \"RaptorDB-Replication.config\") == false)\n {\n Replication.ServerConfiguration s = new Replication.ServerConfiguration();\n s.What.Add(new Replication.WhatItem { Name = \"default\", PackageItemLimit = 10000, Version = 1, B2HQtypes = new List { \"*\" }, HQ2Btypes = new List { \"*\" } });\n s.What.Add(new Replication.WhatItem { Name = \"b2\", PackageItemLimit = 10000, Version = 1, B2HQtypes = new List { \"*\" }, HQ2Btypes = new List { \"config.*\" } });\n s.Where.Add(new Replication.WhereItem { BranchName = \"b1\", Password = \"123\", When = \"*/5 * * * *\", What = \"default\" });\n s.Where.Add(new Replication.WhereItem { BranchName = \"b2\", Password = \"321\", When = \"*/20 * * * *\", What = \"b2\" });\n File.WriteAllText(_Path + \"-RaptorDB-Replication.config\", fastJSON.JSON.ToNiceJSON(s, new fastJSON.JSONParameters { UseExtensions = false }));\n }\n }\n\n internal long GetDataFolderSize()\n {\n // get data folder size\n return GetDirectorySize(_Path);\n }\n\n internal long GetDirectorySize(string path)\n {\n long b = 0;\n foreach (var p in Directory.GetDirectories(path))\n {\n b += GetDirectorySize(p);\n string[] a = Directory.GetFiles(p, \"*.*\");\n\n foreach (string name in a)\n {\n FileInfo info = new FileInfo(name);\n b += info.Length;\n }\n }\n return b;\n }\n\n internal object GetAssemblyForView(string viewname, out string typename)\n {\n return _viewManager.GetAssemblyForView(viewname, out typename);\n }\n\n public T Fetch(Guid docID) where T : class\n {\n return Fetch(docID) as T;\n }\n\n public void FreeMemory()\n {\n long l = GC.GetTotalMemory(true) / (1024 * 1024);\n _log.Debug(\"GC.GetTotalMemory() = \" + l.ToString(\"#,0\"));\n if (l > Global.MemoryLimit)\n {\n _log.Debug(\"Freeing memory on \" + Global.MemoryLimit.ToString(\"#,0\") + \" limit ...\");\n _viewManager.FreeMemory();\n _fulltextindex.FreeMemory();\n _objStore.FreeMemory();\n _fileStore.FreeMemory();\n _objHF.FreeMemory();\n GC.Collect();// GC.MaxGeneration);\n }\n }\n\n public void SaveToDocsOnViewInsert(bool yes)\n {\n Global.SkipDocsOnViewInsert = yes;\n }\n\n #endregion\n }\n}\n" }, { "path": "RaptorDB/RaptorDB.csproj", "content": "\n\n \n {45F6BE30-989A-4749-B6A0-69099C8661F4}\n Debug\n AnyCPU\n \n \n RaptorDB\n False\n Library\n RaptorDB\n OnBuildSuccess\n \n \n \n \n v4.0\n false\n true\n 0\n false\n true\n true\n \n \n \n bin\\Debug\\\n true\n 285212672\n false\n \n \n true\n 4096\n false\n false\n false\n false\n false\n 4\n Full\n prompt\n false\n TRACE;DEBUG\n \n \n bin\\Release\\\n false\n 285212672\n false\n \n \n \n \n false\n 4096\n false\n \n \n true\n false\n false\n false\n 4\n none\n prompt\n false\n \n \n Auto\n AnyCPU\n \n \n TRACE\n \n \n Project\n \n \n ..\\raptordb.snk\n \n \n \n \n \n \n \n \n BuildVersion.cs\n \n \n Code\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n True\n True\n Resources.resx\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n Code\n \n \n \n \n \n \n \n \n \n \n RaptorDB_Doc.nuspec\n Designer\n \n \n \n \n \n {32331D51-5BE0-41E2-AF1A-9B086C5AE809}\n RaptorDB.Common\n \n \n \n \n ResXFileCodeGenerator\n Resources.Designer.cs\n Designer\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n md \"$(SolutionDir)output\\net40\"\ncopy \"$(TargetPath)\" \"$(SolutionDir)output\\net40\\$(TargetFileName)\"\n \n" }, { "path": "RaptorDB/RaptorDBServer.cs", "content": "using System;\nusing System.Collections.Generic;\nusing System.Text;\nusing RaptorDB.Common;\nusing System.Reflection;\nusing System.IO;\nusing System.Threading.Tasks;\n\nnamespace RaptorDB\n{\n public delegate void Handler(Packet data, ReturnPacket ret);\n\n public class instance_handler\n {\n public bool Initialized = false;\n public string dbpath;\n public MethodInfo register;\n public MethodInfo save;\n public RaptorDB rdb;\n public DateTime lastUsed = DateTime.MinValue;\n public bool hasExtensions = false;\n public SafeDictionary saveCache = new SafeDictionary();\n public SafeDictionary ssideCache = new SafeDictionary();\n public SafeDictionary sswcideCache = new SafeDictionary();\n }\n\n public class RaptorDBServer\n {\n public RaptorDBServer(int port, string DataPath)\n {\n _path = Directory.GetCurrentDirectory();\n AppDomain.CurrentDomain.AssemblyResolve += new ResolveEventHandler(CurrentDomain_AssemblyResolve);\n _server = new NetworkServer();\n\n if (_S == \"/\")// unix system\n _datapath = DataPath.Replace(\"\\\\\", \"/\");\n else\n _datapath = DataPath;\n\n if (_datapath.EndsWith(_S) == false)\n _datapath += _S;\n\n // check if \"instances\" folder exist -> multi instance mode\n if (Directory.Exists(_datapath + \"instances\"))\n {\n _log.Debug(\"Insances exist, loading...\");\n _multiInstance = true;\n foreach (var d in Directory.GetDirectories(_datapath + \"instances\"))\n {\n var dn = new DirectoryInfo(d);\n var i = new instance_handler();\n i.dbpath = d;\n if (Directory.Exists(d + _S + \"Extensions\"))\n i.hasExtensions = true;\n _instances.Add(dn.Name.ToLower(), i);\n }\n }\n _defaultInstance.rdb = RaptorDB.Open(DataPath);\n _defaultInstance.register = _defaultInstance.rdb.GetType().GetMethod(\"RegisterView\", BindingFlags.Instance | BindingFlags.Public);\n _defaultInstance.save = _defaultInstance.rdb.GetType().GetMethod(\"Save\", BindingFlags.Instance | BindingFlags.Public);\n\n Initialize();\n _server.Start(port, processpayload);\n\n // add timer to cleanup connected clients\n _concleanuptimer = new System.Timers.Timer(30 * 1000);\n _concleanuptimer.AutoReset = true;\n _concleanuptimer.Enabled = true;\n _concleanuptimer.Elapsed += _concleanuptimer_Elapsed;\n\n _unusedinstancetimer = new System.Timers.Timer(300 * 1000);// FIX : configuration here\n _unusedinstancetimer.AutoReset = true;\n _unusedinstancetimer.Enabled = true;\n _unusedinstancetimer.Elapsed += _unusedinstancetimer_Elapsed;\n }\n\n private object _lock = new object();\n private void _unusedinstancetimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)\n {\n lock (_lock)\n {\n bool freed = false;\n // clear unused rdb instances\n if (_multiInstance)\n {\n foreach (var i in _instances)\n {\n if (i.Value.rdb != null &&\n FastDateTime.Now.Subtract(i.Value.lastUsed).TotalMinutes > 60) // FIX : configuration here\n {\n var r = i.Value;\n r.rdb.Shutdown();\n r.Initialized = false;\n r.register = null;\n r.save = null;\n r.saveCache = new SafeDictionary();\n r.ssideCache = new SafeDictionary();\n r.sswcideCache = new SafeDictionary();\n r.rdb = null;\n\n freed = true;\n }\n }\n if (freed)\n GC.Collect();\n }\n }\n }\n\n void _concleanuptimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)\n {\n _connectedClients.Clear();\n }\n\n private bool _multiInstance = false;\n private SafeDictionary _instances = new SafeDictionary();\n private instance_handler _defaultInstance = new instance_handler();\n\n private string _S = Path.DirectorySeparatorChar.ToString();\n private Dictionary _users = new Dictionary();\n private string _path = \"\";\n private string _datapath = \"\";\n private ILog _log = LogManager.GetLogger(typeof(RaptorDBServer));\n private NetworkServer _server;\n //private RaptorDB _raptor;\n //private MethodInfo register = null;\n //private MethodInfo save = null;\n //private SafeDictionary _savecache = new SafeDictionary();\n //private SafeDictionary _ssidecache = new SafeDictionary();\n //private SafeDictionary _sswcidecache = new SafeDictionary();\n private Dictionary _handlers = new Dictionary();\n private const string _RaptorDB_users_config = \"RaptorDB-Users.config\";\n private SafeDictionary _connectedClients = new SafeDictionary();\n private System.Timers.Timer _concleanuptimer;\n private System.Timers.Timer _unusedinstancetimer;\n\n public int ConnectedClients { get { return _connectedClients.Count(); } }\n\n private Assembly CurrentDomain_AssemblyResolve(object sender, ResolveEventArgs args) // FIX : handle instance??\n {\n if (File.Exists(args.Name))\n return Assembly.LoadFrom(args.Name);\n string[] ss = args.Name.Split(',');\n string fname = ss[0] + \".dll\";\n if (File.Exists(fname))\n return Assembly.LoadFrom(fname);\n fname = \"Extensions\" + _S + fname;\n if (File.Exists(fname))\n return Assembly.LoadFrom(fname);\n else return null;\n }\n\n private MethodInfo GetSave(Type type)\n {\n MethodInfo m = null;\n if (_defaultInstance.saveCache.TryGetValue(type, out m))\n return m;\n\n m = _defaultInstance.save.MakeGenericMethod(new Type[] { type });\n _defaultInstance.saveCache.Add(type, m);\n return m;\n }\n\n public void Shutdown()\n {\n WriteUsers();\n _server.Stop();\n _defaultInstance.rdb.Shutdown();\n\n foreach (var i in _instances)\n {\n _log.Debug(\"Shutting down instance : \" + i.Key);\n if (i.Value.rdb != null)\n i.Value.rdb.Shutdown();\n }\n }\n\n private void WriteUsers()\n {\n // write users to user.config file\n StringBuilder sb = new StringBuilder();\n sb.AppendLine(\"# FORMAT : username , pasword hash\");\n sb.AppendLine(\"# To disable a user comment the line with the '#'\");\n foreach (var kv in _users)\n {\n sb.AppendLine(kv.Key + \" , \" + kv.Value);\n }\n\n File.WriteAllText(_datapath + _RaptorDB_users_config, sb.ToString());\n }\n\n private object processpayload(object data)\n {\n Packet p = (Packet)data;\n ReturnPacket ret = new ReturnPacket(true);\n\n if (Authenticate(p) == false)\n return new ReturnPacket(false, \"Authentication failed\");\n if (p.Command == \"_close\")\n {\n _connectedClients.Remove(p.ClientID);\n return ret;\n }\n else\n _connectedClients.Add(p.ClientID, true);\n\n try\n {\n Handler d = null;\n if (_handlers.TryGetValue(p.Command, out d))\n d(p, ret);\n else\n _log.Error(\"Command handler not found : \" + p.Command);\n }\n catch (Exception ex)\n {\n ret.OK = false;\n _log.Error(ex);\n }\n return ret;\n }\n\n private RaptorDB GetInstance(string name)\n {\n // load or get instance\n instance_handler inst = null;\n _instances.TryGetValue(name.ToLower(), out inst);\n if(inst==null)\n {\n // no instance found -> err\n _log.Debug(\"instance name not found : \" + name);\n return null;\n }\n if(inst.rdb==null)\n {\n // try loading raptordb instance\n var r = RaptorDB.Open(inst.dbpath);\n inst.rdb = r;\n // fix : create register and save \n //inst.register \n //inst.save\n\n if (inst.hasExtensions)\n {\n // fix: load extension folder\n }\n else\n {\n // fix: load default extenstions\n }\n }\n return inst.rdb;\n }\n\n private void InitializeCommandsDictionary()\n {\n // FIX : route to instance on p.InstanceName\n\n _handlers.Add(\"\" + COMMANDS.Save,\n (p, ret) =>\n {\n var m = GetSave(p.Data.GetType());\n ret.OK = true;\n m.Invoke(_defaultInstance.rdb, new object[] { p.Docid, p.Data });\n });\n\n _handlers.Add(\"\" + COMMANDS.SaveBytes,\n (p, ret) =>\n {\n ret.OK = _defaultInstance.rdb.SaveBytes(p.Docid, (byte[])p.Data);\n });\n\n _handlers.Add(\"\" + COMMANDS.QueryType,\n (p, ret) =>\n {\n var param = (object[])p.Data;\n Type t = Type.GetType((string)param[0]);\n string viewname = _defaultInstance.rdb.GetViewName(t);\n ret.OK = true;\n ret.Data = _defaultInstance.rdb.Query(viewname, (string)param[1], p.Start, p.Count, p.OrderBy);\n });\n\n _handlers.Add(\"\" + COMMANDS.QueryStr,\n (p, ret) =>\n {\n ret.OK = true;\n ret.Data = _defaultInstance.rdb.Query(p.Viewname, (string)p.Data, p.Start, p.Count, p.OrderBy);\n });\n\n _handlers.Add(\"\" + COMMANDS.Fetch,\n (p, ret) =>\n {\n ret.OK = true;\n ret.Data = _defaultInstance.rdb.Fetch(p.Docid);\n });\n\n _handlers.Add(\"\" + COMMANDS.FetchBytes,\n (p, ret) =>\n {\n ret.OK = true;\n ret.Data = _defaultInstance.rdb.FetchBytes(p.Docid);\n });\n\n _handlers.Add(\"\" + COMMANDS.Backup,\n (p, ret) =>\n {\n ret.OK = _defaultInstance.rdb.Backup();\n });\n\n _handlers.Add(\"\" + COMMANDS.Delete,\n (p, ret) =>\n {\n ret.OK = _defaultInstance.rdb.Delete(p.Docid);\n });\n\n _handlers.Add(\"\" + COMMANDS.DeleteBytes,\n (p, ret) =>\n {\n ret.OK = _defaultInstance.rdb.DeleteBytes(p.Docid);\n });\n\n _handlers.Add(\"\" + COMMANDS.Restore,\n (p, ret) =>\n {\n ret.OK = true;\n Task.Factory.StartNew(() => _defaultInstance.rdb.Restore());\n });\n\n _handlers.Add(\"\" + COMMANDS.AddUser,\n (p, ret) =>\n {\n var param = (object[])p.Data;\n ret.OK = AddUser((string)param[0], (string)param[1], (string)param[2]);\n });\n\n _handlers.Add(\"\" + COMMANDS.ServerSide,\n (p, ret) =>\n {\n var param = (object[])p.Data;\n ret.OK = true;\n ret.Data = _defaultInstance.rdb.ServerSide(GetServerSideFuncCache(param[0].ToString(), param[1].ToString()), param[2].ToString());\n });\n\n _handlers.Add(\"\" + COMMANDS.FullText,\n (p, ret) =>\n {\n var param = (object[])p.Data;\n ret.OK = true;\n ret.Data = _defaultInstance.rdb.FullTextSearch(\"\" + param[0]);\n });\n\n _handlers.Add(\"\" + COMMANDS.CountType,\n (p, ret) =>\n {\n // count type\n var param = (object[])p.Data;\n Type t = Type.GetType((string)param[0]);\n string viewname2 = _defaultInstance.rdb.GetViewName(t);\n ret.OK = true;\n ret.Data = _defaultInstance.rdb.Count(viewname2, (string)param[1]);\n });\n\n _handlers.Add(\"\" + COMMANDS.CountStr,\n (p, ret) =>\n {\n // count str\n ret.OK = true;\n ret.Data = _defaultInstance.rdb.Count(p.Viewname, (string)p.Data);\n });\n\n _handlers.Add(\"\" + COMMANDS.GCount,\n (p, ret) =>\n {\n Type t = Type.GetType(p.Viewname);\n string viewname3 = _defaultInstance.rdb.GetViewName(t);\n ret.OK = true;\n ret.Data = _defaultInstance.rdb.Count(viewname3, (string)p.Data);\n });\n\n _handlers.Add(\"\" + COMMANDS.DocHistory,\n (p, ret) =>\n {\n ret.OK = true;\n ret.Data = _defaultInstance.rdb.FetchHistory(p.Docid);\n });\n\n _handlers.Add(\"\" + COMMANDS.FileHistory,\n (p, ret) =>\n {\n ret.OK = true;\n ret.Data = _defaultInstance.rdb.FetchBytesHistory(p.Docid);\n });\n\n _handlers.Add(\"\" + COMMANDS.FetchVersion,\n (p, ret) =>\n {\n ret.OK = true;\n ret.Data = _defaultInstance.rdb.FetchVersion((int)p.Data);\n });\n\n _handlers.Add(\"\" + COMMANDS.FetchFileVersion,\n (p, ret) =>\n {\n ret.OK = true;\n ret.Data = _defaultInstance.rdb.FetchBytesVersion((int)p.Data);\n });\n\n _handlers.Add(\"\" + COMMANDS.CheckAssembly,\n (p, ret) =>\n {\n ret.OK = true;\n string typ = \"\";\n ret.Data = _defaultInstance.rdb.GetAssemblyForView(p.Viewname, out typ);\n ret.Error = typ;\n });\n _handlers.Add(\"\" + COMMANDS.FetchHistoryInfo,\n (p, ret) =>\n {\n ret.OK = true;\n ret.Data = _defaultInstance.rdb.FetchHistoryInfo(p.Docid);\n });\n\n _handlers.Add(\"\" + COMMANDS.FetchByteHistoryInfo,\n (p, ret) =>\n {\n ret.OK = true;\n ret.Data = _defaultInstance.rdb.FetchBytesHistoryInfo(p.Docid);\n });\n\n _handlers.Add(\"\" + COMMANDS.ViewDelete,\n (p, ret) =>\n {\n ret.OK = true;\n var param = (object[])p.Data;\n ret.Data = _defaultInstance.rdb.ViewDelete((string)param[0], (string)param[1]);\n });\n\n _handlers.Add(\"\" + COMMANDS.ViewDelete_t,\n (p, ret) =>\n {\n ret.OK = true;\n var param = (object[])p.Data;\n Type t = Type.GetType((string)param[0]);\n string viewname4 = _defaultInstance.rdb.GetViewName(t);\n ret.Data = _defaultInstance.rdb.ViewDelete(viewname4, (string)param[1]);\n });\n\n _handlers.Add(\"\" + COMMANDS.ViewInsert,\n (p, ret) =>\n {\n ret.OK = true;\n var param = (object[])p.Data;\n ret.Data = _defaultInstance.rdb.ViewInsert((string)param[0], p.Docid, param[1]);\n });\n\n _handlers.Add(\"\" + COMMANDS.ViewInsert_t,\n (p, ret) =>\n {\n ret.OK = true;\n var param = (object[])p.Data;\n Type t = Type.GetType((string)param[0]);\n string viewname5 = _defaultInstance.rdb.GetViewName(t);\n ret.Data = _defaultInstance.rdb.ViewInsert(viewname5, p.Docid, param[1]);\n });\n\n _handlers.Add(\"\" + COMMANDS.DocCount,\n (p, ret) =>\n {\n ret.OK = true;\n ret.Data = _defaultInstance.rdb.DocumentCount();\n });\n\n _handlers.Add(\"\" + COMMANDS.GetObjectHF,\n (p, ret) =>\n {\n ret.OK = true;\n ret.Data = _defaultInstance.rdb.GetKVHF().GetObjectHF((string)p.Data);\n });\n\n _handlers.Add(\"\" + COMMANDS.SetObjectHF,\n (p, ret) =>\n {\n ret.OK = true;\n var param = (object[])p.Data;\n _defaultInstance.rdb.GetKVHF().SetObjectHF((string)param[0], param[1]);\n });\n\n _handlers.Add(\"\" + COMMANDS.DeleteKeyHF,\n (p, ret) =>\n {\n ret.OK = true;\n ret.Data = _defaultInstance.rdb.GetKVHF().DeleteKeyHF((string)p.Data);\n });\n\n _handlers.Add(\"\" + COMMANDS.CountHF,\n (p, ret) =>\n {\n ret.OK = true;\n ret.Data = _defaultInstance.rdb.GetKVHF().CountHF();\n });\n\n _handlers.Add(\"\" + COMMANDS.ContainsHF,\n (p, ret) =>\n {\n ret.OK = true;\n ret.Data = _defaultInstance.rdb.GetKVHF().ContainsHF((string)p.Data);\n });\n\n _handlers.Add(\"\" + COMMANDS.GetKeysHF,\n (p, ret) =>\n {\n ret.OK = true;\n ret.Data = _defaultInstance.rdb.GetKVHF().GetKeysHF();\n });\n\n _handlers.Add(\"\" + COMMANDS.CompactStorageHF,\n (p, ret) =>\n {\n ret.OK = true;\n _defaultInstance.rdb.GetKVHF().CompactStorageHF();\n });\n\n _handlers.Add(\"\" + COMMANDS.IncrementHF,\n (p, ret) =>\n {\n ret.OK = true;\n var param = (object[])p.Data;\n if (param[1] is int)\n ret.Data = _defaultInstance.rdb.GetKVHF().Increment((string)param[0], (int)param[1]);\n else\n ret.Data = _defaultInstance.rdb.GetKVHF().Increment((string)param[0], (decimal)param[1]);\n });\n\n _handlers.Add(\"\" + COMMANDS.DecrementHF,\n (p, ret) =>\n {\n ret.OK = true;\n var param = (object[])p.Data;\n if (param[1] is int)\n ret.Data = _defaultInstance.rdb.GetKVHF().Decrement((string)param[0], (int)param[1]);\n else\n ret.Data = _defaultInstance.rdb.GetKVHF().Decrement((string)param[0], (decimal)param[1]);\n });\n\n _handlers.Add(\"\" + COMMANDS.ServerSideWithArgs,\n (p, ret) =>\n {\n var param = (object[])p.Data;\n ret.OK = true;\n ret.Data = _defaultInstance.rdb.ServerSide(GetServerSideFuncWithArgsCache(param[0].ToString(), param[1].ToString()), param[2].ToString(), param[3]);\n });\n\n _handlers.Add(\"\" + COMMANDS.FreeMemory,\n (p, ret) =>\n {\n ret.OK = true;\n _log.Debug(\"Free memory called from client\");\n _defaultInstance.rdb.FreeMemory();\n });\n }\n\n private ServerSideFuncWithArgs GetServerSideFuncWithArgsCache(string type, string method)\n {\n ServerSideFuncWithArgs func;\n _log.Debug(\"Calling Server side Function with args : \" + method + \" on type \" + type);\n if (_defaultInstance.sswcideCache.TryGetValue(type + method, out func) == false)\n {\n Type tt = Type.GetType(type);\n\n func = (ServerSideFuncWithArgs)Delegate.CreateDelegate(typeof(ServerSideFuncWithArgs), tt, method);\n _defaultInstance.sswcideCache.Add(type + method, func);\n }\n return func;\n }\n\n private ServerSideFunc GetServerSideFuncCache(string type, string method)\n {\n ServerSideFunc func;\n _log.Debug(\"Calling Server side Function : \" + method + \" on type \" + type);\n if (_defaultInstance.ssideCache.TryGetValue(type + method, out func) == false)\n {\n Type tt = Type.GetType(type);\n\n func = (ServerSideFunc)Delegate.CreateDelegate(typeof(ServerSideFunc), tt, method);\n _defaultInstance.ssideCache.Add(type + method, func);\n }\n return func;\n }\n\n private uint GenHash(string user, string pwd)\n {\n return Helper.MurMur.Hash(Encoding.UTF8.GetBytes(user.ToLower() + \"|\" + pwd));\n }\n\n private bool AddUser(string user, string oldpwd, string newpwd)\n {\n uint hash = 0;\n if (_users.TryGetValue(user.ToLower(), out hash) == false)\n {\n _users.Add(user.ToLower(), GenHash(user, newpwd));\n return true;\n }\n if (hash == GenHash(user, oldpwd))\n {\n _users[user.ToLower()] = GenHash(user, newpwd);\n return true;\n }\n return false;\n }\n\n private bool Authenticate(Packet p)\n {\n uint pwd;\n if (_users.TryGetValue(p.Username.ToLower(), out pwd))\n {\n uint hash = uint.Parse(p.PasswordHash);\n if (hash == pwd) return true;\n }\n _log.Debug(\"Authentication failed for '\" + p.Username + \"' hash = \" + p.PasswordHash);\n return false;\n }\n\n private void Initialize()\n {\n // load users here\n if (File.Exists(_datapath + _RaptorDB_users_config))\n {\n foreach (string line in File.ReadAllLines(_datapath + _RaptorDB_users_config))\n {\n if (line.Contains(\"#\") == false)\n {\n string[] s = line.Split(',');\n _users.Add(s[0].Trim().ToLower(), uint.Parse(s[1].Trim()));\n }\n }\n }\n // add default admin user if not exists\n if (_users.ContainsKey(\"admin\") == false)\n _users.Add(\"admin\", GenHash(\"admin\", \"admin\"));\n\n // exe folder\n // |-Extensions\n Directory.CreateDirectory(_path + _S + \"Extensions\");\n\n // open extensions folder\n string path = _path + _S + \"Extensions\";\n\n foreach (var f in Directory.GetFiles(path, \"*.dll\"))\n {\n // - load all dll files\n // - register views \n _log.Debug(\"loading dll for views : \" + f);\n Assembly a = Assembly.Load(f);\n foreach (var t in a.GetTypes())\n {\n foreach (var att in t.GetCustomAttributes(typeof(RegisterViewAttribute), false))\n {\n try\n {\n object o = Activator.CreateInstance(t);\n // handle types when view also\n Type[] args = t.GetGenericArguments();\n if (args.Length == 0)\n args = t.BaseType.GetGenericArguments();\n Type tt = args[0];\n var m = _defaultInstance.register.MakeGenericMethod(new Type[] { tt });\n m.Invoke(_defaultInstance.rdb, new object[] { o });\n }\n catch (Exception ex)\n {\n _log.Error(ex);\n }\n }\n }\n }\n\n InitializeCommandsDictionary();\n }\n }\n}\n" }, { "path": "RaptorDB/Replication/Configuration.cs", "content": "using System.Collections.Generic;\n\nnamespace RaptorDB.Replication\n{\n public class WhereItem\n {\n public string BranchName;\n public string Password;\n public string What;\n public string When;\n }\n\n public class WhatItem\n {\n public WhatItem()\n {\n HQ2Btypes = new List();\n B2HQtypes = new List();\n }\n public string Name;\n public int Version = 1;\n public bool PropogateHQDeletes = true;\n public int PackageItemLimit = 10000;\n public List HQ2Btypes;\n public List B2HQtypes;\n }\n\n public class ServerConfiguration\n {\n public ServerConfiguration()\n {\n Where = new List();\n What = new List();\n ReplicationPort = 9999;\n }\n public int ReplicationPort;\n //public string EmbeddedClientHandler;\n public List Where;\n public List What;\n }\n\n public enum REPMODE\n {\n Branch,\n Server\n }\n\n public class ClientConfiguration\n {\n public ClientConfiguration()\n {\n ServerReplicationPort = 9999;\n }\n public string ServerAddress = \"\";\n public int ServerReplicationPort;\n public string Password = \"\";\n public string BranchName = \"\";\n }\n}\n" }, { "path": "RaptorDB/Replication/Packets.cs", "content": "namespace RaptorDB.Replication\n{\n public class ReplicationPacket\n {\n //public int number;\n public string passwordhash;\n public string branchname;// source name\n public uint datahash; \n public string filename;\n public object data;\n public string command;\n public int lastrecord;\n }\n}\n" }, { "path": "RaptorDB/Replication/Readme.txt", "content": "DATA Folder\n |\n\t |- Replication > (branch mode)\n\t | |-: branch.dat\n\t | |\n\t . |- Inbox >\n\t | |-: 0000000n.mgdat.gz, \n\t\t\t\t|\n\t\t\t\t|- Outbox > \n\n\n- if inbox contains : \"0000000n.counter\" then error occurred and the text is in \"0000000n.error.txt\"\n- you can skip the offending document if you increment the \"counter\" file (when you can't overcome the exception)\n- files will be downloaded to the inbox folder in branch mode\n- \"branch.dat\" in the \"Replication\" folder stores counter information for replication \n\n\n\nDATA Folder\n |\n\t |- Replication > (HQ mode)\n\t | |-: BranchName1.last\n\t | |\n\t | |- Inbox >\n\t | | |\n\t . | |- BranchName1 >\n\t | | |-: 0000000n.mgdat.gz\n\t | | |\n\t\t\t\t|\n\t\t\t\t|- Outbox >\n\t\t\t\t| | \n\t\t\t\t| |- BranchName1 > \n\t\t\t\t| |- BranchName2 >\n\n\n- if inbox contains : \"0000000n.counter\" then error occurred and the text is in \"0000000n.error.txt\"\n- you can skip the offending document if you increment the \"counter\" file (when you can't overcome the exception)\n- files will be downloaded to the inbox folder in branch mode\n\n\n" }, { "path": "RaptorDB/Replication/ReplicationClient.cs", "content": "using System;\nusing System.IO;\nusing System.IO.Compression;\nusing RaptorDB.Common;\nusing System.Text.RegularExpressions;\n\nnamespace RaptorDB.Replication\n{\n public class ClientRepConfig\n {\n public bool isConfigured;\n public string whencron = \"* * * * *\";\n public WhatItem what;\n public int lastHQCounter;\n public int lastCounter;\n public int outPackageNumber;\n public int inPackageNumber;\n public int lastPackageIndex;\n }\n\n public class ClientWhatWhenConfig\n {\n public WhatItem what;\n public string whencron;\n }\n\n /// \n /// Replication package processing is done in RaptorDB.cs\n /// \n internal class ReplicationClient\n {\n public ReplicationClient(string dataFolder, string config, IDocStorage docs)\n {\n _log.Debug(\"starting replication client : \" + dataFolder);\n _docs = docs;\n _path = dataFolder;\n // read client config file\n _config = fastJSON.JSON.ToObject(config);\n Initialize();\n }\n\n private void Initialize()\n {\n Directory.CreateDirectory(_path + \"Replication\");\n Directory.CreateDirectory(_path + \"Replication\" + _S + \"Inbox\");\n Directory.CreateDirectory(_path + \"Replication\" + _S + \"Outbox\");\n _InboxPath = _path + \"Replication\" + _S + \"Inbox\" + _S;\n _OutboxPath = _path + \"Replication\" + _S + \"Outbox\" + _S;\n // setup cron job\n _cron = new CronDaemon();\n\n _clientConfig = new ClientRepConfig();\n // read what config\n if (File.Exists(_path + \"Replication\" + _S + \"branch.dat\"))\n _clientConfig = fastBinaryJSON.BJSON.ToObject(File.ReadAllBytes(_path + \"Replication\" + _S + \"branch.dat\"));\n // starting jobs\n _cron.AddJob(_clientConfig.whencron, Replicate);\n }\n\n\n private ILog _log = LogManager.GetLogger(typeof(ReplicationClient));\n IDocStorage _docs;\n private CronDaemon _cron;\n private string _S = Path.DirectorySeparatorChar.ToString();\n private NetworkClient _client;\n private Replication.ClientConfiguration _config;\n private ClientRepConfig _clientConfig;\n private string _path;\n private string _OutboxPath;\n private string _InboxPath;\n private int INTERNALLIMIT = Global.PackageSizeItemCountLimit;\n\n public void Shutdown()\n {\n if (_cron != null)\n _cron.Stop();\n\n SaveConfig();\n }\n\n private void SaveConfig()\n {\n if (_clientConfig == null)\n return;\n if (_clientConfig.isConfigured == false)\n return;\n // write config to disk\n byte[] b = fastBinaryJSON.BJSON.ToBJSON(_clientConfig);\n File.WriteAllBytes(_path + \"Replication\" + _S + \"branch.dat\", b);\n }\n\n private object _lock = new object();\n private void Replicate()\n {\n lock (_lock)\n {\n try\n {\n if (ConnectToHQ())\n {\n SendPackageToHQ();\n GetPackageFormHQ();\n }\n }\n catch (Exception ex)\n {\n _log.Error(ex);\n }\n finally\n {\n if (_client != null)\n {\n _client.Close();\n _client = null;\n }\n }\n }\n }\n\n private void GetPackageFormHQ()\n {\n ReplicationPacket p = createpacket();\n p.command = \"getpackageforbranch\";\n p.lastrecord = _clientConfig.lastHQCounter;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n if (ret.OK)\n {\n if (ret.Data != null)\n {\n ReplicationPacket pack = (ReplicationPacket)ret.Data;\n\n if (pack.datahash == Helper.MurMur.Hash((byte[])pack.data))\n {\n _log.Debug(\"package recieved from server : \" + pack.filename);\n _log.Debug(\"package size : \" + (pack.data as byte[]).Length.ToString(\"#,0\"));\n File.WriteAllBytes(_InboxPath + pack.filename, (byte[])pack.data);\n p = createpacket();\n p.command = \"hqpackageok\";\n p.filename = pack.filename;\n p.lastrecord = pack.lastrecord;\n _clientConfig.lastHQCounter = pack.lastrecord;\n SaveConfig();\n ret = (ReturnPacket)_client.Send(p);\n if (ret.OK)\n return;\n }\n }\n }\n }\n\n private void SendPackageToHQ()\n {\n string fn = CreatePackageForSend();\n if (fn != \"\")\n {\n ReplicationPacket p = createpacket();\n p.command = \"packageforhq\";\n p.data = File.ReadAllBytes(fn);\n p.datahash = Helper.MurMur.Hash((byte[])p.data);\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n string path = Path.GetDirectoryName(fn);\n string fnn = Path.GetFileNameWithoutExtension(fn);\n foreach (var f in Directory.GetFiles(path, fnn + \".*\"))\n File.Delete(f);\n }\n }\n\n private ReplicationPacket createpacket()\n {\n ReplicationPacket p = new ReplicationPacket();\n p.branchname = _config.BranchName;\n p.passwordhash = Helper.MurMur.Hash(Helper.GetBytes(_config.BranchName + \"|\" + _config.Password)).ToString();\n return p;\n }\n\n private bool ConnectToHQ()\n {\n\n if (_client == null)\n {\n _client = new NetworkClient(_config.ServerAddress, _config.ServerReplicationPort);\n }\n // authenticate and get branch config\n ReplicationPacket p = createpacket();\n p.command = \"getbranchconfig\";\n\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n if (ret.OK)\n {\n ClientWhatWhenConfig c = (ClientWhatWhenConfig)ret.Data;\n\n _clientConfig.what = c.what;\n\n _clientConfig.isConfigured = true;\n\n if (_clientConfig.whencron != c.whencron)\n {\n _cron.Stop();\n _clientConfig.whencron = c.whencron;\n _cron = new CronDaemon();\n _cron.AddJob(_clientConfig.whencron, Replicate);\n }\n\n SaveConfig();\n }\n return ret.OK;\n }\n\n private string CreatePackageForSend()\n {\n int maxc = INTERNALLIMIT;\n if (_clientConfig.what.PackageItemLimit > 0)\n maxc = _clientConfig.what.PackageItemLimit;\n string outFolder = _OutboxPath;\n int packageNumber = _clientConfig.outPackageNumber;\n int i = _clientConfig.lastCounter;\n string filename = outFolder + packageNumber.ToString(\"0000000000\") + \".mgdat\";\n int total = _docs.RecordCount();\n if (i < total)\n {\n StorageFile package = new StorageFile(filename, SF_FORMAT.JSON, true);\n while (maxc > 0 && i < total)\n {\n var meta = _docs.GetMeta(i);\n if (meta == null)\n break;\n if (meta.isReplicated == false && MatchType(meta.typename))\n {\n object obj = _docs.GetObject(i, out meta);\n package.WriteObject(meta.key, obj);\n maxc--;\n }\n\n i++;\n }\n package.Shutdown();\n packageNumber++;\n // compress the file\n using (FileStream read = File.OpenRead(filename))\n using (FileStream outp = File.Create(filename + \".gz\"))\n CompressForBackup(read, outp);\n\n // delete uncompressed file \n File.Delete(filename);\n\n _clientConfig.lastCounter = i;\n _clientConfig.outPackageNumber = packageNumber;\n SaveConfig();\n return filename + \".gz\";\n }\n return \"\";\n }\n\n private bool MatchType(string typename)\n {\n // match type filter\n foreach (var i in _clientConfig.what.B2HQtypes)\n {\n // do wildcard search\n Regex reg = new Regex(\"^\" + i.Replace(\"*\", \".*\").Replace(\"?\", \".\"), RegexOptions.IgnoreCase);\n if (reg.IsMatch(typename))\n return true;\n }\n\n return false;\n }\n\n private static void CompressForBackup(Stream source, Stream destination)\n {\n using (GZipStream gz = new GZipStream(destination, CompressionMode.Compress))\n PumpDataForBackup(source, gz);\n }\n\n private static void PumpDataForBackup(Stream input, Stream output)\n {\n byte[] bytes = new byte[4096 * 2];\n int n;\n while ((n = input.Read(bytes, 0, bytes.Length)) != 0)\n output.Write(bytes, 0, n);\n }\n }\n}\n" }, { "path": "RaptorDB/Replication/ReplicationServer.cs", "content": "using System;\nusing RaptorDB.Common;\nusing System.IO;\nusing System.IO.Compression;\nusing System.Text.RegularExpressions;\n\nnamespace RaptorDB.Replication\n{\n internal class ReplicationServer\n {\n public ReplicationServer(string datapath, string config, IDocStorage docs)\n {\n _docs = docs;\n _Path = datapath;\n Initialize(config);\n }\n IDocStorage _docs;\n private string _S = Path.DirectorySeparatorChar.ToString();\n private string _Path;\n private ILog _log = LogManager.GetLogger(typeof(ReplicationServer));\n private ServerConfiguration _config;\n private NetworkServer _server;\n private string _InboxPath;\n private string _OutboxPath;\n private int INTERNALLIMIT = Global.PackageSizeItemCountLimit;\n private SafeDictionary _branchLastDocs = new SafeDictionary();\n\n\n private void Initialize(string config)\n {\n _log.Debug(\"Starting replication server...\");\n Directory.CreateDirectory(_Path + \"Replication\");\n Directory.CreateDirectory(_Path + \"Replication\" + _S + \"Inbox\");\n Directory.CreateDirectory(_Path + \"Replication\" + _S + \"Outbox\");\n _InboxPath = _Path + \"Replication\" + _S + \"Inbox\";\n _OutboxPath = _Path + \"Replication\" + _S + \"Outbox\";\n\n _config = fastJSON.JSON.ToObject(config);\n if (_config == null)\n {\n _log.Error(\"unable to read the configuration for replication, check the config file\");\n return;\n }\n\n // read branch lastdoc counts here\n foreach (var b in _config.Where)\n {\n int i = -1;\n if (File.Exists(_Path + \"Replication\" + _S + b.BranchName + \".last\"))\n i = Helper.ToInt32(File.ReadAllBytes(_Path + \"Replication\" + _S + b.BranchName + \".last\"), 0);\n Directory.CreateDirectory(_Path + \"Replication\" + _S + \"Inbox\" + _S + b.BranchName);\n Directory.CreateDirectory(_Path + \"Replication\" + _S + \"Outbox\" + _S + b.BranchName);\n _branchLastDocs.Add(b.BranchName.ToLower(), i);\n }\n\n _server = new NetworkServer();\n _server.Start(_config.ReplicationPort, processpayload);\n }\n\n public void Shutdown()\n {\n WriteBranchCounters();\n // shutdown every thing\n _server.Stop();\n }\n\n private void WriteBranchCounters()\n {\n // write branch counts etc. to disk\n foreach (var b in _branchLastDocs)\n {\n File.WriteAllBytes(_Path + \"Replication\" + _S + b.Key + \".last\", Helper.GetBytes(b.Value, false));\n _log.Debug(\"last counter for branch : \" + b.Key + \" = \" + b.Value);\n }\n }\n\n private object processpayload(object data)\n {\n ReplicationPacket p = (ReplicationPacket)data;\n\n if (Authenticate(p) == false)\n return new ReturnPacket(false, \"Authentication failed\");\n\n ReturnPacket ret = new ReturnPacket(true);\n try\n {\n switch (p.command)\n {\n case \"getbranchconfig\":\n ret.OK = true;\n ret.Data = GetBranchConfig(p.branchname);\n break;\n case \"getpackageforbranch\":\n ret.OK = true;\n ReplicationPacket pack = GetPackageForBranch(p);\n ret.Data = pack;\n break;\n case \"packageforhq\":\n ret.OK = PackageForHQ(p);\n break;\n case \"hqpackageok\":\n ret.OK = true;\n File.Delete(_OutboxPath + _S + p.branchname + _S + p.filename);\n // set last rec on hq\n _branchLastDocs.Add(p.branchname.ToLower(), p.lastrecord);\n WriteBranchCounters();\n break;\n }\n }\n catch (Exception ex)\n {\n ret.OK = false;\n _log.Error(ex);\n }\n return ret;\n }\n\n\n private ClientWhatWhenConfig GetBranchConfig(string branchname)\n {\n WhatItem ret = _config.What.Find((WhatItem w) => { return w.Name.ToLower() == branchname.ToLower(); });\n\n if (ret == null)\n ret = _config.What.Find((WhatItem w) => { return w.Name.ToLower() == \"default\"; });\n\n ClientWhatWhenConfig c = new ClientWhatWhenConfig();\n c.what = ret;\n var where = _config.Where.Find(w => { return w.BranchName.ToLower() == branchname.ToLower(); });\n if (where != null)\n c.whencron = where.When;\n else\n c.whencron = \"* * * * *\";\n\n return c;\n }\n\n private bool PackageForHQ(ReplicationPacket p)\n {\n uint hash = Helper.MurMur.Hash((byte[])p.data);\n if (hash != p.datahash)\n return false;\n // save file to \\replication\\inbox\\branchname\n Directory.CreateDirectory(_InboxPath + _S + p.branchname);\n string fn = _InboxPath + _S + p.branchname + _S + p.filename;\n _log.Debug(\"package recieved from : \" + p.branchname);\n _log.Debug(\"package name : \" + p.filename);\n _log.Debug(\"package size : \" + (p.data as byte[]).Length.ToString(\"#,0\"));\n File.WriteAllBytes(fn, (byte[])p.data);\n return true;\n }\n\n private ReplicationPacket GetPackageForBranch(ReplicationPacket packet)\n {\n int last = _branchLastDocs[packet.branchname.ToLower()];\n // skip retry for the same package\n if (packet.lastrecord >= _branchLastDocs[packet.branchname.ToLower()])\n {\n string fn = CreatePackageForSend(packet, out last);\n ReplicationPacket p = new ReplicationPacket();\n p.filename = Path.GetFileName(fn);\n p.data = File.ReadAllBytes(fn);\n p.datahash = Helper.MurMur.Hash((byte[])p.data);\n p.lastrecord = last;\n return p;\n }\n else\n return null;\n }\n\n private bool Authenticate(ReplicationPacket p)\n {\n uint pwd = uint.Parse(p.passwordhash);\n bool auth = false;\n foreach (var w in _config.Where)\n {\n uint hash = Helper.MurMur.Hash(Helper.GetBytes(w.BranchName + \"|\" + w.Password));\n if (hash == pwd) auth = true;\n }\n if (auth == false)\n _log.Debug(\"Authentication failed for '\" + p.branchname + \"' hash = \" + p.passwordhash);\n return auth;\n }\n\n private string CreatePackageForSend(ReplicationPacket packet, out int last)\n {\n int maxc = INTERNALLIMIT;\n WhatItem what = GetBranchConfig(packet.branchname).what;\n if (what.PackageItemLimit > 0)\n maxc = what.PackageItemLimit;\n string outFolder = _OutboxPath;\n int packageNumber = packet.lastrecord;\n int i = packet.lastrecord;\n string filename = outFolder + _S + packet.branchname + _S + packageNumber.ToString(\"0000000000\") + \".mgdat\";\n\n if (i < _docs.RecordCount())\n {\n StorageFile package = new StorageFile(filename, SF_FORMAT.JSON, true);\n while (maxc > 0)\n {\n var meta = _docs.GetMeta(i);\n if (meta == null)\n break;\n if (meta.isReplicated == false && MatchType(meta.typename, what))\n {\n if (meta.isDeleted == false || what.PropogateHQDeletes)\n {\n object obj = _docs.GetObject(i, out meta);\n package.WriteObject(meta.key, obj); \n maxc--;\n }\n }\n\n i++;\n }\n package.Shutdown();\n packageNumber++;\n // compress the file\n using (FileStream read = File.OpenRead(filename))\n using (FileStream outp = File.Create(filename + \".gz\"))\n CompressForBackup(read, outp);\n\n // delete uncompressed file \n File.Delete(filename);\n }\n\n last = i;\n return filename + \".gz\";\n }\n\n private bool MatchType(string typename, WhatItem what)\n {\n // match type filter\n foreach (var i in what.HQ2Btypes)\n {\n // do wildcard search\n Regex reg = new Regex(\"^\" + i.Replace(\"*\", \".*\").Replace(\"?\", \".\"), RegexOptions.IgnoreCase);\n if (reg.IsMatch(typename))\n return true;\n }\n\n return false;\n }\n\n private static void CompressForBackup(Stream source, Stream destination)\n {\n using (GZipStream gz = new GZipStream(destination, CompressionMode.Compress))\n PumpDataForBackup(source, gz);\n }\n\n private static void PumpDataForBackup(Stream input, Stream output)\n {\n byte[] bytes = new byte[4096 * 2];\n int n;\n while ((n = input.Read(bytes, 0, bytes.Length)) != 0)\n output.Write(bytes, 0, n);\n }\n }\n}\n" }, { "path": "RaptorDB/Replication/msg.txt", "content": "The following error occurred and the json document is below, you can skip this \ndocument if you wish by incrementing the %c% file :\n\n%ex%\n------------------------------------------------------------------------------\nThe json document is :\n\n%js%\n\n" }, { "path": "RaptorDB/Storage/KeyStore.cs", "content": "using System;\nusing System.Collections.Generic;\nusing System.Text;\nusing System.IO;\nusing RaptorDB.Common;\n\nnamespace RaptorDB\n{\n #region [ KeyStoreString ]\n internal class KeyStoreString : IDisposable\n {\n public KeyStoreString(string filename, bool caseSensitve)\n {\n _db = KeyStore.Open(filename, true);\n _caseSensitive = caseSensitve;\n }\n bool _caseSensitive = false;\n\n KeyStore _db;\n\n\n public void Set(string key, string val)\n {\n Set(key, fastJSON.Reflection.UnicodeGetBytes(val));\n }\n\n public void Set(string key, byte[] val)\n {\n string str = (_caseSensitive ? key : key.ToLower());\n byte[] bkey = fastJSON.Reflection.UnicodeGetBytes(str);\n int hc = (int)Helper.MurMur.Hash(bkey);\n MemoryStream ms = new MemoryStream();\n ms.Write(Helper.GetBytes(bkey.Length, false), 0, 4);\n ms.Write(bkey, 0, bkey.Length);\n ms.Write(val, 0, val.Length);\n\n _db.SetBytes(hc, ms.ToArray());\n }\n\n public bool Get(string key, out string val)\n {\n val = null;\n byte[] bval;\n bool b = Get(key, out bval);\n if (b)\n {\n val = fastJSON.Reflection.UnicodeGetString(bval);\n }\n return b;\n }\n\n public bool Get(string key, out byte[] val)\n {\n string str = (_caseSensitive ? key : key.ToLower());\n val = null;\n byte[] bkey = fastJSON.Reflection.UnicodeGetBytes(str);\n int hc = (int)Helper.MurMur.Hash(bkey);\n\n if (_db.GetBytes(hc, out val))\n {\n // unpack data\n byte[] g = null;\n if (UnpackData(val, out val, out g))\n {\n if (Helper.CompareMemCmp(bkey, g) != 0)\n {\n // if data not equal check duplicates (hash conflict)\n List ints = new List(_db.GetDuplicates(hc));\n ints.Reverse();\n foreach (int i in ints)\n {\n byte[] bb = _db.FetchRecordBytes(i);\n if (UnpackData(bb, out val, out g))\n {\n if (Helper.CompareMemCmp(bkey, g) == 0)\n return true;\n }\n }\n return false;\n }\n return true;\n }\n }\n return false;\n }\n\n public int Count()\n {\n return (int)_db.Count();\n }\n\n public int RecordCount()\n {\n return (int)_db.RecordCount();\n }\n\n public void SaveIndex()\n {\n _db.SaveIndex();\n }\n\n public void Shutdown()\n {\n _db.Shutdown();\n }\n\n public void Dispose()\n {\n _db.Shutdown();\n }\n\n private bool UnpackData(byte[] buffer, out byte[] val, out byte[] key)\n {\n int len = Helper.ToInt32(buffer, 0, false);\n key = new byte[len];\n Buffer.BlockCopy(buffer, 4, key, 0, len);\n val = new byte[buffer.Length - 4 - len];\n Buffer.BlockCopy(buffer, 4 + len, val, 0, buffer.Length - 4 - len);\n\n return true;\n }\n\n public string ReadData(int recnumber)\n {\n byte[] val;\n byte[] key;\n byte[] b = _db.FetchRecordBytes(recnumber);\n if (UnpackData(b, out val, out key))\n {\n return fastJSON.Reflection.UnicodeGetString(val);\n }\n return \"\";\n }\n\n internal void FreeMemory()\n {\n _db.FreeMemory();\n }\n }\n #endregion\n\n #region [ KeyStoreGuid removed ]\n //internal class KeyStoreGuid : IDisposable //, IDocStorage\n //{\n // public KeyStoreGuid(string filename)\n // {\n // _db = KeyStore.Open(filename, true);\n // }\n\n // KeyStore _db;\n\n // public void Set(Guid key, string val)\n // {\n // Set(key, Encoding.Unicode.GetBytes(val));\n // }\n\n // public int Set(Guid key, byte[] val)\n // {\n // byte[] bkey = key.ToByteArray();\n // int hc = (int)Helper.MurMur.Hash(bkey);\n // MemoryStream ms = new MemoryStream();\n // ms.Write(Helper.GetBytes(bkey.Length, false), 0, 4);\n // ms.Write(bkey, 0, bkey.Length);\n // ms.Write(val, 0, val.Length);\n\n // return _db.SetBytes(hc, ms.ToArray());\n // }\n\n // public bool Get(Guid key, out string val)\n // {\n // val = null;\n // byte[] bval;\n // bool b = Get(key, out bval);\n // if (b)\n // {\n // val = Encoding.Unicode.GetString(bval);\n // }\n // return b;\n // }\n\n // public bool Get(Guid key, out byte[] val)\n // {\n // val = null;\n // byte[] bkey = key.ToByteArray();\n // int hc = (int)Helper.MurMur.Hash(bkey);\n\n // if (_db.Get(hc, out val))\n // {\n // // unpack data\n // byte[] g = null;\n // if (UnpackData(val, out val, out g))\n // {\n // if (Helper.CompareMemCmp(bkey, g) != 0)\n // {\n // // if data not equal check duplicates (hash conflict)\n // List ints = new List(_db.GetDuplicates(hc));\n // ints.Reverse();\n // foreach (int i in ints)\n // {\n // byte[] bb = _db.FetchRecordBytes(i);\n // if (UnpackData(bb, out val, out g))\n // {\n // if (Helper.CompareMemCmp(bkey, g) == 0)\n // return true;\n // }\n // }\n // return false;\n // }\n // return true;\n // }\n // }\n // return false;\n // }\n\n // public void SaveIndex()\n // {\n // _db.SaveIndex();\n // }\n\n // public void Shutdown()\n // {\n // _db.Shutdown();\n // }\n\n // public void Dispose()\n // {\n // _db.Shutdown();\n // }\n\n // public byte[] FetchRecordBytes(int record)\n // {\n // return _db.FetchRecordBytes(record);\n // }\n\n // public int Count()\n // {\n // return (int)_db.Count();\n // }\n\n // public int RecordCount()\n // {\n // return (int)_db.RecordCount();\n // }\n\n // private bool UnpackData(byte[] buffer, out byte[] val, out byte[] key)\n // {\n // int len = Helper.ToInt32(buffer, 0, false);\n // key = new byte[len];\n // Buffer.BlockCopy(buffer, 4, key, 0, len);\n // val = new byte[buffer.Length - 4 - len];\n // Buffer.BlockCopy(buffer, 4 + len, val, 0, buffer.Length - 4 - len);\n\n // return true;\n // }\n\n // internal byte[] Get(int recnumber, out Guid docid)\n // {\n // bool isdeleted = false;\n // return Get(recnumber, out docid, out isdeleted);\n // }\n\n // public bool RemoveKey(Guid key)\n // {\n // byte[] bkey = key.ToByteArray();\n // int hc = (int)Helper.MurMur.Hash(bkey);\n // MemoryStream ms = new MemoryStream();\n // ms.Write(Helper.GetBytes(bkey.Length, false), 0, 4);\n // ms.Write(bkey, 0, bkey.Length);\n // return _db.Delete(hc, ms.ToArray());\n // }\n\n // public byte[] Get(int recnumber, out Guid docid, out bool isdeleted)\n // {\n // docid = Guid.Empty;\n // byte[] buffer = _db.FetchRecordBytes(recnumber, out isdeleted);\n // if (buffer == null) return null;\n // if (buffer.Length == 0) return null;\n // byte[] key;\n // byte[] val;\n // // unpack data\n // UnpackData(buffer, out val, out key);\n // docid = new Guid(key);\n // return val;\n // }\n\n // internal int CopyTo(StorageFile backup, int start)\n // {\n // return _db.CopyTo(backup, start);\n // }\n //}\n #endregion\n\n internal class KeyStore : IDisposable, IDocStorage where T : IComparable\n {\n public KeyStore(string Filename, byte MaxKeySize, bool AllowDuplicateKeys)\n {\n Initialize(Filename, MaxKeySize, AllowDuplicateKeys);\n }\n\n public KeyStore(string Filename, bool AllowDuplicateKeys)\n {\n Initialize(Filename, Global.DefaultStringKeySize, AllowDuplicateKeys);\n }\n\n private ILog log = LogManager.GetLogger(typeof(KeyStore));\n\n private string _Path = \"\";\n private string _FileName = \"\";\n private byte _MaxKeySize;\n private StorageFile _archive;\n private MGIndex _index;\n private string _datExtension = \".mgdat\";\n private string _idxExtension = \".mgidx\";\n private IGetBytes _T = null;\n private System.Timers.Timer _savetimer;\n private BoolIndex _deleted;\n\n\n public static KeyStore Open(string Filename, bool AllowDuplicateKeys)\n {\n return new KeyStore(Filename, AllowDuplicateKeys);\n }\n\n public static KeyStore Open(string Filename, byte MaxKeySize, bool AllowDuplicateKeys)\n {\n return new KeyStore(Filename, MaxKeySize, AllowDuplicateKeys);\n }\n\n object _savelock = new object();\n public void SaveIndex()\n {\n if (_index == null)\n return;\n lock (_savelock)\n {\n log.Debug(\"saving to disk\");\n _index.SaveIndex();\n _deleted.SaveIndex();\n log.Debug(\"index saved\");\n }\n }\n\n public IEnumerable GetDuplicates(T key)\n {\n // get duplicates from index\n return _index.GetDuplicates(key);\n }\n\n public byte[] FetchRecordBytes(int record)\n {\n return _archive.ReadBytes(record);\n }\n\n public long Count()\n {\n int c = _archive.Count();\n return c - _deleted.GetBits().CountOnes() * 2;\n }\n\n public bool Get(T key, out string val)\n {\n byte[] b = null;\n val = \"\";\n bool ret = GetBytes(key, out b);\n if (ret)\n {\n if (b != null)\n val = fastJSON.Reflection.UnicodeGetString(b);\n else\n val = \"\";\n }\n return ret;\n }\n\n public bool GetObject(T key, out object val)\n {\n int off;\n val = null;\n if (_index.Get(key, out off))\n {\n val = _archive.ReadObject(off);\n return true;\n }\n return false;\n }\n\n public bool GetBytes(T key, out byte[] val)\n {\n int off;\n val = null;\n // search index\n if (_index.Get(key, out off))\n {\n val = _archive.ReadBytes(off);\n return true;\n }\n return false;\n }\n\n public int SetString(T key, string data)\n {\n return SetBytes(key, fastJSON.Reflection.UnicodeGetBytes(data));\n }\n\n public int SetObject(T key, object doc)\n {\n int recno = -1;\n // save to storage\n recno = (int) _archive.WriteObject(key, doc);\n // save to index\n _index.Set(key, recno);\n\n return recno;\n }\n\n public int SetBytes(T key, byte[] data)\n {\n int recno = -1;\n // save to storage\n recno = (int)_archive.WriteData(key, data);\n // save to index\n _index.Set(key, recno);\n\n return recno;\n }\n\n private object _shutdownlock = new object();\n public void Shutdown()\n {\n lock (_shutdownlock)\n {\n if (_index != null)\n log.Debug(\"Shutting down\");\n else\n return;\n _savetimer.Enabled = false;\n SaveIndex();\n SaveLastRecord();\n\n if (_deleted != null)\n _deleted.Shutdown();\n if (_index != null)\n _index.Shutdown();\n if (_archive != null)\n _archive.Shutdown();\n _index = null;\n _archive = null;\n _deleted = null;\n //log.Debug(\"Shutting down log\");\n //LogManager.Shutdown();\n }\n }\n\n public void Dispose()\n {\n Shutdown();\n }\n\n #region [ P R I V A T E M E T H O D S ]\n private void SaveLastRecord()\n {\n // save the last record number in the index file\n _index.SaveLastRecordNumber(_archive.Count());\n }\n\n private void Initialize(string filename, byte maxkeysize, bool AllowDuplicateKeys)\n {\n _MaxKeySize = RDBDataType.GetByteSize(maxkeysize);\n _T = RDBDataType.ByteHandler();\n\n _Path = Path.GetDirectoryName(filename);\n Directory.CreateDirectory(_Path);\n\n _FileName = Path.GetFileNameWithoutExtension(filename);\n string db = _Path + Path.DirectorySeparatorChar + _FileName + _datExtension;\n string idx = _Path + Path.DirectorySeparatorChar + _FileName + _idxExtension;\n\n //LogManager.Configure(_Path + Path.DirectorySeparatorChar + _FileName + \".txt\", 500, false);\n\n _index = new MGIndex(_Path, _FileName + _idxExtension, _MaxKeySize, /*Global.PageItemCount,*/ AllowDuplicateKeys);\n\n if (Global.SaveAsBinaryJSON)\n _archive = new StorageFile(db, SF_FORMAT.BSON, false);\n else\n _archive = new StorageFile(db, SF_FORMAT.JSON, false);\n\n _deleted = new BoolIndex(_Path, _FileName , \"_deleted.idx\");\n\n log.Debug(\"Current Count = \" + RecordCount().ToString(\"#,0\"));\n\n CheckIndexState();\n\n log.Debug(\"Starting save timer\");\n _savetimer = new System.Timers.Timer();\n _savetimer.Elapsed += new System.Timers.ElapsedEventHandler(_savetimer_Elapsed);\n _savetimer.Interval = Global.SaveIndexToDiskTimerSeconds * 1000;\n _savetimer.AutoReset = true;\n _savetimer.Start();\n\n }\n\n private void CheckIndexState()\n {\n log.Debug(\"Checking Index state...\");\n int last = _index.GetLastIndexedRecordNumber();\n int count = _archive.Count();\n if (last < count)\n {\n log.Debug(\"Rebuilding index...\");\n log.Debug(\" last index count = \" + last);\n log.Debug(\" data items count = \" + count);\n // check last index record and archive record\n // rebuild index if needed\n for (int i = last; i < count; i++)\n {\n bool deleted = false;\n T key = _archive.GetKey(i, out deleted);\n if (deleted == false)\n _index.Set(key, i);\n else\n _index.RemoveKey(key);\n\n if (i % 100000 == 0)\n log.Debug(\"100,000 items re-indexed\");\n }\n log.Debug(\"Rebuild index done.\");\n }\n }\n\n void _savetimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)\n {\n SaveIndex();\n }\n\n #endregion\n\n public int RecordCount()\n {\n return _archive.Count();\n }\n\n public int[] GetHistory(T key)\n {\n List a = new List();\n foreach (int i in GetDuplicates(key))\n {\n a.Add(i);\n }\n return a.ToArray();\n }\n\n internal byte[] FetchRecordBytes(int record, out bool isdeleted)\n {\n StorageItem meta;\n byte[] b = _archive.ReadBytes(record, out meta);\n isdeleted = meta.isDeleted;\n return b;\n }\n\n internal bool Delete(T id)\n {\n // write a delete record\n int rec = (int)_archive.Delete(id);\n _deleted.Set(true, rec);\n return _index.RemoveKey(id);\n }\n\n internal bool DeleteReplicated(T id)\n {\n // write a delete record for replicated object\n int rec = (int)_archive.DeleteReplicated(id);\n _deleted.Set(true, rec);\n return _index.RemoveKey(id);\n }\n\n internal int CopyTo(StorageFile storagefile, long startrecord)\n {\n return _archive.CopyTo(storagefile, startrecord);\n }\n\n public byte[] GetBytes(int rowid, out StorageItem meta)\n {\n return _archive.ReadBytes(rowid, out meta);\n }\n\n internal void FreeMemory()\n {\n _index.FreeMemory();\n }\n\n public object GetObject(int rowid, out StorageItem meta)\n {\n return _archive.ReadObject(rowid, out meta);\n }\n\n public StorageItem GetMeta(int rowid)\n {\n return _archive.ReadMeta(rowid);\n }\n\n internal int SetReplicationObject(T key, object doc)\n {\n int recno = -1;\n // save to storage\n recno = (int) _archive.WriteReplicationObject(key, doc);\n // save to index\n _index.Set(key, recno);\n\n return recno;\n }\n }\n}\n" }, { "path": "RaptorDB/Storage/KeyStoreHF.cs", "content": "using RaptorDB.Common;\nusing System;\nusing System.Collections.Generic;\nusing System.IO;\nusing System.Linq;\n\nnamespace RaptorDB\n{\n // high frequency key value store\n public class KeyStoreHF : IKeyStoreHF\n {\n internal class AllocationBlock\n {\n public string key;\n public byte keylen;\n public int datalength;\n public bool isCompressed;\n public bool isBinaryJSON;\n public bool deleteKey;\n public List Blocks = new List();\n public int blocknumber;\n }\n\n MGIndex _keys;\n StorageFileHF _datastore;\n object _lock = new object();\n ushort _BlockSize = 2048;\n private const int _KILOBYTE = 1024;\n ILog _log = LogManager.GetLogger(typeof(KeyStoreHF));\n\n byte[] _blockheader = new byte[]{\n 0,0,0,0, // 0 block # (used for validate block reads and rebuild)\n 0,0,0,0, // 4 next block # \n 0, // 8 flags bits 0:iscompressed 1:isbinary 2:deletekey\n 0,0,0,0, // 9 data length (compute alloc blocks needed)\n 0, // 13 key length \n 0, // 14 key type 0=guid 1=string\n };\n private string _Path = \"\";\n private string _S = Path.DirectorySeparatorChar.ToString();\n private bool _isDirty = false;\n private string _dirtyFilename = \"temp.$\";\n\n public KeyStoreHF(string folder)\n {\n _Path = folder;\n Directory.CreateDirectory(_Path);\n if (_Path.EndsWith(_S) == false) _Path += _S;\n\n if (File.Exists(_Path + _dirtyFilename))\n {\n _log.Error(\"Last shutdown failed, rebuilding data files...\");\n RebuildDataFiles();\n }\n _datastore = new StorageFileHF(_Path + \"data.mghf\", Global.HighFrequencyKVDiskBlockSize);\n _keys = new MGIndex(_Path, \"keys.idx\", 255, /*Global.PageItemCount,*/ false);\n //_datastore.Initialize();\n _BlockSize = _datastore.GetBlockSize();\n }\n\n //// mgindex special storage for strings ctor -> no idx file\n //// use SaveData() GetData()\n //public KeyStoreHF(string folder, string filename)\n //{\n // _Path = folder;\n // Directory.CreateDirectory(_Path);\n // if (_Path.EndsWith(_S) == false) _Path += _S;\n\n // _datastore = new StorageFileHF(_Path + filename, Global.HighFrequencyKVDiskBlockSize);\n // //_datastore.Initialize();\n // _BlockSize = _datastore.GetBlockSize();\n //}\n\n public int CountHF()\n {\n if (_keys != null)\n return _keys.Count();\n else\n return 0;\n }\n\n public object GetObjectHF(string key)\n {\n lock (_lock)\n {\n int alloc;\n if (_keys.Get(key, out alloc))\n {\n AllocationBlock ab = FillAllocationBlock(alloc);\n if (ab.deleteKey == false)\n {\n byte[] data = readblockdata(ab);\n\n return fastBinaryJSON.BJSON.ToObject(data);\n }\n }\n }\n\n return null;\n }\n\n public bool SetObjectHF(string key, object obj)\n {\n byte[] k = Helper.GetBytes(key);\n if (k.Length > 255)\n {\n _log.Error(\"Key length > 255 : \" + key);\n throw new Exception(\"Key must be less than 255 characters\");\n //return false;\n }\n lock (_lock)\n {\n if (_isDirty == false)\n WriteDirtyFile();\n\n AllocationBlock ab = null;\n int firstblock = 0;\n if (_keys.Get(key, out firstblock))// key exists already\n ab = FillAllocationBlock(firstblock);\n\n SaveNew(key, k, obj);\n if (ab != null)\n {\n // free old blocks\n ab.Blocks.Add(ab.blocknumber);\n _datastore.FreeBlocks(ab.Blocks);\n }\n return true;\n }\n }\n\n public bool DeleteKeyHF(string key)\n {\n lock (_lock)\n {\n int alloc;\n if (_keys.Get(key, out alloc))\n {\n if (_isDirty == false)\n WriteDirtyFile();\n\n byte[] keybytes = Helper.GetBytes(key);\n AllocationBlock ab = FillAllocationBlock(alloc);\n\n ab.keylen = (byte)keybytes.Length;\n\n _keys.RemoveKey(key);// remove key from index\n\n // write ab\n ab.deleteKey = true;\n ab.datalength = 0;\n\n byte[] header = CreateAllocHeader(ab, keybytes);\n\n _datastore.SeekBlock(ab.blocknumber);\n _datastore.WriteBlockBytes(header, 0, header.Length);\n\n // free old data blocks\n _datastore.FreeBlocks(ab.Blocks);\n\n return true;\n }\n }\n return false;\n }\n\n public void CompactStorageHF()\n {\n lock (_lock)\n {\n try\n {\n _log.Debug(\"Compacting storage file ...\");\n if (Directory.Exists(_Path + \"temp\"))\n Directory.Delete(_Path + \"temp\", true);\n\n KeyStoreHF newfile = new KeyStoreHF(_Path + \"temp\");\n string[] keys = _keys.GetKeys().Cast().ToArray();\n _log.Debug(\"Number of keys : \" + keys.Length);\n foreach (var k in keys)\n {\n newfile.SetObjectHF(k, GetObjectHF(k));\n }\n newfile.Shutdown();\n _log.Debug(\"Compact done.\");\n // shutdown and move files and restart here\n if (Directory.Exists(_Path + \"old\"))\n Directory.Delete(_Path + \"old\", true);\n Directory.CreateDirectory(_Path + \"old\");\n _datastore.Shutdown();\n _keys.Shutdown();\n _log.Debug(\"Moving files...\");\n foreach (var f in Directory.GetFiles(_Path, \"*.*\"))\n File.Move(f, _Path + \"old\" + _S + Path.GetFileName(f));\n\n foreach (var f in Directory.GetFiles(_Path + \"temp\", \"*.*\"))\n File.Move(f, _Path + Path.GetFileName(f));\n\n Directory.Delete(_Path + \"temp\", true);\n //Directory.Delete(_Path + \"old\", true); // FEATURE : delete or keep?\n _log.Debug(\"Re-opening storage file\");\n _datastore = new StorageFileHF(_Path + \"data.mghf\", Global.HighFrequencyKVDiskBlockSize);\n _keys = new MGIndex(_Path, \"keys.idx\", 255, false);\n\n _BlockSize = _datastore.GetBlockSize();\n }\n catch (Exception ex)\n {\n _log.Error(ex);\n }\n }\n }\n\n public string[] GetKeysHF()\n {\n lock (_lock)\n return _keys.GetKeys().Cast().ToArray(); // FEATURE : ugly and dirty !?\n }\n\n public bool ContainsHF(string key)\n {\n lock (_lock)\n {\n int i = 0;\n return _keys.Get(key, out i);\n }\n }\n\n public void Shutdown()\n {\n _datastore.Shutdown();\n if (_keys != null)\n _keys.Shutdown();\n\n if (File.Exists(_Path + _dirtyFilename))\n File.Delete(_Path + _dirtyFilename);\n }\n\n internal void FreeMemory()\n {\n if (_keys != null)\n _keys.FreeMemory();\n }\n\n #region [ private methods ]\n private byte[] readblockdata(AllocationBlock ab)\n {\n byte[] data = new byte[ab.datalength];\n long offset = 0;\n int len = ab.datalength;\n int dbsize = _BlockSize - _blockheader.Length - ab.keylen;\n ab.Blocks.ForEach(x =>\n {\n byte[] b = _datastore.ReadBlock(x);\n int c = len;\n if (c > dbsize) c = dbsize;\n Buffer.BlockCopy(b, _blockheader.Length + ab.keylen, data, (int)offset, c);\n offset += c;\n len -= c;\n });\n if (ab.isCompressed)\n data = MiniLZO.Decompress(data);\n return data;\n }\n\n private object _dfile = new object();\n private void WriteDirtyFile()\n {\n lock (_dfile)\n {\n _isDirty = true;\n if (File.Exists(_Path + _dirtyFilename) == false)\n File.WriteAllText(_Path + _dirtyFilename, \"dirty\");\n }\n }\n\n private void SaveNew(string key, byte[] keybytes, object obj)\n {\n byte[] data;\n AllocationBlock ab = new AllocationBlock();\n ab.key = key;\n ab.keylen = (byte)keybytes.Length;\n\n data = fastBinaryJSON.BJSON.ToBJSON(obj);\n ab.isBinaryJSON = true;\n\n if (data.Length > (int)Global.CompressDocumentOverKiloBytes * _KILOBYTE)\n {\n ab.isCompressed = true;\n data = MiniLZO.Compress(data);\n }\n ab.datalength = data.Length;\n\n int firstblock = internalSave(keybytes, data, ab);\n\n // save keys\n _keys.Set(key, firstblock);\n }\n\n private int internalSave(byte[] keybytes, byte[] data, AllocationBlock ab)\n {\n int firstblock = _datastore.GetFreeBlockNumber();\n int blocknum = firstblock;\n byte[] header = CreateAllocHeader(ab, keybytes);\n int dblocksize = _BlockSize - header.Length;\n int offset = 0;\n // compute data block count\n int datablockcount = (data.Length / dblocksize) + 1;\n // save data blocks\n int counter = 0;\n int len = data.Length;\n while (datablockcount > 0)\n {\n datablockcount--;\n int next = 0;\n if (datablockcount > 0)\n next = _datastore.GetFreeBlockNumber();\n\n Buffer.BlockCopy(Helper.GetBytes(counter, false), 0, header, 0, 4); // set block number\n Buffer.BlockCopy(Helper.GetBytes(next, false), 0, header, 4, 4); // set next pointer\n\n _datastore.SeekBlock(blocknum);\n _datastore.WriteBlockBytes(header, 0, header.Length);\n int c = len;\n if (c > dblocksize)\n c = dblocksize;\n _datastore.WriteBlockBytes(data, offset, c);\n\n if (next > 0)\n blocknum = next;\n offset += c;\n len -= c;\n counter++;\n }\n return firstblock;\n }\n\n private byte[] CreateAllocHeader(AllocationBlock ab, byte[] keybytes)\n {\n byte[] alloc = new byte[_blockheader.Length + keybytes.Length];\n\n if (ab.isCompressed)\n alloc[8] = 1;\n if (ab.isBinaryJSON)\n alloc[8] += 2;\n if (ab.deleteKey)\n alloc[8] += 4;\n\n Buffer.BlockCopy(Helper.GetBytes(ab.datalength, false), 0, alloc, 9, 4);\n alloc[13] = ab.keylen;\n alloc[14] = 1; // string keys for now\n Buffer.BlockCopy(keybytes, 0, alloc, _blockheader.Length, ab.keylen);\n\n return alloc;\n }\n\n private AllocationBlock FillAllocationBlock(int blocknumber)\n {\n AllocationBlock ab = new AllocationBlock();\n\n ab.blocknumber = blocknumber;\n ab.Blocks.Add(blocknumber);\n\n byte[] b = _datastore.ReadBlockBytes(blocknumber, _blockheader.Length + 255);\n\n int blocknumexpected = 0;\n\n int next = ParseBlockHeader(ab, b, blocknumexpected);\n\n blocknumexpected++;\n\n while (next > 0)\n {\n ab.Blocks.Add(next);\n b = _datastore.ReadBlockBytes(next, _blockheader.Length + ab.keylen);\n next = ParseBlockHeader(ab, b, blocknumexpected);\n blocknumexpected++;\n }\n\n return ab;\n }\n\n private int ParseBlockHeader(AllocationBlock ab, byte[] b, int blocknumberexpected)\n {\n int bnum = Helper.ToInt32(b, 0);\n if (bnum != blocknumberexpected)\n {\n _log.Error(\"Block numbers does not match, looking for : \" + blocknumberexpected);\n //throw new Exception(\"Block numbers does not match, looking for : \" + blocknumberexpected);\n return -1;\n }\n if (b[14] != 1)\n {\n _log.Error(\"Expecting string keys only, got : \" + b[14]);\n //throw new Exception(\"Expecting string keys only, got : \" + b[11]);\n return -1;\n }\n\n int next = Helper.ToInt32(b, 4);\n\n if (ab.keylen == 0)\n {\n byte flags = b[8];\n\n if ((flags & 0x01) > 0)\n ab.isCompressed = true;\n if ((flags & 0x02) > 0)\n ab.isBinaryJSON = true;\n if ((flags & 0x04) > 0)\n ab.deleteKey = true;\n\n ab.datalength = Helper.ToInt32(b, 9);\n byte keylen = b[13];\n ab.keylen = keylen;\n ab.key = Helper.GetString(b, _blockheader.Length, keylen);\n }\n return next;\n }\n\n private void RebuildDataFiles()\n {\n MGIndex keys = null;\n try\n {\n // remove old free list\n if (File.Exists(_Path + \"data.bmp\"))\n File.Delete(_Path + \"data.bmp\");\n\n _datastore = new StorageFileHF(_Path + \"data.mghf\", Global.HighFrequencyKVDiskBlockSize);\n _BlockSize = _datastore.GetBlockSize();\n if (File.Exists(_Path + \"keys.idx\"))\n {\n _log.Debug(\"removing old keys index\");\n foreach (var f in Directory.GetFiles(_Path, \"keys.*\"))\n File.Delete(f);\n }\n\n keys = new MGIndex(_Path, \"keys.idx\", 255, /*Global.PageItemCount,*/ false);\n\n MGRB visited = new MGRB();\n\n int c = _datastore.NumberofBlocks();\n\n for (int i = 1; i < c; i++) // go through blocks skip first\n {\n if (visited.Get(i))\n continue;\n byte[] b = _datastore.ReadBlockBytes(i, _blockheader.Length + 255);\n int bnum = Helper.ToInt32(b, 0);\n if (bnum > 0) // check if a start block\n {\n visited.Set(i, true);\n _datastore.FreeBlock(i); // mark as free\n continue;\n }\n\n AllocationBlock ab = new AllocationBlock();\n // start block found\n int blocknumexpected = 0;\n\n int next = ParseBlockHeader(ab, b, blocknumexpected);\n int last = 0;\n bool freelast = false;\n AllocationBlock old = null;\n\n if (ab.key == null)\n continue;\n\n if (keys.Get(ab.key, out last))\n {\n old = this.FillAllocationBlock(last);\n freelast = true;\n }\n blocknumexpected++;\n bool failed = false;\n if (ab.deleteKey == false)\n {\n while (next > 0) // read the blocks\n {\n ab.Blocks.Add(next);\n b = _datastore.ReadBlockBytes(next, _blockheader.Length + ab.keylen);\n next = ParseBlockHeader(ab, b, blocknumexpected);\n if (next == -1) // non matching block\n {\n failed = true;\n break;\n }\n blocknumexpected++;\n }\n }\n else\n {\n failed = true;\n keys.RemoveKey(ab.key);\n }\n // new data ok\n if (failed == false)\n {\n keys.Set(ab.key, i);// valid block found\n if (freelast && old != null)// free the old blocks\n _datastore.FreeBlocks(old.Blocks);\n }\n\n visited.Set(i, true);\n }\n\n // all ok delete temp.$ file\n if (File.Exists(_Path + _dirtyFilename))\n File.Delete(_Path + _dirtyFilename);\n }\n catch (Exception ex)\n {\n _log.Error(ex);\n }\n finally\n {\n _log.Debug(\"Shutting down files and index\");\n _datastore.Shutdown();\n keys.SaveIndex();\n keys.Shutdown();\n }\n }\n #endregion\n\n //internal void FreeBlocks(List list)\n //{\n // lock (_lock)\n // _datastore.FreeBlocks(list);\n //}\n\n\n //// for .string files\n //internal int SaveData(string key, byte[] data)\n //{\n // lock (_lock)\n // {\n // byte[] kb = Helper.GetBytes(key);\n // AllocationBlock ab = new AllocationBlock();\n // ab.key = key;\n // ab.keylen = (byte)kb.Length;\n // ab.isCompressed = false;\n // ab.isBinaryJSON = true;\n // ab.datalength = data.Length;\n\n // return internalSave(kb, data, ab);\n // }\n //}\n\n //// for .string files\n //internal byte[] GetData(int blocknumber, out List usedblocks)\n //{\n // lock (_lock)\n // {\n // AllocationBlock ab = FillAllocationBlock(blocknumber);\n // usedblocks = ab.Blocks;\n // byte[] data = readblockdata(ab);\n\n // return data;\n // }\n //}\n\n public int Increment(string key, int amount)\n {\n byte[] k = Helper.GetBytes(key);\n if (k.Length > 255)\n {\n _log.Error(\"Key length > 255 : \" + key);\n throw new Exception(\"Key must be less than 255 characters\");\n //return false;\n }\n lock (_lock)\n {\n if (_isDirty == false)\n WriteDirtyFile();\n\n AllocationBlock ab = null;\n int firstblock = 0;\n if (_keys.Get(key, out firstblock))// key exists already\n ab = FillAllocationBlock(firstblock);\n\n object obj = amount;\n if (ab.deleteKey == false)\n {\n byte[] data = readblockdata(ab);\n\n obj = fastBinaryJSON.BJSON.ToObject(data);\n\n // add here\n if (obj is int)\n obj = ((int)obj) + amount;\n else if (obj is long)\n obj = ((long)obj) + amount;\n else if (obj is decimal)\n obj = ((decimal)obj) + amount;\n else\n return (int)obj;\n }\n\n SaveNew(key, k, obj);\n if (ab != null)\n {\n // free old blocks\n ab.Blocks.Add(ab.blocknumber);\n _datastore.FreeBlocks(ab.Blocks);\n }\n return (int)obj;\n }\n }\n\n public int Decrement(string key, int amount)\n {\n return (int)Increment(key, -amount);\n }\n\n public decimal Increment(string key, decimal amount)\n {\n byte[] k = Helper.GetBytes(key);\n if (k.Length > 255)\n {\n _log.Error(\"Key length > 255 : \" + key);\n throw new Exception(\"Key must be less than 255 characters\");\n //return false;\n }\n lock (_lock)\n {\n if (_isDirty == false)\n WriteDirtyFile();\n\n AllocationBlock ab = null;\n int firstblock = 0;\n if (_keys.Get(key, out firstblock))// key exists already\n ab = FillAllocationBlock(firstblock);\n\n object obj = amount;\n if (ab.deleteKey == false)\n {\n byte[] data = readblockdata(ab);\n\n obj = fastBinaryJSON.BJSON.ToObject(data);\n\n // add here\n if (obj is decimal)\n obj = ((decimal)obj) + amount;\n else\n return (decimal)obj;\n }\n\n SaveNew(key, k, obj);\n if (ab != null)\n {\n // free old blocks\n ab.Blocks.Add(ab.blocknumber);\n _datastore.FreeBlocks(ab.Blocks);\n }\n return (decimal)obj;\n }\n }\n\n public decimal Decrement(string key, decimal amount)\n {\n return Increment(key, -amount);\n }\n }\n}\n" }, { "path": "RaptorDB/Storage/StorageFile.cs", "content": "using System;\nusing System.IO;\nusing System.Text;\nusing System.Collections.Generic;\nusing RaptorDB.Common;\nusing fastBinaryJSON;\nusing fastJSON;\n\nnamespace RaptorDB\n{\n internal class StorageData\n {\n public StorageItem meta;\n public byte[] data;\n }\n\n public class StorageItem\n {\n public T key;\n public string typename;\n public DateTime date = FastDateTime.Now;\n public bool isDeleted;\n public bool isReplicated;\n public int dataLength;\n public byte isCompressed; // 0 = no, 1 = MiniLZO\n }\n\n public interface IDocStorage\n {\n int RecordCount();\n\n byte[] GetBytes(int rowid, out StorageItem meta);\n object GetObject(int rowid, out StorageItem meta);\n StorageItem GetMeta(int rowid);\n\n bool GetObject(T key, out object doc);\n }\n\n public enum SF_FORMAT\n {\n BSON,\n JSON\n }\n\n internal struct SplitFile\n {\n public long start;\n public long uptolength;\n public FileStream file;\n }\n\n public class StorageFile\n {\n FileStream _datawrite;\n FileStream _recfilewrite;\n FileStream _recfileread = null;\n FileStream _dataread = null;\n\n private string _filename = \"\";\n private string _recfilename = \"\";\n private long _lastRecordNum = 0;\n private long _lastWriteOffset = _fileheader.Length;\n private object _readlock = new object();\n private bool _dirty = false;\n IGetBytes _T = null;\n ILog _log = LogManager.GetLogger(typeof(StorageFile));\n private SF_FORMAT _saveFormat = SF_FORMAT.BSON;\n\n // **** change this if storage format changed ****\n internal static int _CurrentVersion = 2;\n\n //private ushort _splitMegaBytes = 0; // 0 = off \n //private bool _enableSplits = false;\n private List _files = new List();\n private List _uptoindexes = new List();\n // no splits in view mode \n private bool _viewmode = false;\n private SplitFile _lastsplitfile;\n\n public static byte[] _fileheader = { (byte)'M', (byte)'G', (byte)'D', (byte)'B',\n 0, // 4 -- storage file version number,\n 0 // 5 -- not used\n };\n private static string _splitfileExtension = \"00000\";\n private const int _KILOBYTE = 1024;\n // record format :\n // 1 type (0 = raw no meta data, 1 = bson meta, 2 = json meta) \n // 4 byte meta/data length, \n // n byte meta serialized data if exists \n // m byte data (if meta exists then m is in meta.dataLength)\n\n /// \n /// View data storage mode (no splits, bson save) \n /// \n /// \n public StorageFile(string filename)\n {\n _viewmode = true;\n _saveFormat = SF_FORMAT.BSON;\n // add version number\n _fileheader[5] = (byte)_CurrentVersion;\n Initialize(filename, false);\n }\n /// \n /// \n /// \n /// \n /// \n /// = true -> don't create mgrec files (used for backup and replication mode)\n public StorageFile(string filename, SF_FORMAT format, bool StorageOnlyMode)\n {\n _saveFormat = format;\n if (StorageOnlyMode) _viewmode = true; // no file splits\n // add version number\n _fileheader[5] = (byte)_CurrentVersion;\n Initialize(filename, StorageOnlyMode);\n }\n\n private StorageFile(string filename, bool StorageOnlyMode)\n {\n Initialize(filename, StorageOnlyMode);\n }\n\n private void Initialize(string filename, bool StorageOnlyMode)\n {\n _T = RDBDataType.ByteHandler();\n _filename = filename;\n\n // search for mgdat00000 extensions -> split files load\n if (File.Exists(filename + _splitfileExtension))\n {\n LoadSplitFiles(filename);\n }\n\n if (File.Exists(filename) == false)\n _datawrite = new FileStream(filename, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite);\n else\n _datawrite = new FileStream(filename, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite);\n\n _dataread = new FileStream(_filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);\n\n if (_datawrite.Length == 0)\n {\n // new file\n _datawrite.Write(_fileheader, 0, _fileheader.Length);\n _datawrite.Flush();\n _lastWriteOffset = _fileheader.Length;\n }\n else\n {\n long i = _datawrite.Seek(0L, SeekOrigin.End);\n if (_files.Count == 0)\n _lastWriteOffset = i;\n else\n _lastWriteOffset += i; // add to the splits\n }\n\n if (StorageOnlyMode == false)\n {\n // load rec pointers\n _recfilename = filename.Substring(0, filename.LastIndexOf('.')) + \".mgrec\";\n if (File.Exists(_recfilename) == false)\n _recfilewrite = new FileStream(_recfilename, FileMode.CreateNew, FileAccess.Write, FileShare.ReadWrite);\n else\n _recfilewrite = new FileStream(_recfilename, FileMode.Open, FileAccess.Write, FileShare.ReadWrite);\n\n _recfileread = new FileStream(_recfilename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);\n\n _lastRecordNum = (int)(_recfilewrite.Length / 8);\n _recfilewrite.Seek(0L, SeekOrigin.End);\n }\n }\n\n private void LoadSplitFiles(string filename)\n {\n _log.Debug(\"Loading split files...\");\n _lastWriteOffset = 0;\n for (int i = 0; ; i++)\n {\n string _filename = filename + i.ToString(_splitfileExtension);\n if (File.Exists(_filename) == false)\n break;\n FileStream file = new FileStream(_filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);\n SplitFile sf = new SplitFile();\n sf.start = _lastWriteOffset;\n _lastWriteOffset += file.Length;\n sf.file = file;\n sf.uptolength = _lastWriteOffset;\n _files.Add(sf);\n _uptoindexes.Add(sf.uptolength);\n }\n _lastsplitfile = _files[_files.Count - 1];\n _log.Debug(\"Number of split files = \" + _files.Count);\n }\n\n public static int GetStorageFileHeaderVersion(string filename)\n {\n string fn = filename + _splitfileExtension; // if split files -> load the header from the first file -> mgdat00000\n if (File.Exists(fn) == false)\n fn = filename; // else use the mgdat file \n\n if (File.Exists(fn))\n {\n var fs = new FileStream(fn, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite);\n fs.Seek(0L, SeekOrigin.Begin);\n byte[] b = new byte[_fileheader.Length];\n fs.Read(b, 0, _fileheader.Length);\n fs.Close();\n return b[5];\n }\n return _CurrentVersion;\n }\n\n public int Count()\n {\n return (int)_lastRecordNum;// (int)(_recfilewrite.Length >> 3);\n }\n\n public long WriteRawData(byte[] b)\n {\n return internalWriteData(null, b, true);\n }\n\n public long Delete(T key)\n {\n StorageItem meta = new StorageItem();\n meta.key = key;\n meta.isDeleted = true;\n\n return internalWriteData(meta, null, false);\n }\n\n public long DeleteReplicated(T key)\n {\n StorageItem meta = new StorageItem();\n meta.key = key;\n meta.isReplicated = true;\n meta.isDeleted = true;\n\n return internalWriteData(meta, null, false);\n }\n\n public long WriteObject(T key, object obj)\n {\n StorageItem meta = new StorageItem();\n meta.key = key;\n meta.typename = Reflection.Instance.GetTypeAssemblyName(obj.GetType());\n byte[] data;\n if (_saveFormat == SF_FORMAT.BSON)\n data = BJSON.ToBJSON(obj);\n else\n data = Helper.GetBytes(JSON.ToJSON(obj));\n if (data.Length > (int)Global.CompressDocumentOverKiloBytes * _KILOBYTE)\n {\n meta.isCompressed = 1;\n data = MiniLZO.Compress(data); //MiniLZO\n }\n return internalWriteData(meta, data, false);\n }\n\n public long WriteReplicationObject(T key, object obj)\n {\n StorageItem meta = new StorageItem();\n meta.key = key;\n meta.isReplicated = true;\n meta.typename = Reflection.Instance.GetTypeAssemblyName(obj.GetType());\n byte[] data;\n if (_saveFormat == SF_FORMAT.BSON)\n data = BJSON.ToBJSON(obj);\n else\n data = Helper.GetBytes(JSON.ToJSON(obj));\n if (data.Length > (int)Global.CompressDocumentOverKiloBytes * _KILOBYTE)\n {\n meta.isCompressed = 1;\n data = MiniLZO.Compress(data);\n }\n return internalWriteData(meta, data, false);\n }\n\n public long WriteData(T key, byte[] data)\n {\n StorageItem meta = new StorageItem();\n meta.key = key;\n\n if (data.Length > (int)Global.CompressDocumentOverKiloBytes * _KILOBYTE)\n {\n meta.isCompressed = 1;\n data = MiniLZO.Compress(data);\n }\n\n return internalWriteData(meta, data, false);\n }\n\n public byte[] ReadBytes(long recnum)\n {\n StorageItem meta;\n return ReadBytes(recnum, out meta);\n }\n\n public object ReadObject(long recnum)\n {\n StorageItem meta = null;\n return ReadObject(recnum, out meta);\n }\n\n public object ReadObject(long recnum, out StorageItem meta)\n {\n byte[] b = ReadBytes(recnum, out meta);\n\n if (b == null)\n return null;\n if (b[0] < 32)\n return BJSON.ToObject(b);\n else\n return JSON.ToObject(Encoding.ASCII.GetString(b));\n }\n\n /// \n /// used for views only\n /// \n /// \n /// \n public byte[] ViewReadRawBytes(long recnum)\n {\n // views can't be split\n if (recnum >= _lastRecordNum)\n return null;\n\n lock (_readlock)\n {\n long offset = ComputeOffset(recnum);\n _dataread.Seek(offset, System.IO.SeekOrigin.Begin);\n byte[] hdr = new byte[5];\n // read header\n _dataread.Read(hdr, 0, 5); // meta length\n int len = Helper.ToInt32(hdr, 1);\n\n int type = hdr[0];\n if (type == 0)\n {\n byte[] data = new byte[len];\n _dataread.Read(data, 0, len);\n return data;\n }\n return null;\n }\n }\n\n public void Shutdown()\n {\n if (_files.Count > 0)\n _files.ForEach(s => FlushClose(s.file));\n\n FlushClose(_dataread);\n FlushClose(_recfileread);\n FlushClose(_recfilewrite);\n FlushClose(_datawrite);\n\n _dataread = null;\n _recfileread = null;\n _recfilewrite = null;\n _datawrite = null;\n }\n\n public static StorageFile ReadForward(string filename)\n {\n StorageFile sf = new StorageFile(filename, true);\n\n return sf;\n }\n\n public StorageItem ReadMeta(long rowid)\n {\n if (rowid >= _lastRecordNum)\n return null;\n lock (_readlock)\n {\n int metalen = 0;\n long off = ComputeOffset(rowid);\n FileStream fs = GetReadFileStreamWithSeek(off);\n StorageItem meta = ReadMetaData(fs, out metalen);\n return meta;\n }\n }\n\n #region [ private / internal ]\n\n private long internalWriteData(StorageItem meta, byte[] data, bool raw)\n {\n lock (_readlock)\n {\n _dirty = true;\n // seek end of file\n long offset = _lastWriteOffset;\n if (_viewmode == false && Global.SplitStorageFilesMegaBytes > 0)\n {\n // current file size > _splitMegaBytes --> new file\n if (offset > (long)Global.SplitStorageFilesMegaBytes * 1024 * 1024)\n CreateNewStorageFile();\n }\n\n if (raw == false)\n {\n if (data != null)\n meta.dataLength = data.Length;\n byte[] metabytes = BJSON.ToBJSON(meta, new BJSONParameters { UseExtensions = false, UseTypedArrays = false });\n\n // write header info\n _datawrite.Write(new byte[] { 1 }, 0, 1); // FEATURE : add json here, write bson for now\n _datawrite.Write(Helper.GetBytes(metabytes.Length, false), 0, 4);\n _datawrite.Write(metabytes, 0, metabytes.Length);\n // update pointer\n _lastWriteOffset += metabytes.Length + 5;\n }\n else\n {\n // write header info\n _datawrite.Write(new byte[] { 0 }, 0, 1); // write raw\n _datawrite.Write(Helper.GetBytes(data.Length, false), 0, 4);\n // update pointer\n _lastWriteOffset += 5;\n }\n\n if (data != null)\n {\n // write data block\n _datawrite.Write(data, 0, data.Length);\n _lastWriteOffset += data.Length;\n }\n // return starting offset -> recno\n long recno = _lastRecordNum++;\n if (_recfilewrite != null)\n _recfilewrite.Write(Helper.GetBytes(offset, false), 0, 8);\n if (Global.FlushStorageFileImmediately)\n {\n _datawrite.Flush();\n if (_recfilewrite != null)\n _recfilewrite.Flush();\n }\n return recno;\n }\n }\n\n private void CreateNewStorageFile()\n {\n _log.Debug(\"Split limit reached = \" + _datawrite.Length);\n int i = _files.Count;\n // close files\n FlushClose(_datawrite);\n FlushClose(_dataread);\n long start = 0;\n if (i > 0)\n start = _lastsplitfile.uptolength; // last file offset\n // rename mgdat to mgdat0000n\n File.Move(_filename, _filename + i.ToString(_splitfileExtension));\n FileStream file = new FileStream(_filename + i.ToString(_splitfileExtension), FileMode.Open, FileAccess.Read, FileShare.ReadWrite);\n SplitFile sf = new SplitFile();\n sf.start = start;\n sf.uptolength = _lastWriteOffset;\n sf.file = file;\n _files.Add(sf);\n\n _uptoindexes.Add(sf.uptolength);\n\n _lastsplitfile = sf;\n // new mgdat file\n _datawrite = new FileStream(_filename, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite);\n _dataread = new FileStream(_filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);\n _log.Debug(\"New storage file created, count = \" + _files.Count);\n }\n\n internal byte[] ReadBytes(long recnum, out StorageItem meta)\n {\n meta = null;\n if (recnum >= _lastRecordNum)\n return null;\n lock (_readlock)\n {\n long off = ComputeOffset(recnum);\n FileStream fs = GetReadFileStreamWithSeek(off);\n byte[] data = internalReadBytes(fs, out meta);\n\n if (meta.isCompressed > 0)\n data = MiniLZO.Decompress(data);\n\n return data;\n }\n }\n\n private long ComputeOffset(long recnum)\n {\n if (_dirty)\n {\n _datawrite.Flush();\n _recfilewrite.Flush();\n }\n long off = recnum << 3;// *8L;\n byte[] b = new byte[8];\n\n _recfileread.Seek(off, SeekOrigin.Begin);\n _recfileread.Read(b, 0, 8);\n off = Helper.ToInt64(b, 0);\n if (off == 0)// kludge\n off = 6;\n return off;\n }\n\n private byte[] internalReadBytes(FileStream fs, out StorageItem meta)\n {\n int metalen = 0;\n meta = ReadMetaData(fs, out metalen);\n if (meta != null)\n {\n if (meta.isDeleted == false)\n {\n byte[] data = new byte[meta.dataLength];\n fs.Read(data, 0, meta.dataLength);\n return data;\n }\n }\n else\n {\n byte[] data = new byte[metalen];\n fs.Read(data, 0, metalen);\n return data;\n }\n return null;\n }\n\n private StorageItem ReadMetaData(FileStream fs, out int metasize)\n {\n byte[] hdr = new byte[5];\n // read header\n fs.Read(hdr, 0, 5); // meta length\n int len = Helper.ToInt32(hdr, 1);\n int type = hdr[0];\n if (type > 0)\n {\n metasize = len + 5;\n hdr = new byte[len];\n fs.Read(hdr, 0, len);\n StorageItem meta;\n if (type == 1)\n meta = BJSON.ToObject>(hdr);\n else\n {\n string str = Helper.GetString(hdr, 0, (short)hdr.Length);\n meta = JSON.ToObject>(str);\n }\n return meta;\n }\n else\n {\n metasize = len;\n return null;\n }\n }\n\n private void FlushClose(FileStream st)\n {\n if (st != null)\n {\n st.Flush(true);\n st.Close();\n }\n }\n\n internal T GetKey(long recnum, out bool deleted)\n {\n lock (_readlock)\n {\n deleted = false;\n long off = ComputeOffset(recnum);\n FileStream fs = GetReadFileStreamWithSeek(off);\n\n int metalen = 0;\n StorageItem meta = ReadMetaData(fs, out metalen);\n deleted = meta.isDeleted;\n return meta.key;\n }\n }\n\n internal int CopyTo(StorageFile storageFile, long startrecord)\n {\n FileStream fs;\n bool inthefiles = false;\n // copy data here\n lock (_readlock)\n {\n long off = ComputeOffset(startrecord);\n fs = GetReadFileStreamWithSeek(off);\n if (fs != _dataread)\n inthefiles = true;\n Pump(fs, storageFile._datawrite);\n }\n\n // pump the remainder of the files also \n if (inthefiles && _files.Count > 0)\n {\n long off = ComputeOffset(startrecord);\n int i = binarysearch(off);\n i++; // next file stream\n for (int j = i; j < _files.Count; j++)\n {\n lock (_readlock)\n {\n fs = _files[j].file;\n fs.Seek(0L, SeekOrigin.Begin);\n Pump(fs, storageFile._datawrite);\n }\n }\n\n // pump the current mgdat\n lock (_readlock)\n {\n _dataread.Seek(0L, SeekOrigin.Begin);\n Pump(_dataread, storageFile._datawrite);\n }\n }\n\n return (int)_lastRecordNum;\n }\n\n private static void Pump(Stream input, Stream output)\n {\n byte[] bytes = new byte[4096 * 2];\n int n;\n while ((n = input.Read(bytes, 0, bytes.Length)) != 0)\n output.Write(bytes, 0, n);\n }\n\n internal IEnumerable> ReadOnlyEnumerate()\n {\n // MGREC files may not exist\n\n //// the total number of records \n //long count = _recfileread.Length >> 3;\n\n //for (long i = 0; i < count; i++)\n //{\n // StorageItem meta;\n // byte[] data = ReadBytes(i, out meta);\n // StorageData sd = new StorageData();\n // sd.meta = meta;\n // if (meta.dataLength > 0)\n // sd.data = data;\n\n // yield return sd;\n //}\n\n long offset = _fileheader.Length;// start; // skip header\n long size = _dataread.Length;\n while (offset < size)\n {\n StorageData sd = new StorageData();\n lock (_readlock)\n {\n _dataread.Seek(offset, SeekOrigin.Begin);\n int metalen = 0;\n StorageItem meta = ReadMetaData(_dataread, out metalen);\n offset += metalen;\n\n sd.meta = meta;\n if (meta.dataLength > 0)\n {\n byte[] data = new byte[meta.dataLength];\n _dataread.Read(data, 0, meta.dataLength);\n sd.data = data;\n }\n offset += meta.dataLength;\n }\n yield return sd;\n }\n }\n\n private FileStream GetReadFileStreamWithSeek(long offset)\n {\n long fileoffset = offset;\n // search split _files for offset and compute fileoffset in the file\n if (_files.Count > 0) // we have splits\n {\n if (offset < _lastsplitfile.uptolength) // offset is in the list\n {\n int i = binarysearch(offset);\n var f = _files[i];\n fileoffset -= f.start; // offset in the file \n f.file.Seek(fileoffset, SeekOrigin.Begin);\n return f.file;\n }\n else\n fileoffset -= _lastsplitfile.uptolength; // offset in the mgdat file\n }\n\n // seek to position in file \n _dataread.Seek(fileoffset, SeekOrigin.Begin);\n return _dataread;\n }\n\n private int binarysearch(long offset)\n {\n //// binary search\n int low = 0;\n int high = _files.Count - 1;\n int midpoint = 0;\n int lastlower = 0;\n\n while (low <= high)\n {\n midpoint = low + (high - low) / 2;\n long k = _uptoindexes[midpoint];\n // check to see if value is equal to item in array\n if (offset == k)\n return midpoint + 1;\n else if (offset < k)\n {\n high = midpoint - 1;\n lastlower = midpoint;\n }\n else\n low = midpoint + 1;\n }\n\n return lastlower;\n }\n #endregion\n }\n}\n" }, { "path": "RaptorDB/Storage/StorageFileHF.cs", "content": "using RaptorDB.Common;\nusing System;\nusing System.Collections.Generic;\nusing System.IO;\nusing System.Threading;\n\nnamespace RaptorDB\n{\n // high frequency storage file with overwrite old values\n internal class StorageFileHF\n {\n private FileStream _datawriteorg;\n private BufferedStream _datawrite;\n private MGRB _freeList = new MGRB();\n private string _filename = \"\";\n private object _readlock = new object();\n ILog _log = LogManager.GetLogger(typeof(StorageFileHF));\n\n // **** change this if storage format changed ****\n private static int _CurrentVersion = 1;\n private int _lastBlockNumber = -1;\n private ushort _BLOCKSIZE = 4096;\n private string _Path = \"\";\n private string _S = Path.DirectorySeparatorChar.ToString();\n\n public static byte[] _fileheader = { (byte)'M', (byte)'G', (byte)'H', (byte)'F',\n 0, // 4 -- storage file version number,\n 0,2, // 5,6 -- block size ushort low, hi\n 1 // 7 -- key type 0 = guid, 1 = string\n };\n\n //private SafeDictionary _masterblock = new SafeDictionary();\n\n public StorageFileHF(string filename, ushort blocksize)\n {\n _Path = Path.GetDirectoryName(filename);\n if (_Path.EndsWith(_S) == false) _Path += _S;\n _filename = Path.GetFileNameWithoutExtension(filename);\n\n Initialize(filename, blocksize);\n }\n\n public void Shutdown()\n {\n WriteFreeListBMPFile();\n _datawrite.Flush();\n FlushClose(_datawriteorg);\n _datawrite = null;\n }\n\n public ushort GetBlockSize()\n {\n return _BLOCKSIZE;\n }\n\n internal void FreeBlocks(List list)\n {\n list.ForEach(x => _freeList.Set(x, true));\n }\n\n internal byte[] ReadBlock(int blocknumber)\n {\n SeekBlock(blocknumber);\n byte[] data = new byte[_BLOCKSIZE];\n _datawrite.Read(data, 0, _BLOCKSIZE);\n\n return data;\n }\n\n internal byte[] ReadBlockBytes(int blocknumber, int bytes)\n {\n SeekBlock(blocknumber);\n byte[] data = new byte[bytes];\n _datawrite.Read(data, 0, bytes);\n\n return data;\n }\n\n internal int GetFreeBlockNumber()\n {\n // get the first free block or append to the end\n if (_freeList.CountOnes() > 0)\n {\n int i = _freeList.GetFirst();\n _freeList.Set(i, false);\n return i;\n }\n else\n return Interlocked.Increment(ref _lastBlockNumber);\n }\n\n private void InitializeFreeList()\n {\n if (_lastBlockNumber < 0)\n {\n // write master block\n _datawrite.Write(new byte[_BLOCKSIZE], 0, _BLOCKSIZE);\n _lastBlockNumber = 1;\n //_masterblock.Add(\"freelist\", -1);\n }\n else\n {\n _freeList = new MGRB();\n // read master block data\n var b = ReadBlock(0);\n if (b[0] == (byte)'F' && b[1] == (byte)'L')\n {\n // get free block num and size\n int block = Helper.ToInt32(b, 2);\n int len = Helper.ToInt32(b, 2 + 4);\n int freeblock = block;\n b = new byte[len];\n var offset = 0;\n bool failed = false;\n // read blocks upto size from block num\n SeekBlock(++block);\n while (len > 0)\n {\n // check header \n var bb = ReadBlock(block);\n if (bb[0] != (byte)'F' || bb[1] != (byte)'L')\n {\n // throw exception??\n _log.Error(\"Free list header does not match : \" + _filename);\n failed = true;\n break;\n }\n int c = len > _BLOCKSIZE ? _BLOCKSIZE - 2 : len;\n Buffer.BlockCopy(bb, 2, b, offset, c);\n len -= c;\n offset += c;\n }\n if (failed == false)\n {\n // read freelist from master block from end of file\n var o = fastBinaryJSON.BJSON.ToObject(b);\n _freeList.Deserialize(o);\n // truncate end of file freelist blocks if lastblock < file size\n if (_datawrite.Length > _lastBlockNumber * _BLOCKSIZE)\n _datawrite.SetLength(_lastBlockNumber * _BLOCKSIZE);\n }\n _lastBlockNumber = freeblock;\n }\n }\n }\n\n internal void SeekBlock(int blocknumber)\n {\n long offset = _fileheader.Length + (long)blocknumber * _BLOCKSIZE;\n // wiil seek past the end of file on fs.Write will zero the difference\n _datawrite.Seek(offset, SeekOrigin.Begin);\n }\n\n internal void WriteBlockBytes(byte[] data, int start, int len)\n {\n _datawrite.Write(data, start, len);\n }\n\n #region [ private / internal ]\n\n private void WriteFreeListBMPFile()\n {\n // write freelist to end of blocks and update master block\n if (_freeList != null)\n {\n _freeList.Optimize();\n var o = _freeList.Serialize();\n var b = fastBinaryJSON.BJSON.ToBJSON(o, new fastBinaryJSON.BJSONParameters { UseExtensions = false });\n\n var len = b.Length;\n var offset = 0;\n // write master block \n SeekBlock(0);\n //_lastBlockNumber++;\n WriteBlockBytes(new byte[] { (byte)'F', (byte)'L' }, 0, 2);\n WriteBlockBytes(Helper.GetBytes(_lastBlockNumber, false), 0, 4);\n WriteBlockBytes(Helper.GetBytes(len, false), 0, 4);\n // seek to end of file\n SeekBlock(_lastBlockNumber+1);\n while (len > 0)\n {\n WriteBlockBytes(new byte[] { (byte)'F', (byte)'L' }, 0, 2);\n WriteBlockBytes(b, offset, len > _BLOCKSIZE ? _BLOCKSIZE - 2 : len);\n len -= (_BLOCKSIZE - 2);\n }\n }\n }\n\n private void Initialize(string filename, ushort blocksize)\n {\n if (File.Exists(filename) == false)\n _datawriteorg = new FileStream(filename, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite);\n else\n _datawriteorg = new FileStream(filename, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite);\n\n _datawrite = new BufferedStream(_datawriteorg);\n\n if (_datawrite.Length == 0)\n {\n CreateFileHeader(blocksize);\n // new file\n _datawrite.Write(_fileheader, 0, _fileheader.Length);\n _datawrite.Flush();\n }\n else\n {\n int filever = ReadFileHeader();\n if (filever < _CurrentVersion)\n {\n // fixx : upgrade storage file here\n }\n _lastBlockNumber = (int)((_datawrite.Length - _fileheader.Length) / _BLOCKSIZE);\n //_lastBlockNumber++;\n }\n\n InitializeFreeList();\n }\n\n private int ReadFileHeader()\n {\n // set _blockize\n _datawrite.Seek(0L, SeekOrigin.Begin);\n byte[] hdr = new byte[_fileheader.Length];\n _datawrite.Read(hdr, 0, _fileheader.Length);\n\n _BLOCKSIZE = 0;\n _BLOCKSIZE = (ushort)(hdr[5] + hdr[6] << 8);\n\n return hdr[4];\n }\n\n private void CreateFileHeader(int blocksize)\n {\n // add version number\n _fileheader[4] = (byte)_CurrentVersion;\n // block size\n _fileheader[5] = (byte)(blocksize & 0xff);\n _fileheader[6] = (byte)(blocksize >> 8);\n _BLOCKSIZE = (ushort)blocksize;\n }\n\n private void FlushClose(FileStream st)\n {\n if (st != null)\n {\n st.Flush(true);\n st.Close();\n }\n }\n #endregion\n\n internal int NumberofBlocks()\n {\n return (int)((_datawrite.Length / _BLOCKSIZE)) + 1;\n }\n\n internal void FreeBlock(int i)\n {\n _freeList.Set(i, true);\n }\n }\n}\n" }, { "path": "RaptorDB/Storage/StringHF.cs", "content": "using RaptorDB.Common;\nusing System;\nusing System.Collections.Generic;\nusing System.IO;\nusing System.Linq;\n\nnamespace RaptorDB\n{\n // high frequency string value store\n public class StringHF //: IKeyStoreHF\n {\n internal class AllocationBlock\n {\n public string key;\n public byte keylen;\n public int datalength;\n public bool isCompressed;\n public bool isBinaryJSON;\n public bool deleteKey;\n public List Blocks = new List();\n public int blocknumber;\n }\n\n //MGIndex _keys;\n StorageFileHF _datastore;\n object _lock = new object();\n ushort _BlockSize = 2048;\n //private const int _KILOBYTE = 1024;\n ILog _log = LogManager.GetLogger(typeof(KeyStoreHF));\n\n byte[] _blockheader = new byte[]{\n 0,0,0,0, // 0 block # (used for validate block reads and rebuild)\n 0,0,0,0, // 4 next block # \n 0, // 8 flags bits 0:iscompressed 1:isbinary 2:deletekey\n 0,0,0,0, // 9 data length (compute alloc blocks needed)\n 0, // 13 key length \n 0, // 14 key type 0=guid 1=string\n };\n private string _Path = \"\";\n private string _S = Path.DirectorySeparatorChar.ToString();\n //private bool _isDirty = false;\n //private string _dirtyFilename = \"temp.$\";\n\n // mgindex special storage for strings ctor -> no idx file\n // use SaveData() GetData()\n public StringHF(string folder, string filename)\n {\n _Path = folder;\n Directory.CreateDirectory(_Path);\n if (_Path.EndsWith(_S) == false) _Path += _S;\n\n _datastore = new StorageFileHF(_Path + filename, Global.HighFrequencyKVDiskBlockSize);\n //_datastore.Initialize();\n _BlockSize = _datastore.GetBlockSize();\n }\n\n public void Shutdown()\n {\n _datastore.Shutdown();\n //if (_keys != null)\n // _keys.Shutdown();\n\n //if (File.Exists(_Path + _dirtyFilename))\n // File.Delete(_Path + _dirtyFilename);\n }\n\n internal void FreeMemory()\n {\n //if (_keys != null)\n // _keys.FreeMemory();\n }\n\n #region [ private methods ]\n private byte[] readblockdata(AllocationBlock ab)\n {\n byte[] data = new byte[ab.datalength];\n long offset = 0;\n int len = ab.datalength;\n int dbsize = _BlockSize - _blockheader.Length - ab.keylen;\n ab.Blocks.ForEach(x =>\n {\n byte[] b = _datastore.ReadBlock(x);\n int c = len;\n if (c > dbsize) c = dbsize;\n Buffer.BlockCopy(b, _blockheader.Length + ab.keylen, data, (int)offset, c);\n offset += c;\n len -= c;\n });\n if (ab.isCompressed)\n data = MiniLZO.Decompress(data);\n return data;\n }\n\n private int internalSave(byte[] keybytes, byte[] data, AllocationBlock ab)\n {\n ab.Blocks = new List();\n int firstblock = _datastore.GetFreeBlockNumber();\n ab.Blocks.Add(firstblock);\n int blocknum = firstblock;\n byte[] header = CreateAllocHeader(ab, keybytes);\n int dblocksize = _BlockSize - header.Length;\n int offset = 0;\n // compute data block count\n int datablockcount = (data.Length / dblocksize) + 1;\n // save data blocks\n int counter = 0;\n int len = data.Length;\n while (datablockcount > 0)\n {\n datablockcount--;\n int next = 0;\n if (datablockcount > 0)\n next = _datastore.GetFreeBlockNumber();\n Buffer.BlockCopy(Helper.GetBytes(counter, false), 0, header, 0, 4); // set block number\n Buffer.BlockCopy(Helper.GetBytes(next, false), 0, header, 4, 4); // set next pointer\n\n _datastore.SeekBlock(blocknum);\n _datastore.WriteBlockBytes(header, 0, header.Length);\n int c = len;\n if (c > dblocksize)\n c = dblocksize;\n _datastore.WriteBlockBytes(data, offset, c);\n\n if (next > 0)\n {\n blocknum = next;\n ab.Blocks.Add(next);\n }\n offset += c;\n len -= c;\n counter++;\n }\n return firstblock;\n }\n\n private byte[] CreateAllocHeader(AllocationBlock ab, byte[] keybytes)\n {\n byte[] alloc = new byte[_blockheader.Length + keybytes.Length];\n\n if (ab.isCompressed)\n alloc[8] = 1;\n if (ab.isBinaryJSON)\n alloc[8] += 2;\n if (ab.deleteKey)\n alloc[8] += 4;\n\n Buffer.BlockCopy(Helper.GetBytes(ab.datalength, false), 0, alloc, 9, 4);\n alloc[13] = ab.keylen;\n alloc[14] = 1; // string keys for now\n Buffer.BlockCopy(keybytes, 0, alloc, _blockheader.Length, ab.keylen);\n\n return alloc;\n }\n\n private AllocationBlock FillAllocationBlock(int blocknumber)\n {\n AllocationBlock ab = new AllocationBlock();\n\n ab.blocknumber = blocknumber;\n ab.Blocks.Add(blocknumber);\n\n byte[] b = _datastore.ReadBlockBytes(blocknumber, _blockheader.Length + 255);\n\n int blocknumexpected = 0;\n\n int next = ParseBlockHeader(ab, b, blocknumexpected);\n\n blocknumexpected++;\n\n while (next > 0)\n {\n ab.Blocks.Add(next);\n b = _datastore.ReadBlockBytes(next, _blockheader.Length + ab.keylen);\n next = ParseBlockHeader(ab, b, blocknumexpected);\n blocknumexpected++;\n }\n\n return ab;\n }\n\n private int ParseBlockHeader(AllocationBlock ab, byte[] b, int blocknumberexpected)\n {\n int bnum = Helper.ToInt32(b, 0);\n if (bnum != blocknumberexpected)\n {\n _log.Error(\"Block numbers does not match, looking for : \" + blocknumberexpected);\n //throw new Exception(\"Block numbers does not match, looking for : \" + blocknumberexpected);\n return -1;\n }\n if (b[14] != 1)\n {\n _log.Error(\"Expecting string keys only, got : \" + b[14]);\n //throw new Exception(\"Expecting string keys only, got : \" + b[11]);\n return -1;\n }\n\n int next = Helper.ToInt32(b, 4);\n\n if (ab.keylen == 0)\n {\n byte flags = b[8];\n\n if ((flags & 0x01) > 0)\n ab.isCompressed = true;\n if ((flags & 0x02) > 0)\n ab.isBinaryJSON = true;\n if ((flags & 0x04) > 0)\n ab.deleteKey = true;\n\n ab.datalength = Helper.ToInt32(b, 9);\n byte keylen = b[13];\n ab.keylen = keylen;\n ab.key = Helper.GetString(b, _blockheader.Length, keylen);\n }\n return next;\n }\n\n #endregion\n\n internal void FreeBlocks(List list)\n {\n lock (_lock)\n _datastore.FreeBlocks(list);\n }\n\n\n // for .string files\n internal int SaveData(string key, byte[] data, out List blocks)\n {\n lock (_lock)\n {\n byte[] kb = Helper.GetBytes(key);\n AllocationBlock ab = new AllocationBlock();\n ab.key = key;\n ab.keylen = (byte)kb.Length;\n ab.isCompressed = false;\n ab.isBinaryJSON = true;\n ab.datalength = data.Length;\n\n int firstblock = internalSave(kb, data, ab);\n blocks = ab.Blocks;\n return firstblock;\n }\n }\n\n // for .string files\n internal byte[] GetData(int blocknumber, out List usedblocks)\n {\n lock (_lock)\n {\n AllocationBlock ab = FillAllocationBlock(blocknumber);\n usedblocks = ab.Blocks;\n byte[] data = readblockdata(ab);\n\n return data;\n }\n }\n }\n}\n" }, { "path": "RaptorDB/Views/Dynamic.cs", "content": "//Copyright (C) Microsoft Corporation. All rights reserved.\n\nusing System.Collections.Generic;\nusing System.Text;\nusing System.Linq.Expressions;\nusing System.Reflection;\nusing System.Reflection.Emit;\nusing System.Threading;\n\nnamespace System.Linq.Dynamic\n{\n // FEATURE : cleanup unused code here\n\n #region [ Classes ]\n internal abstract class DynamicClass\n {\n public override string ToString()\n {\n PropertyInfo[] props = this.GetType().GetProperties(BindingFlags.Instance | BindingFlags.Public);\n StringBuilder sb = new StringBuilder();\n sb.Append(\"{\");\n for (int i = 0; i < props.Length; i++)\n {\n if (i > 0) sb.Append(\", \");\n sb.Append(props[i].Name);\n sb.Append(\"=\");\n sb.Append(props[i].GetValue(this, null));\n }\n sb.Append(\"}\");\n return sb.ToString();\n }\n }\n\n internal class DynamicProperty\n {\n string name;\n Type type;\n\n public DynamicProperty(string name, Type type)\n {\n if (name == null) throw new ArgumentNullException(\"name\");\n if (type == null) throw new ArgumentNullException(\"type\");\n this.name = name;\n this.type = type;\n }\n\n public string Name\n {\n get { return name; }\n }\n\n public Type Type\n {\n get { return type; }\n }\n }\n\n internal static class DynamicExpression\n {\n public static Expression Parse(Type resultType, string expression, params object[] values)\n {\n ExpressionParser parser = new ExpressionParser(null, expression, values);\n return parser.Parse(resultType);\n }\n\n public static LambdaExpression ParseLambda(Type itType, Type resultType, string expression, params object[] values)\n {\n return ParseLambda(new ParameterExpression[] { Expression.Parameter(itType, \"\") }, resultType, expression, values);\n }\n\n public static LambdaExpression ParseLambda(ParameterExpression[] parameters, Type resultType, string expression, params object[] values)\n {\n ExpressionParser parser = new ExpressionParser(parameters, expression, values);\n return Expression.Lambda(parser.Parse(resultType), parameters);\n }\n\n //public static Expression> ParseLambda(string expression, params object[] values)\n //{\n // return (Expression>)ParseLambda(typeof(T), typeof(S), expression, values);\n //}\n\n public static Type CreateClass(params DynamicProperty[] properties)\n {\n return ClassFactory.Instance.GetDynamicClass(properties);\n }\n\n public static Type CreateClass(IEnumerable properties)\n {\n return ClassFactory.Instance.GetDynamicClass(properties);\n }\n }\n\n //internal class DynamicOrdering\n //{\n // public Expression Selector;\n // public bool Ascending;\n //}\n\n internal class Signature : IEquatable\n {\n public DynamicProperty[] properties;\n public int hashCode;\n\n public Signature(IEnumerable properties)\n {\n this.properties = properties.ToArray();\n hashCode = 0;\n foreach (DynamicProperty p in properties)\n {\n hashCode ^= p.Name.GetHashCode() ^ p.Type.GetHashCode();\n }\n }\n\n public override int GetHashCode()\n {\n return hashCode;\n }\n\n public override bool Equals(object obj)\n {\n return obj is Signature ? Equals((Signature)obj) : false;\n }\n\n public bool Equals(Signature other)\n {\n if (properties.Length != other.properties.Length) return false;\n for (int i = 0; i < properties.Length; i++)\n {\n if (properties[i].Name != other.properties[i].Name ||\n properties[i].Type != other.properties[i].Type) return false;\n }\n return true;\n }\n }\n\n internal class ClassFactory\n {\n public static readonly ClassFactory Instance = new ClassFactory();\n\n static ClassFactory() { } // Trigger lazy initialization of static fields\n\n ModuleBuilder module;\n Dictionary classes;\n int classCount;\n ReaderWriterLock rwLock;\n\n private ClassFactory()\n {\n AssemblyName name = new AssemblyName(\"DynamicClasses\");\n#if NETSTANDARD2_0\n AssemblyBuilder assembly = AssemblyBuilder.DefineDynamicAssembly(name, AssemblyBuilderAccess.Run);\n#else\n AssemblyBuilder assembly = AppDomain.CurrentDomain.DefineDynamicAssembly(name, AssemblyBuilderAccess.Run);\n#endif\n#if ENABLE_LINQ_PARTIAL_TRUST\n new ReflectionPermission(PermissionState.Unrestricted).Assert();\n#endif\n try\n {\n module = assembly.DefineDynamicModule(\"Module\");\n }\n finally\n {\n#if ENABLE_LINQ_PARTIAL_TRUST\n PermissionSet.RevertAssert();\n#endif\n }\n classes = new Dictionary();\n rwLock = new ReaderWriterLock();\n }\n\n public Type GetDynamicClass(IEnumerable properties)\n {\n rwLock.AcquireReaderLock(Timeout.Infinite);\n try\n {\n Signature signature = new Signature(properties);\n Type type;\n if (!classes.TryGetValue(signature, out type))\n {\n type = CreateDynamicClass(signature.properties);\n classes.Add(signature, type);\n }\n return type;\n }\n finally\n {\n rwLock.ReleaseReaderLock();\n }\n }\n\n Type CreateDynamicClass(DynamicProperty[] properties)\n {\n LockCookie cookie = rwLock.UpgradeToWriterLock(Timeout.Infinite);\n try\n {\n string typeName = \"DynamicClass\" + (classCount + 1);\n#if ENABLE_LINQ_PARTIAL_TRUST\n new ReflectionPermission(PermissionState.Unrestricted).Assert();\n#endif\n try\n {\n TypeBuilder tb = this.module.DefineType(typeName, TypeAttributes.Class |\n TypeAttributes.Public, typeof(DynamicClass));\n FieldInfo[] fields = GenerateProperties(tb, properties);\n GenerateEquals(tb, fields);\n GenerateGetHashCode(tb, fields);\n#if NETSTANDARD2_0\n Type result = tb.CreateTypeInfo();\n#else\n Type result = tb.CreateType();\n#endif\n classCount++;\n return result;\n }\n finally\n {\n#if ENABLE_LINQ_PARTIAL_TRUST\n PermissionSet.RevertAssert();\n#endif\n }\n }\n finally\n {\n rwLock.DowngradeFromWriterLock(ref cookie);\n }\n }\n\n FieldInfo[] GenerateProperties(TypeBuilder tb, DynamicProperty[] properties)\n {\n FieldInfo[] fields = new FieldBuilder[properties.Length];\n for (int i = 0; i < properties.Length; i++)\n {\n DynamicProperty dp = properties[i];\n FieldBuilder fb = tb.DefineField(\"_\" + dp.Name, dp.Type, FieldAttributes.Private);\n PropertyBuilder pb = tb.DefineProperty(dp.Name, PropertyAttributes.HasDefault, dp.Type, null);\n MethodBuilder mbGet = tb.DefineMethod(\"get_\" + dp.Name,\n MethodAttributes.Public | MethodAttributes.SpecialName | MethodAttributes.HideBySig,\n dp.Type, Type.EmptyTypes);\n ILGenerator genGet = mbGet.GetILGenerator();\n genGet.Emit(OpCodes.Ldarg_0);\n genGet.Emit(OpCodes.Ldfld, fb);\n genGet.Emit(OpCodes.Ret);\n MethodBuilder mbSet = tb.DefineMethod(\"set_\" + dp.Name,\n MethodAttributes.Public | MethodAttributes.SpecialName | MethodAttributes.HideBySig,\n null, new Type[] { dp.Type });\n ILGenerator genSet = mbSet.GetILGenerator();\n genSet.Emit(OpCodes.Ldarg_0);\n genSet.Emit(OpCodes.Ldarg_1);\n genSet.Emit(OpCodes.Stfld, fb);\n genSet.Emit(OpCodes.Ret);\n pb.SetGetMethod(mbGet);\n pb.SetSetMethod(mbSet);\n fields[i] = fb;\n }\n return fields;\n }\n\n void GenerateEquals(TypeBuilder tb, FieldInfo[] fields)\n {\n MethodBuilder mb = tb.DefineMethod(\"Equals\",\n MethodAttributes.Public | MethodAttributes.ReuseSlot |\n MethodAttributes.Virtual | MethodAttributes.HideBySig,\n typeof(bool), new Type[] { typeof(object) });\n ILGenerator gen = mb.GetILGenerator();\n LocalBuilder other = gen.DeclareLocal(tb);\n Label next = gen.DefineLabel();\n gen.Emit(OpCodes.Ldarg_1);\n gen.Emit(OpCodes.Isinst, tb);\n gen.Emit(OpCodes.Stloc, other);\n gen.Emit(OpCodes.Ldloc, other);\n gen.Emit(OpCodes.Brtrue_S, next);\n gen.Emit(OpCodes.Ldc_I4_0);\n gen.Emit(OpCodes.Ret);\n gen.MarkLabel(next);\n foreach (FieldInfo field in fields)\n {\n Type ft = field.FieldType;\n Type ct = typeof(EqualityComparer<>).MakeGenericType(ft);\n next = gen.DefineLabel();\n gen.EmitCall(OpCodes.Call, ct.GetMethod(\"get_Default\"), null);\n gen.Emit(OpCodes.Ldarg_0);\n gen.Emit(OpCodes.Ldfld, field);\n gen.Emit(OpCodes.Ldloc, other);\n gen.Emit(OpCodes.Ldfld, field);\n gen.EmitCall(OpCodes.Callvirt, ct.GetMethod(\"Equals\", new Type[] { ft, ft }), null);\n gen.Emit(OpCodes.Brtrue_S, next);\n gen.Emit(OpCodes.Ldc_I4_0);\n gen.Emit(OpCodes.Ret);\n gen.MarkLabel(next);\n }\n gen.Emit(OpCodes.Ldc_I4_1);\n gen.Emit(OpCodes.Ret);\n }\n\n void GenerateGetHashCode(TypeBuilder tb, FieldInfo[] fields)\n {\n MethodBuilder mb = tb.DefineMethod(\"GetHashCode\",\n MethodAttributes.Public | MethodAttributes.ReuseSlot |\n MethodAttributes.Virtual | MethodAttributes.HideBySig,\n typeof(int), Type.EmptyTypes);\n ILGenerator gen = mb.GetILGenerator();\n gen.Emit(OpCodes.Ldc_I4_0);\n foreach (FieldInfo field in fields)\n {\n Type ft = field.FieldType;\n Type ct = typeof(EqualityComparer<>).MakeGenericType(ft);\n gen.EmitCall(OpCodes.Call, ct.GetMethod(\"get_Default\"), null);\n gen.Emit(OpCodes.Ldarg_0);\n gen.Emit(OpCodes.Ldfld, field);\n gen.EmitCall(OpCodes.Callvirt, ct.GetMethod(\"GetHashCode\", new Type[] { ft }), null);\n gen.Emit(OpCodes.Xor);\n }\n gen.Emit(OpCodes.Ret);\n }\n }\n\n internal sealed class ParseException : Exception\n {\n int position;\n\n public ParseException(string message, int position)\n : base(message)\n {\n this.position = position;\n }\n\n public int Position\n {\n get { return position; }\n }\n\n public override string ToString()\n {\n return string.Format(Res.ParseExceptionFormat, Message, position);\n }\n }\n#endregion\n\n internal class ExpressionParser\n {\n#region [ internal ]\n struct Token\n {\n public TokenId id;\n public string text;\n public int pos;\n }\n\n enum TokenId\n {\n Unknown,\n End,\n Identifier,\n StringLiteral,\n IntegerLiteral,\n RealLiteral,\n Exclamation,\n Percent,\n Amphersand,\n OpenParen,\n CloseParen,\n Asterisk,\n Plus,\n Comma,\n Minus,\n Dot,\n Slash,\n Colon,\n LessThan,\n Equal,\n GreaterThan,\n Question,\n OpenBracket,\n CloseBracket,\n Bar,\n ExclamationEqual,\n DoubleAmphersand,\n LessThanEqual,\n LessGreater,\n DoubleEqual,\n GreaterThanEqual,\n DoubleBar\n }\n\n interface ILogicalSignatures\n {\n void F(bool x, bool y);\n void F(bool? x, bool? y);\n }\n\n interface IArithmeticSignatures\n {\n void F(int x, int y);\n void F(uint x, uint y);\n void F(long x, long y);\n void F(ulong x, ulong y);\n void F(float x, float y);\n void F(double x, double y);\n void F(decimal x, decimal y);\n void F(int? x, int? y);\n void F(uint? x, uint? y);\n void F(long? x, long? y);\n void F(ulong? x, ulong? y);\n void F(float? x, float? y);\n void F(double? x, double? y);\n void F(decimal? x, decimal? y);\n }\n\n interface IRelationalSignatures : IArithmeticSignatures\n {\n void F(string x, string y);\n void F(char x, char y);\n void F(DateTime x, DateTime y);\n void F(TimeSpan x, TimeSpan y);\n void F(char? x, char? y);\n void F(DateTime? x, DateTime? y);\n void F(TimeSpan? x, TimeSpan? y);\n }\n\n interface IEqualitySignatures : IRelationalSignatures\n {\n void F(bool x, bool y);\n void F(bool? x, bool? y);\n }\n\n interface IAddSignatures : IArithmeticSignatures\n {\n void F(DateTime x, TimeSpan y);\n void F(TimeSpan x, TimeSpan y);\n void F(DateTime? x, TimeSpan? y);\n void F(TimeSpan? x, TimeSpan? y);\n }\n\n interface ISubtractSignatures : IAddSignatures\n {\n void F(DateTime x, DateTime y);\n void F(DateTime? x, DateTime? y);\n }\n\n interface INegationSignatures\n {\n void F(int x);\n void F(long x);\n void F(float x);\n void F(double x);\n void F(decimal x);\n void F(int? x);\n void F(long? x);\n void F(float? x);\n void F(double? x);\n void F(decimal? x);\n }\n\n interface INotSignatures\n {\n void F(bool x);\n void F(bool? x);\n }\n\n interface IEnumerableSignatures\n {\n void Where(bool predicate);\n void Any();\n void Any(bool predicate);\n void All(bool predicate);\n void Count();\n void Count(bool predicate);\n void Min(object selector);\n void Max(object selector);\n void Sum(int selector);\n void Sum(int? selector);\n void Sum(long selector);\n void Sum(long? selector);\n void Sum(float selector);\n void Sum(float? selector);\n void Sum(double selector);\n void Sum(double? selector);\n void Sum(decimal selector);\n void Sum(decimal? selector);\n void Average(int selector);\n void Average(int? selector);\n void Average(long selector);\n void Average(long? selector);\n void Average(float selector);\n void Average(float? selector);\n void Average(double selector);\n void Average(double? selector);\n void Average(decimal selector);\n void Average(decimal? selector);\n }\n\n static readonly Type[] predefinedTypes = {\n typeof(Object),\n typeof(Boolean),\n typeof(Char),\n typeof(String),\n typeof(SByte),\n typeof(Byte),\n typeof(Int16),\n typeof(UInt16),\n typeof(Int32),\n typeof(UInt32),\n typeof(Int64),\n typeof(UInt64),\n typeof(Single),\n typeof(Double),\n typeof(Decimal),\n typeof(DateTime),\n typeof(TimeSpan),\n typeof(Guid),\n typeof(Math),\n typeof(Convert)\n };\n#endregion\n\n#region [ gunk ]\n\n\n //// *, /, %, mod operators\n //Expression ParseMultiplicative()\n //{\n // Expression left = ParseUnary();\n // //while (token.id == TokenId.Asterisk || token.id == TokenId.Slash ||\n // // token.id == TokenId.Percent || TokenIdentifierIs(\"mod\"))\n // //{\n // // Token op = token;\n // // NextToken();\n // // Expression right = ParseUnary();\n // // CheckAndPromoteOperands(typeof(IArithmeticSignatures), op.text, ref left, ref right, op.pos);\n // // switch (op.id)\n // // {\n // // case TokenId.Asterisk:\n // // left = Expression.Multiply(left, right);\n // // break;\n // // case TokenId.Slash:\n // // left = Expression.Divide(left, right);\n // // break;\n // // case TokenId.Percent:\n // // case TokenId.Identifier:\n // // left = Expression.Modulo(left, right);\n // // break;\n // // }\n // //}\n // return left;\n //}\n\n //Expression GenerateAdd(Expression left, Expression right)\n //{\n // if (left.Type == typeof(string) && right.Type == typeof(string))\n // {\n // return GenerateStaticMethodCall(\"Concat\", left, right);\n // }\n // return Expression.Add(left, right);\n //}\n\n //Expression GenerateSubtract(Expression left, Expression right)\n //{\n // return Expression.Subtract(left, right);\n //}\n\n //Expression GenerateStringConcat(Expression left, Expression right)\n //{\n // return Expression.Call(\n // null,\n // typeof(string).GetMethod(\"Concat\", new[] { typeof(object), typeof(object) }),\n // new[] { left, right });\n //}\n\n#pragma warning disable 0219\n //public IEnumerable ParseOrdering()\n //{\n // List orderings = new List();\n // while (true)\n // {\n // Expression expr = ParseExpression();\n // bool ascending = true;\n // if (TokenIdentifierIs(\"asc\") || TokenIdentifierIs(\"ascending\"))\n // {\n // NextToken();\n // }\n // else if (TokenIdentifierIs(\"desc\") || TokenIdentifierIs(\"descending\"))\n // {\n // NextToken();\n // ascending = false;\n // }\n // orderings.Add(new DynamicOrdering { Selector = expr, Ascending = ascending });\n // if (token.id != TokenId.Comma) break;\n // NextToken();\n // }\n // ValidateToken(TokenId.End, Res.SyntaxError);\n // return orderings;\n //}\n#pragma warning restore 0219\n\n //Expression ParseIt()\n //{\n // if (it == null)\n // throw ParseError(Res.NoItInScope);\n // NextToken();\n // return it;\n //}\n\n //Expression ParseIif()\n //{\n // int errorPos = token.pos;\n // NextToken();\n // Expression[] args = ParseArgumentList();\n // if (args.Length != 3)\n // throw ParseError(errorPos, Res.IifRequiresThreeArgs);\n // return GenerateConditional(args[0], args[1], args[2], errorPos);\n //}\n\n //Expression GenerateConditional(Expression test, Expression expr1, Expression expr2, int errorPos)\n //{\n // if (test.Type != typeof(bool))\n // throw ParseError(errorPos, Res.FirstExprMustBeBool);\n // if (expr1.Type != expr2.Type)\n // {\n // Expression expr1as2 = expr2 != nullLiteral ? PromoteExpression(expr1, expr2.Type, true) : null;\n // Expression expr2as1 = expr1 != nullLiteral ? PromoteExpression(expr2, expr1.Type, true) : null;\n // if (expr1as2 != null && expr2as1 == null)\n // {\n // expr1 = expr1as2;\n // }\n // else if (expr2as1 != null && expr1as2 == null)\n // {\n // expr2 = expr2as1;\n // }\n // else\n // {\n // string type1 = expr1 != nullLiteral ? expr1.Type.Name : \"null\";\n // string type2 = expr2 != nullLiteral ? expr2.Type.Name : \"null\";\n // if (expr1as2 != null && expr2as1 != null)\n // throw ParseError(errorPos, Res.BothTypesConvertToOther, type1, type2);\n // throw ParseError(errorPos, Res.NeitherTypeConvertsToOther, type1, type2);\n // }\n // }\n // return Expression.Condition(test, expr1, expr2);\n //}\n\n //Expression ParseNew()\n //{\n // NextToken();\n // ValidateToken(TokenId.OpenParen, Res.OpenParenExpected);\n // NextToken();\n // List properties = new List();\n // List expressions = new List();\n // while (true)\n // {\n // int exprPos = token.pos;\n // Expression expr = ParseExpression();\n // string propName;\n // if (TokenIdentifierIs(\"as\"))\n // {\n // NextToken();\n // propName = GetIdentifier();\n // NextToken();\n // }\n // else\n // {\n // MemberExpression me = expr as MemberExpression;\n // if (me == null) throw ParseError(exprPos, Res.MissingAsClause);\n // propName = me.Member.Name;\n // }\n // expressions.Add(expr);\n // properties.Add(new DynamicProperty(propName, expr.Type));\n // if (token.id != TokenId.Comma) break;\n // NextToken();\n // }\n // ValidateToken(TokenId.CloseParen, Res.CloseParenOrCommaExpected);\n // NextToken();\n // Type type = DynamicExpression.CreateClass(properties);\n // MemberBinding[] bindings = new MemberBinding[properties.Count];\n // for (int i = 0; i < bindings.Length; i++)\n // bindings[i] = Expression.Bind(type.GetProperty(properties[i].Name), expressions[i]);\n // return Expression.MemberInit(Expression.New(type), bindings);\n //}\n\n //Expression ParseLambdaInvocation(LambdaExpression lambda)\n //{\n // int errorPos = token.pos;\n // NextToken();\n // Expression[] args = ParseArgumentList();\n // MethodBase method;\n // if (FindMethod(lambda.Type, \"Invoke\", false, args, out method) != 1)\n // throw ParseError(errorPos, Res.ArgsIncompatibleWithLambda);\n // return Expression.Invoke(lambda, args);\n //}\n\n\n //Expression GenerateConversion(Expression expr, Type type, int errorPos)\n //{\n // Type exprType = expr.Type;\n // if (exprType == type) return expr;\n // if (exprType.IsValueType && type.IsValueType)\n // {\n // if ((IsNullableType(exprType) || IsNullableType(type)) &&\n // GetNonNullableType(exprType) == GetNonNullableType(type))\n // return Expression.Convert(expr, type);\n // if ((IsNumericType(exprType) || IsEnumType(exprType)) &&\n // (IsNumericType(type)) || IsEnumType(type))\n // return Expression.ConvertChecked(expr, type);\n // }\n // if (exprType.IsAssignableFrom(type) || type.IsAssignableFrom(exprType) ||\n // exprType.IsInterface || type.IsInterface)\n // return Expression.Convert(expr, type);\n // throw ParseError(errorPos, Res.CannotConvertValue,\n // GetTypeName(exprType), GetTypeName(type));\n //}\n\n\n //static Type FindGenericType(Type generic, Type type)\n //{\n // while (type != null && type != typeof(object))\n // {\n // if (type.IsGenericType && type.GetGenericTypeDefinition() == generic) return type;\n // if (generic.IsInterface)\n // {\n // foreach (Type intfType in type.GetInterfaces())\n // {\n // Type found = FindGenericType(generic, intfType);\n // if (found != null) return found;\n // }\n // }\n // type = type.BaseType;\n // }\n // return null;\n //}\n\n //Expression ParseAggregate(Expression instance, Type elementType, string methodName, int errorPos)\n //{\n // ParameterExpression outerIt = it;\n // ParameterExpression innerIt = Expression.Parameter(elementType, \"\");\n // it = innerIt;\n // Expression[] args = ParseArgumentList();\n // it = outerIt;\n // MethodBase signature;\n // if (FindMethod(typeof(IEnumerableSignatures), methodName, false, args, out signature) != 1)\n // throw ParseError(errorPos, Res.NoApplicableAggregate, methodName);\n // Type[] typeArgs;\n // if (signature.Name == \"Min\" || signature.Name == \"Max\")\n // {\n // typeArgs = new Type[] { elementType, args[0].Type };\n // }\n // else\n // {\n // typeArgs = new Type[] { elementType };\n // }\n // if (args.Length == 0)\n // {\n // args = new Expression[] { instance };\n // }\n // else\n // {\n // args = new Expression[] { instance, Expression.Lambda(args[0], innerIt) };\n // }\n // return Expression.Call(typeof(Enumerable), signature.Name, typeArgs, args);\n //}\n\n Expression[] ParseArgumentList()\n {\n ValidateToken(TokenId.OpenParen, Res.OpenParenExpected);\n NextToken();\n Expression[] args = token.id != TokenId.CloseParen ? ParseArguments() : new Expression[0];\n ValidateToken(TokenId.CloseParen, Res.CloseParenOrCommaExpected);\n NextToken();\n return args;\n }\n\n Expression[] ParseArguments()\n {\n List argList = new List();\n while (true)\n {\n argList.Add(ParseExpression());\n if (token.id != TokenId.Comma) break;\n NextToken();\n }\n return argList.ToArray();\n }\n\n //Expression ParseElementAccess(Expression expr)\n //{\n // int errorPos = token.pos;\n // ValidateToken(TokenId.OpenBracket, Res.OpenParenExpected);\n // NextToken();\n // Expression[] args = ParseArguments();\n // ValidateToken(TokenId.CloseBracket, Res.CloseBracketOrCommaExpected);\n // NextToken();\n // if (expr.Type.IsArray)\n // {\n // if (expr.Type.GetArrayRank() != 1 || args.Length != 1)\n // throw ParseError(errorPos, Res.CannotIndexMultiDimArray);\n // Expression index = PromoteExpression(args[0], typeof(int), true);\n // if (index == null)\n // throw ParseError(errorPos, Res.InvalidIndex);\n // return Expression.ArrayIndex(expr, index);\n // }\n // else\n // {\n // MethodBase mb;\n // switch (FindIndexer(expr.Type, args, out mb))\n // {\n // case 0:\n // throw ParseError(errorPos, Res.NoApplicableIndexer,\n // GetTypeName(expr.Type));\n // case 1:\n // return Expression.Call(expr, (MethodInfo)mb, args);\n // default:\n // throw ParseError(errorPos, Res.AmbiguousIndexerInvocation,\n // GetTypeName(expr.Type));\n // }\n // }\n //}\n\n //static bool IsPredefinedType(Type type)\n //{\n // foreach (Type t in predefinedTypes) if (t == type) return true;\n // return false;\n //}\n\n //static bool IsNumericType(Type type)\n //{\n // return GetNumericTypeKind(type) != 0;\n //}\n\n //static bool IsEnumType(Type type)\n //{\n // return GetNonNullableType(type).IsEnum;\n //}\n\n //static readonly string keywordIt = \"it\";\n //static readonly string keywordIif = \"iif\";\n //static readonly string keywordNew = \"new\";\n\n#endregion\n\n static readonly Expression trueLiteral = Expression.Constant(true);\n static readonly Expression falseLiteral = Expression.Constant(false);\n static readonly Expression nullLiteral = Expression.Constant(null);\n\n static Dictionary keywords;\n\n Dictionary symbols;\n IDictionary externals;\n Dictionary literals;\n ParameterExpression it;\n string text;\n int textPos;\n int textLen;\n char ch;\n Token token;\n\n public ExpressionParser(ParameterExpression[] parameters, string expression, object[] values)\n {\n if (expression == null) throw new ArgumentNullException(\"expression\");\n if (keywords == null) keywords = CreateKeywords();\n symbols = new Dictionary(StringComparer.OrdinalIgnoreCase);\n literals = new Dictionary();\n if (parameters != null) ProcessParameters(parameters);\n if (values != null) ProcessValues(values);\n text = expression;\n textLen = text.Length;\n SetTextPos(0);\n NextToken();\n }\n\n void ProcessParameters(ParameterExpression[] parameters)\n {\n foreach (ParameterExpression pe in parameters)\n if (!String.IsNullOrEmpty(pe.Name))\n AddSymbol(pe.Name, pe);\n if (parameters.Length == 1 && String.IsNullOrEmpty(parameters[0].Name))\n it = parameters[0];\n }\n\n void ProcessValues(object[] values)\n {\n for (int i = 0; i < values.Length; i++)\n {\n object value = values[i];\n if (i == values.Length - 1 && value is IDictionary)\n {\n externals = (IDictionary)value;\n }\n else\n {\n AddSymbol(\"@\" + i.ToString(System.Globalization.CultureInfo.InvariantCulture), value);\n }\n }\n }\n\n void AddSymbol(string name, object value)\n {\n if (symbols.ContainsKey(name))\n throw ParseError(Res.DuplicateIdentifier, name);\n symbols.Add(name, value);\n }\n\n public Expression Parse(Type resultType)\n {\n int exprPos = token.pos;\n Expression expr = ParseExpression();\n if (resultType != null)\n if ((expr = PromoteExpression(expr, resultType, true)) == null)\n throw ParseError(exprPos, Res.ExpressionTypeMismatch, GetTypeName(resultType));\n ValidateToken(TokenId.End, Res.SyntaxError);\n return expr;\n }\n\n\n // ?: operator\n Expression ParseExpression()\n {\n int errorPos = token.pos;\n Expression expr = ParseLogicalOr();\n //if (token.id == TokenId.Question)\n //{\n // NextToken();\n // Expression expr1 = ParseExpression();\n // ValidateToken(TokenId.Colon, Res.ColonExpected);\n // NextToken();\n // Expression expr2 = ParseExpression();\n // expr = GenerateConditional(expr, expr1, expr2, errorPos);\n //}\n return expr;\n }\n\n // ||, or operator\n Expression ParseLogicalOr()\n {\n Expression left = ParseLogicalAnd();\n while (token.id == TokenId.DoubleBar || TokenIdentifierIs(\"or\"))\n {\n Token op = token;\n NextToken();\n Expression right = ParseLogicalAnd();\n CheckAndPromoteOperands(typeof(ILogicalSignatures), op.text, ref left, ref right, op.pos);\n left = Expression.OrElse(left, right);\n }\n return left;\n }\n\n // &&, and operator\n Expression ParseLogicalAnd()\n {\n Expression left = ParseComparison();\n while (token.id == TokenId.DoubleAmphersand || TokenIdentifierIs(\"and\"))\n {\n Token op = token;\n NextToken();\n Expression right = ParseComparison();\n CheckAndPromoteOperands(typeof(ILogicalSignatures), op.text, ref left, ref right, op.pos);\n left = Expression.AndAlso(left, right);\n }\n return left;\n }\n\n // =, ==, !=, <>, >, >=, <, <= operators\n Expression ParseComparison()\n {\n Expression left = ParseAdditive();\n while (token.id == TokenId.Equal || token.id == TokenId.DoubleEqual ||\n token.id == TokenId.ExclamationEqual || token.id == TokenId.LessGreater ||\n token.id == TokenId.GreaterThan || token.id == TokenId.GreaterThanEqual ||\n token.id == TokenId.LessThan || token.id == TokenId.LessThanEqual)\n {\n Token op = token;\n NextToken();\n Expression right = ParseAdditive();\n bool isEquality = op.id == TokenId.Equal || op.id == TokenId.DoubleEqual ||\n op.id == TokenId.ExclamationEqual || op.id == TokenId.LessGreater;\n if (isEquality && !left.Type.IsValueType && !right.Type.IsValueType)\n {\n if (left.Type != right.Type)\n {\n if (left.Type.IsAssignableFrom(right.Type))\n {\n right = Expression.Convert(right, left.Type);\n }\n else if (right.Type.IsAssignableFrom(left.Type))\n {\n left = Expression.Convert(left, right.Type);\n }\n else\n {\n throw IncompatibleOperandsError(op.text, left, right, op.pos);\n }\n }\n }\n //else if (IsEnumType(left.Type) || IsEnumType(right.Type))\n //{\n // if (left.Type != right.Type)\n // {\n // Expression e;\n // if ((e = PromoteExpression(right, left.Type, true)) != null)\n // {\n // right = e;\n // }\n // else if ((e = PromoteExpression(left, right.Type, true)) != null)\n // {\n // left = e;\n // }\n // else\n // {\n // throw IncompatibleOperandsError(op.text, left, right, op.pos);\n // }\n // }\n //}\n else\n {\n if (left.Type == typeof(Guid))\n right = Expression.Constant(Guid.Parse(right.ToString().Replace(\"\\\"\", \"\").Replace(\"'\", \"\")));\n else if (left.Type == typeof(DateTime))\n right = Expression.Constant(DateTime.Parse(right.ToString().Replace(\"\\\"\", \"\").Replace(\"'\", \"\")));\n else\n CheckAndPromoteOperands(isEquality ? typeof(IEqualitySignatures) : typeof(IRelationalSignatures),\n op.text, ref left, ref right, op.pos);\n }\n switch (op.id)\n {\n case TokenId.Equal:\n case TokenId.DoubleEqual:\n left = GenerateEqual(left, right);\n break;\n case TokenId.ExclamationEqual:\n case TokenId.LessGreater:\n left = GenerateNotEqual(left, right);\n break;\n case TokenId.GreaterThan:\n left = GenerateGreaterThan(left, right);\n break;\n case TokenId.GreaterThanEqual:\n left = GenerateGreaterThanEqual(left, right);\n break;\n case TokenId.LessThan:\n left = GenerateLessThan(left, right);\n break;\n case TokenId.LessThanEqual:\n left = GenerateLessThanEqual(left, right);\n break;\n }\n }\n return left;\n }\n\n // +, -, & operators\n Expression ParseAdditive()\n {\n Expression left = ParseUnary();// ParseMultiplicative();\n //while (token.id == TokenId.Plus || token.id == TokenId.Minus ||\n // token.id == TokenId.Amphersand)\n //{\n // Token op = token;\n // NextToken();\n // Expression right = ParseMultiplicative();\n // switch (op.id)\n // {\n // case TokenId.Plus:\n // if (left.Type == typeof(string) || right.Type == typeof(string))\n // goto case TokenId.Amphersand;\n // CheckAndPromoteOperands(typeof(IAddSignatures), op.text, ref left, ref right, op.pos);\n // left = GenerateAdd(left, right);\n // break;\n // case TokenId.Minus:\n // CheckAndPromoteOperands(typeof(ISubtractSignatures), op.text, ref left, ref right, op.pos);\n // left = GenerateSubtract(left, right);\n // break;\n // case TokenId.Amphersand:\n // left = GenerateStringConcat(left, right);\n // break;\n // }\n //}\n return left;\n }\n\n // -, !, not unary operators\n Expression ParseUnary()\n {\n if (token.id == TokenId.Minus || token.id == TokenId.Exclamation ||\n TokenIdentifierIs(\"not\"))\n {\n Token op = token;\n NextToken();\n if (op.id == TokenId.Minus && (token.id == TokenId.IntegerLiteral ||\n token.id == TokenId.RealLiteral))\n {\n token.text = \"-\" + token.text;\n token.pos = op.pos;\n return ParsePrimary();\n }\n Expression expr = ParseUnary();\n if (op.id == TokenId.Minus)\n {\n CheckAndPromoteOperand(typeof(INegationSignatures), op.text, ref expr, op.pos);\n expr = Expression.Negate(expr);\n }\n else\n {\n CheckAndPromoteOperand(typeof(INotSignatures), op.text, ref expr, op.pos);\n expr = Expression.Not(expr);\n }\n return expr;\n }\n return ParsePrimary();\n }\n\n Expression ParsePrimary()\n {\n Expression expr = ParsePrimaryStart();\n while (true)\n {\n if (token.id == TokenId.Dot)\n {\n NextToken();\n expr = ParseMemberAccess(null, expr);\n }\n //else if (token.id == TokenId.OpenBracket)\n //{\n // expr = ParseElementAccess(expr);\n //}\n else\n {\n break;\n }\n }\n return expr;\n }\n\n Expression ParsePrimaryStart()\n {\n switch (token.id)\n {\n case TokenId.Identifier:\n return ParseIdentifier();\n case TokenId.StringLiteral:\n return ParseStringLiteral();\n case TokenId.IntegerLiteral:\n return ParseIntegerLiteral();\n case TokenId.RealLiteral:\n return ParseRealLiteral();\n case TokenId.OpenParen:\n return ParseParenExpression();\n default:\n throw ParseError(Res.ExpressionExpected);\n }\n }\n\n Expression ParseStringLiteral()\n {\n ValidateToken(TokenId.StringLiteral);\n char quote = token.text[0];\n string s = token.text.Substring(1, token.text.Length - 2);\n int start = 0;\n while (true)\n {\n int i = s.IndexOf(quote, start);\n if (i < 0) break;\n s = s.Remove(i, 1);\n start = i + 1;\n }\n if (quote == '\\'')\n {\n if (s.Length != 1)\n throw ParseError(Res.InvalidCharacterLiteral);\n NextToken();\n return CreateLiteral(s[0], s);\n }\n NextToken();\n return CreateLiteral(s, s);\n }\n\n Expression ParseIntegerLiteral()\n {\n ValidateToken(TokenId.IntegerLiteral);\n string text = token.text;\n if (text[0] != '-')\n {\n ulong value;\n if (!UInt64.TryParse(text, out value))\n throw ParseError(Res.InvalidIntegerLiteral, text);\n NextToken();\n if (value <= (ulong)Int32.MaxValue) return CreateLiteral((int)value, text);\n if (value <= (ulong)UInt32.MaxValue) return CreateLiteral((uint)value, text);\n if (value <= (ulong)Int64.MaxValue) return CreateLiteral((long)value, text);\n return CreateLiteral(value, text);\n }\n else\n {\n long value;\n if (!Int64.TryParse(text, out value))\n throw ParseError(Res.InvalidIntegerLiteral, text);\n NextToken();\n if (value >= Int32.MinValue && value <= Int32.MaxValue)\n return CreateLiteral((int)value, text);\n return CreateLiteral(value, text);\n }\n }\n\n Expression ParseRealLiteral()\n {\n ValidateToken(TokenId.RealLiteral);\n string text = token.text;\n object value = null;\n char last = text[text.Length - 1];\n if (last == 'F' || last == 'f')\n {\n float f;\n if (Single.TryParse(text.Substring(0, text.Length - 1), out f)) value = f;\n }\n else\n {\n double d;\n if (Double.TryParse(text, out d)) value = d;\n }\n if (value == null) throw ParseError(Res.InvalidRealLiteral, text);\n NextToken();\n return CreateLiteral(value, text);\n }\n\n Expression CreateLiteral(object value, string text)\n {\n ConstantExpression expr = Expression.Constant(value);\n literals.Add(expr, text);\n return expr;\n }\n\n Expression ParseParenExpression()\n {\n ValidateToken(TokenId.OpenParen, Res.OpenParenExpected);\n NextToken();\n Expression e = ParseExpression();\n ValidateToken(TokenId.CloseParen, Res.CloseParenOrOperatorExpected);\n NextToken();\n return e;\n }\n\n Expression ParseIdentifier()\n {\n ValidateToken(TokenId.Identifier);\n object value;\n if (keywords.TryGetValue(token.text, out value))\n {\n if (value is Type) return ParseTypeAccess((Type)value);\n //if (value == (object)keywordIt) return ParseIt();\n //if (value == (object)keywordIif) return ParseIif();\n //if (value == (object)keywordNew) return ParseNew();\n NextToken();\n return (Expression)value;\n }\n if (symbols.TryGetValue(token.text, out value) ||\n externals != null && externals.TryGetValue(token.text, out value))\n {\n Expression expr = value as Expression;\n if (expr == null)\n {\n expr = Expression.Constant(value);\n }\n //else\n //{\n // LambdaExpression lambda = expr as LambdaExpression;\n // if (lambda != null) return ParseLambdaInvocation(lambda);\n //}\n NextToken();\n return expr;\n }\n if (it != null) return ParseMemberAccess(null, it);\n throw ParseError(Res.UnknownIdentifier, token.text);\n }\n\n Expression ParseTypeAccess(Type type)\n {\n int errorPos = token.pos;\n NextToken();\n if (token.id == TokenId.Question)\n {\n if (!type.IsValueType || IsNullableType(type))\n throw ParseError(errorPos, Res.TypeHasNoNullableForm, GetTypeName(type));\n type = typeof(Nullable<>).MakeGenericType(type);\n NextToken();\n }\n //if (token.id == TokenId.OpenParen)\n //{\n // Expression[] args = ParseArgumentList();\n // MethodBase method;\n // switch (FindBestMethod(type.GetConstructors(), args, out method))\n // {\n // case 0:\n // if (args.Length == 1)\n // return GenerateConversion(args[0], type, errorPos);\n // throw ParseError(errorPos, Res.NoMatchingConstructor, GetTypeName(type));\n // case 1:\n // return Expression.New((ConstructorInfo)method, args);\n // default:\n // throw ParseError(errorPos, Res.AmbiguousConstructorInvocation, GetTypeName(type));\n // }\n //}\n ValidateToken(TokenId.Dot, Res.DotOrOpenParenExpected);\n NextToken();\n return ParseMemberAccess(type, null);\n }\n\n internal static Dictionary _betweens = new Dictionary();\n Expression CallBetween(Expression instance, Type checktype, Type datatype, Expression[] args)\n {\n MethodInfo method = null;\n _betweens.TryGetValue(datatype, out method);\n\n if (method == null)\n foreach (var m in typeof(RDBExtension).GetMethods())\n {\n if (m.Name.ToLower() == \"between\")\n {\n var pi = m.GetParameters();\n if (pi[1].ParameterType == checktype)\n {\n method = m;\n _betweens.Add(datatype, method);\n break;\n }\n else if (pi[1].ParameterType.IsGenericParameter)\n {\n method = m.MakeGenericMethod(args[0].Type);\n _betweens.Add(datatype, method);\n break;\n }\n }\n }\n var pname = instance.ToString();\n return Expression.Call(method, new Expression[] { ConstantExpression.Variable(datatype, pname), args[0], args[1] });\n }\n\n internal static Dictionary _ins = new Dictionary();\n Expression CallIn(Expression instance, Type checktype, Type datatype, Expression[] args)\n {\n MethodInfo method = null;\n _ins.TryGetValue(datatype, out method);\n\n if (method == null)\n foreach (var m in typeof(RDBExtension).GetMethods())\n {\n if (m.Name.ToLower() == \"in\")\n {\n var pi = m.GetParameters();\n if (pi[0].ParameterType.IsGenericParameter)\n {\n method = m.MakeGenericMethod(datatype);\n _ins.Add(datatype, method);\n break;\n }\n }\n }\n var pname = instance.ToString();\n // calling on byte column status.in(1,3) 1,3 = int32 not byte\n if(datatype != args[0].Type)\n {\n for (int i = 0; i < args.Length; i++)\n {\n args[i] = ConstantExpression.Constant( Convert.ChangeType(\n (args[i] as ConstantExpression).Value, datatype));\n }\n }\n return Expression.Call(method, new Expression[] { ConstantExpression.Variable(datatype, pname), Expression.NewArrayInit(datatype, args) });\n }\n\n Expression ParseMemberAccess(Type type, Expression instance)\n {\n if (instance != null) type = instance.Type;\n int errorPos = token.pos;\n string id = GetIdentifier();\n NextToken();\n#region commented\n //if (token.id == TokenId.OpenParen)\n //{\n // //if (instance != null && type != typeof(string))\n // //{\n // // Type enumerableType = FindGenericType(typeof(IEnumerable<>), type);\n // // if (enumerableType != null)\n // // {\n // // Type elementType = enumerableType.GetGenericArguments()[0];\n // // return ParseAggregate(instance, elementType, id, errorPos);\n // // }\n // //}\n // Expression[] args = ParseArgumentList();\n // MethodBase mb;\n // switch (FindMethod(type, id, instance == null, args, out mb))\n // {\n // case 0:\n // throw ParseError(errorPos, Res.NoApplicableMethod,\n // id, GetTypeName(type));\n // case 1:\n // MethodInfo method = (MethodInfo)mb;\n // if (!IsPredefinedType(method.DeclaringType))\n // throw ParseError(errorPos, Res.MethodsAreInaccessible, GetTypeName(method.DeclaringType));\n // if (method.ReturnType == typeof(void))\n // throw ParseError(errorPos, Res.MethodIsVoid,\n // id, GetTypeName(method.DeclaringType));\n // return Expression.Call(instance, (MethodInfo)method, args);\n // default:\n // throw ParseError(errorPos, Res.AmbiguousMethodInvocation,\n // id, GetTypeName(type));\n // }\n //}\n //else \n#endregion\n {\n MemberInfo member = FindPropertyOrField(type, id, instance == null);\n if (member == null)\n {\n if (id.ToLower() == \"between\") \n {\n Expression[] args = ParseArgumentList();\n if (type == typeof(DateTime))\n return CallBetween(instance, typeof(string), typeof(DateTime), args);\n else\n return CallBetween(instance, type, type, args);\n }\n else if(id.ToLower() == \"in\") // in(1,3,5,7)\n {\n var args = ParseArgumentList();\n int i = args.Length;\n return CallIn(instance, type, type, args);\n }\n else\n throw ParseError(errorPos, Res.UnknownPropertyOrField,\n id, GetTypeName(type));\n }\n return member is PropertyInfo ?\n Expression.Property(instance, (PropertyInfo)member) :\n Expression.Field(instance, (FieldInfo)member);\n }\n }\n\n static bool IsNullableType(Type type)\n {\n return type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Nullable<>);\n }\n\n static Type GetNonNullableType(Type type)\n {\n return IsNullableType(type) ? type.GetGenericArguments()[0] : type;\n }\n\n static string GetTypeName(Type type)\n {\n Type baseType = GetNonNullableType(type);\n string s = baseType.Name;\n if (type != baseType) s += '?';\n return s;\n }\n\n static bool IsSignedIntegralType(Type type)\n {\n return GetNumericTypeKind(type) == 2;\n }\n\n static bool IsUnsignedIntegralType(Type type)\n {\n return GetNumericTypeKind(type) == 3;\n }\n\n static int GetNumericTypeKind(Type type)\n {\n type = GetNonNullableType(type);\n if (type.IsEnum) return 0;\n switch (Type.GetTypeCode(type))\n {\n case TypeCode.Char:\n case TypeCode.Single:\n case TypeCode.Double:\n case TypeCode.Decimal:\n return 1;\n case TypeCode.SByte:\n case TypeCode.Int16:\n case TypeCode.Int32:\n case TypeCode.Int64:\n return 2;\n case TypeCode.Byte:\n case TypeCode.UInt16:\n case TypeCode.UInt32:\n case TypeCode.UInt64:\n return 3;\n default:\n return 0;\n }\n }\n\n void CheckAndPromoteOperand(Type signatures, string opName, ref Expression expr, int errorPos)\n {\n Expression[] args = new Expression[] { expr };\n MethodBase method;\n if (FindMethod(signatures, \"F\", false, args, out method) != 1)\n throw ParseError(errorPos, Res.IncompatibleOperand,\n opName, GetTypeName(args[0].Type));\n expr = args[0];\n }\n\n void CheckAndPromoteOperands(Type signatures, string opName, ref Expression left, ref Expression right, int errorPos)\n {\n Expression[] args = new Expression[] { left, right };\n MethodBase method;\n if (FindMethod(signatures, \"F\", false, args, out method) != 1)\n throw IncompatibleOperandsError(opName, left, right, errorPos);\n left = args[0];\n right = args[1];\n }\n\n Exception IncompatibleOperandsError(string opName, Expression left, Expression right, int pos)\n {\n return ParseError(pos, Res.IncompatibleOperands,\n opName, GetTypeName(left.Type), GetTypeName(right.Type));\n }\n\n MemberInfo FindPropertyOrField(Type type, string memberName, bool staticAccess)\n {\n BindingFlags flags = BindingFlags.Public | BindingFlags.DeclaredOnly |\n (staticAccess ? BindingFlags.Static : BindingFlags.Instance);\n foreach (Type t in SelfAndBaseTypes(type))\n {\n MemberInfo[] members = t.FindMembers(MemberTypes.Property | MemberTypes.Field,\n flags, Type.FilterNameIgnoreCase, memberName);\n if (members.Length != 0) return members[0];\n }\n return null;\n }\n\n int FindMethod(Type type, string methodName, bool staticAccess, Expression[] args, out MethodBase method)\n {\n BindingFlags flags = BindingFlags.Public | BindingFlags.DeclaredOnly |\n (staticAccess ? BindingFlags.Static : BindingFlags.Instance);\n foreach (Type t in SelfAndBaseTypes(type))\n {\n MemberInfo[] members = t.FindMembers(MemberTypes.Method,\n flags, Type.FilterNameIgnoreCase, methodName);\n int count = FindBestMethod(members.Cast(), args, out method);\n if (count != 0) return count;\n }\n method = null;\n return 0;\n }\n\n int FindIndexer(Type type, Expression[] args, out MethodBase method)\n {\n foreach (Type t in SelfAndBaseTypes(type))\n {\n MemberInfo[] members = t.GetDefaultMembers();\n if (members.Length != 0)\n {\n IEnumerable methods = members.\n OfType().\n Select(p => (MethodBase)p.GetGetMethod()).\n Where(m => m != null);\n int count = FindBestMethod(methods, args, out method);\n if (count != 0) return count;\n }\n }\n method = null;\n return 0;\n }\n\n static IEnumerable SelfAndBaseTypes(Type type)\n {\n if (type.IsInterface)\n {\n List types = new List();\n AddInterface(types, type);\n return types;\n }\n return SelfAndBaseClasses(type);\n }\n\n static IEnumerable SelfAndBaseClasses(Type type)\n {\n while (type != null)\n {\n yield return type;\n type = type.BaseType;\n }\n }\n\n static void AddInterface(List types, Type type)\n {\n if (!types.Contains(type))\n {\n types.Add(type);\n foreach (Type t in type.GetInterfaces()) AddInterface(types, t);\n }\n }\n\n class MethodData\n {\n public MethodBase MethodBase;\n public ParameterInfo[] Parameters;\n public Expression[] Args;\n }\n\n int FindBestMethod(IEnumerable methods, Expression[] args, out MethodBase method)\n {\n MethodData[] applicable = methods.\n Select(m => new MethodData { MethodBase = m, Parameters = m.GetParameters() }).\n Where(m => IsApplicable(m, args)).\n ToArray();\n if (applicable.Length > 1)\n {\n applicable = applicable.\n Where(m => applicable.All(n => m == n || IsBetterThan(args, m, n))).\n ToArray();\n }\n if (applicable.Length == 1)\n {\n MethodData md = applicable[0];\n for (int i = 0; i < args.Length; i++) args[i] = md.Args[i];\n method = md.MethodBase;\n }\n else\n {\n method = null;\n }\n return applicable.Length;\n }\n\n bool IsApplicable(MethodData method, Expression[] args)\n {\n if (method.Parameters.Length != args.Length) return false;\n Expression[] promotedArgs = new Expression[args.Length];\n for (int i = 0; i < args.Length; i++)\n {\n ParameterInfo pi = method.Parameters[i];\n if (pi.IsOut) return false;\n Expression promoted = PromoteExpression(args[i], pi.ParameterType, false);\n if (promoted == null) return false;\n promotedArgs[i] = promoted;\n }\n method.Args = promotedArgs;\n return true;\n }\n\n Expression PromoteExpression(Expression expr, Type type, bool exact)\n {\n if (expr.Type == type) return expr;\n if (expr is ConstantExpression)\n {\n ConstantExpression ce = (ConstantExpression)expr;\n if (ce == nullLiteral)\n {\n if (!type.IsValueType || IsNullableType(type))\n return Expression.Constant(null, type);\n }\n else\n {\n string text;\n if (literals.TryGetValue(ce, out text))\n {\n Type target = GetNonNullableType(type);\n Object value = null;\n switch (Type.GetTypeCode(ce.Type))\n {\n case TypeCode.Int32:\n case TypeCode.UInt32:\n case TypeCode.Int64:\n case TypeCode.UInt64:\n value = ParseNumber(text, target);\n break;\n case TypeCode.Double:\n if (target == typeof(decimal)) value = ParseNumber(text, target);\n break;\n case TypeCode.String:\n value = ParseEnum(text, target);\n break;\n }\n if (value != null)\n return Expression.Constant(value, type);\n }\n }\n }\n if (IsCompatibleWith(expr.Type, type))\n {\n if (type.IsValueType || exact) return Expression.Convert(expr, type);\n return expr;\n }\n return null;\n }\n\n static object ParseNumber(string text, Type type)\n {\n switch (Type.GetTypeCode(GetNonNullableType(type)))\n {\n case TypeCode.SByte:\n sbyte sb;\n if (sbyte.TryParse(text, out sb)) return sb;\n break;\n case TypeCode.Byte:\n byte b;\n if (byte.TryParse(text, out b)) return b;\n break;\n case TypeCode.Int16:\n short s;\n if (short.TryParse(text, out s)) return s;\n break;\n case TypeCode.UInt16:\n ushort us;\n if (ushort.TryParse(text, out us)) return us;\n break;\n case TypeCode.Int32:\n int i;\n if (int.TryParse(text, out i)) return i;\n break;\n case TypeCode.UInt32:\n uint ui;\n if (uint.TryParse(text, out ui)) return ui;\n break;\n case TypeCode.Int64:\n long l;\n if (long.TryParse(text, out l)) return l;\n break;\n case TypeCode.UInt64:\n ulong ul;\n if (ulong.TryParse(text, out ul)) return ul;\n break;\n case TypeCode.Single:\n float f;\n if (float.TryParse(text, out f)) return f;\n break;\n case TypeCode.Double:\n double d;\n if (double.TryParse(text, out d)) return d;\n break;\n case TypeCode.Decimal:\n decimal e;\n if (decimal.TryParse(text, out e)) return e;\n break;\n }\n return null;\n }\n\n static object ParseEnum(string name, Type type)\n {\n if (type.IsEnum)\n {\n MemberInfo[] memberInfos = type.FindMembers(MemberTypes.Field,\n BindingFlags.Public | BindingFlags.DeclaredOnly | BindingFlags.Static,\n Type.FilterNameIgnoreCase, name);\n if (memberInfos.Length != 0) return ((FieldInfo)memberInfos[0]).GetValue(null);\n }\n return null;\n }\n\n static bool IsCompatibleWith(Type source, Type target)\n {\n if (source == target) return true;\n if (!target.IsValueType) return target.IsAssignableFrom(source);\n Type st = GetNonNullableType(source);\n Type tt = GetNonNullableType(target);\n if (st != source && tt == target) return false;\n TypeCode sc = st.IsEnum ? TypeCode.Object : Type.GetTypeCode(st);\n TypeCode tc = tt.IsEnum ? TypeCode.Object : Type.GetTypeCode(tt);\n switch (sc)\n {\n case TypeCode.SByte:\n switch (tc)\n {\n case TypeCode.SByte:\n case TypeCode.Int16:\n case TypeCode.Int32:\n case TypeCode.Int64:\n case TypeCode.Single:\n case TypeCode.Double:\n case TypeCode.Decimal:\n return true;\n }\n break;\n case TypeCode.Byte:\n switch (tc)\n {\n case TypeCode.Byte:\n case TypeCode.Int16:\n case TypeCode.UInt16:\n case TypeCode.Int32:\n case TypeCode.UInt32:\n case TypeCode.Int64:\n case TypeCode.UInt64:\n case TypeCode.Single:\n case TypeCode.Double:\n case TypeCode.Decimal:\n return true;\n }\n break;\n case TypeCode.Int16:\n switch (tc)\n {\n case TypeCode.Int16:\n case TypeCode.Int32:\n case TypeCode.Int64:\n case TypeCode.Single:\n case TypeCode.Double:\n case TypeCode.Decimal:\n return true;\n }\n break;\n case TypeCode.UInt16:\n switch (tc)\n {\n case TypeCode.UInt16:\n case TypeCode.Int32:\n case TypeCode.UInt32:\n case TypeCode.Int64:\n case TypeCode.UInt64:\n case TypeCode.Single:\n case TypeCode.Double:\n case TypeCode.Decimal:\n return true;\n }\n break;\n case TypeCode.Int32:\n switch (tc)\n {\n case TypeCode.Int32:\n case TypeCode.Int64:\n case TypeCode.Single:\n case TypeCode.Double:\n case TypeCode.Decimal:\n return true;\n }\n break;\n case TypeCode.UInt32:\n switch (tc)\n {\n case TypeCode.UInt32:\n case TypeCode.Int64:\n case TypeCode.UInt64:\n case TypeCode.Single:\n case TypeCode.Double:\n case TypeCode.Decimal:\n return true;\n }\n break;\n case TypeCode.Int64:\n switch (tc)\n {\n case TypeCode.Int64:\n case TypeCode.Single:\n case TypeCode.Double:\n case TypeCode.Decimal:\n return true;\n }\n break;\n case TypeCode.UInt64:\n switch (tc)\n {\n case TypeCode.UInt64:\n case TypeCode.Single:\n case TypeCode.Double:\n case TypeCode.Decimal:\n return true;\n }\n break;\n case TypeCode.Single:\n switch (tc)\n {\n case TypeCode.Single:\n case TypeCode.Double:\n return true;\n }\n break;\n default:\n if (st == tt) return true;\n break;\n }\n return false;\n }\n\n static bool IsBetterThan(Expression[] args, MethodData m1, MethodData m2)\n {\n bool better = false;\n for (int i = 0; i < args.Length; i++)\n {\n int c = CompareConversions(args[i].Type,\n m1.Parameters[i].ParameterType,\n m2.Parameters[i].ParameterType);\n if (c < 0) return false;\n if (c > 0) better = true;\n }\n return better;\n }\n\n // Return 1 if s -> t1 is a better conversion than s -> t2\n // Return -1 if s -> t2 is a better conversion than s -> t1\n // Return 0 if neither conversion is better\n static int CompareConversions(Type s, Type t1, Type t2)\n {\n if (t1 == t2) return 0;\n if (s == t1) return 1;\n if (s == t2) return -1;\n bool t1t2 = IsCompatibleWith(t1, t2);\n bool t2t1 = IsCompatibleWith(t2, t1);\n if (t1t2 && !t2t1) return 1;\n if (t2t1 && !t1t2) return -1;\n if (IsSignedIntegralType(t1) && IsUnsignedIntegralType(t2)) return 1;\n if (IsSignedIntegralType(t2) && IsUnsignedIntegralType(t1)) return -1;\n return 0;\n }\n\n Expression GenerateEqual(Expression left, Expression right)\n {\n return Expression.Equal(left, right);\n }\n\n Expression GenerateNotEqual(Expression left, Expression right)\n {\n return Expression.NotEqual(left, right);\n }\n\n Expression GenerateGreaterThan(Expression left, Expression right)\n {\n if (left.Type == typeof(string))\n {\n return Expression.GreaterThan(\n GenerateStaticMethodCall(\"Compare\", left, right),\n Expression.Constant(0)\n );\n }\n return Expression.GreaterThan(left, right);\n }\n\n Expression GenerateGreaterThanEqual(Expression left, Expression right)\n {\n if (left.Type == typeof(string))\n {\n return Expression.GreaterThanOrEqual(\n GenerateStaticMethodCall(\"Compare\", left, right),\n Expression.Constant(0)\n );\n }\n return Expression.GreaterThanOrEqual(left, right);\n }\n\n Expression GenerateLessThan(Expression left, Expression right)\n {\n if (left.Type == typeof(string))\n {\n return Expression.LessThan(\n GenerateStaticMethodCall(\"Compare\", left, right),\n Expression.Constant(0)\n );\n }\n return Expression.LessThan(left, right);\n }\n\n Expression GenerateLessThanEqual(Expression left, Expression right)\n {\n if (left.Type == typeof(string))\n {\n return Expression.LessThanOrEqual(\n GenerateStaticMethodCall(\"Compare\", left, right),\n Expression.Constant(0)\n );\n }\n return Expression.LessThanOrEqual(left, right);\n }\n\n MethodInfo GetStaticMethod(string methodName, Expression left, Expression right)\n {\n return left.Type.GetMethod(methodName, new[] { left.Type, right.Type });\n }\n\n Expression GenerateStaticMethodCall(string methodName, Expression left, Expression right)\n {\n return Expression.Call(null, GetStaticMethod(methodName, left, right), new[] { left, right });\n }\n\n void SetTextPos(int pos)\n {\n textPos = pos;\n ch = textPos < textLen ? text[textPos] : '\\0';\n }\n\n void NextChar()\n {\n if (textPos < textLen) textPos++;\n ch = textPos < textLen ? text[textPos] : '\\0';\n }\n\n void NextToken()\n {\n while (Char.IsWhiteSpace(ch)) NextChar();\n TokenId t;\n int tokenPos = textPos;\n switch (ch)\n {\n case '!':\n NextChar();\n if (ch == '=')\n {\n NextChar();\n t = TokenId.ExclamationEqual;\n }\n else\n {\n t = TokenId.Exclamation;\n }\n break;\n case '%':\n NextChar();\n t = TokenId.Percent;\n break;\n case '&':\n NextChar();\n if (ch == '&')\n {\n NextChar();\n t = TokenId.DoubleAmphersand;\n }\n else\n {\n t = TokenId.Amphersand;\n }\n break;\n case '(':\n NextChar();\n t = TokenId.OpenParen;\n break;\n case ')':\n NextChar();\n t = TokenId.CloseParen;\n break;\n case '*':\n NextChar();\n t = TokenId.Asterisk;\n break;\n case '+':\n NextChar();\n t = TokenId.Plus;\n break;\n case ',':\n NextChar();\n t = TokenId.Comma;\n break;\n case '-':\n NextChar();\n t = TokenId.Minus;\n break;\n case '.':\n NextChar();\n t = TokenId.Dot;\n break;\n case '/':\n NextChar();\n t = TokenId.Slash;\n break;\n case ':':\n NextChar();\n t = TokenId.Colon;\n break;\n case '<':\n NextChar();\n if (ch == '=')\n {\n NextChar();\n t = TokenId.LessThanEqual;\n }\n else if (ch == '>')\n {\n NextChar();\n t = TokenId.LessGreater;\n }\n else\n {\n t = TokenId.LessThan;\n }\n break;\n case '=':\n NextChar();\n if (ch == '=')\n {\n NextChar();\n t = TokenId.DoubleEqual;\n }\n else\n {\n t = TokenId.Equal;\n }\n break;\n case '>':\n NextChar();\n if (ch == '=')\n {\n NextChar();\n t = TokenId.GreaterThanEqual;\n }\n else\n {\n t = TokenId.GreaterThan;\n }\n break;\n case '?':\n NextChar();\n t = TokenId.Question;\n break;\n case '[':\n NextChar();\n t = TokenId.OpenBracket;\n break;\n case ']':\n NextChar();\n t = TokenId.CloseBracket;\n break;\n case '|':\n NextChar();\n if (ch == '|')\n {\n NextChar();\n t = TokenId.DoubleBar;\n }\n else\n {\n t = TokenId.Bar;\n }\n break;\n case '\"':\n case '\\'':\n char quote = ch;\n do\n {\n NextChar();\n while (textPos < textLen && ch != quote) NextChar();\n if (textPos == textLen)\n throw ParseError(textPos, Res.UnterminatedStringLiteral);\n NextChar();\n } while (ch == quote);\n t = TokenId.StringLiteral;\n break;\n default:\n if (Char.IsLetter(ch) || ch == '@' || ch == '_')\n {\n do\n {\n NextChar();\n } while (Char.IsLetterOrDigit(ch) || ch == '_');\n t = TokenId.Identifier;\n break;\n }\n if (Char.IsDigit(ch))\n {\n t = TokenId.IntegerLiteral;\n do\n {\n NextChar();\n } while (Char.IsDigit(ch));\n if (ch == '.')\n {\n t = TokenId.RealLiteral;\n NextChar();\n ValidateDigit();\n do\n {\n NextChar();\n } while (Char.IsDigit(ch));\n }\n if (ch == 'E' || ch == 'e')\n {\n t = TokenId.RealLiteral;\n NextChar();\n if (ch == '+' || ch == '-') NextChar();\n ValidateDigit();\n do\n {\n NextChar();\n } while (Char.IsDigit(ch));\n }\n if (ch == 'F' || ch == 'f') NextChar();\n break;\n }\n if (textPos == textLen)\n {\n t = TokenId.End;\n break;\n }\n throw ParseError(textPos, Res.InvalidCharacter, ch);\n }\n token.id = t;\n token.text = text.Substring(tokenPos, textPos - tokenPos);\n token.pos = tokenPos;\n }\n\n bool TokenIdentifierIs(string id)\n {\n return token.id == TokenId.Identifier && String.Equals(id, token.text, StringComparison.OrdinalIgnoreCase);\n }\n\n string GetIdentifier()\n {\n ValidateToken(TokenId.Identifier, Res.IdentifierExpected);\n string id = token.text;\n if (id.Length > 1 && id[0] == '@') id = id.Substring(1);\n return id;\n }\n\n void ValidateDigit()\n {\n if (!Char.IsDigit(ch)) throw ParseError(textPos, Res.DigitExpected);\n }\n\n void ValidateToken(TokenId t, string errorMessage)\n {\n if (token.id != t) throw ParseError(errorMessage);\n }\n\n void ValidateToken(TokenId t)\n {\n if (token.id != t) throw ParseError(Res.SyntaxError);\n }\n\n Exception ParseError(string format, params object[] args)\n {\n return ParseError(token.pos, format, args);\n }\n\n Exception ParseError(int pos, string format, params object[] args)\n {\n return new ParseException(string.Format(System.Globalization.CultureInfo.CurrentCulture, format, args), pos);\n }\n\n static Dictionary CreateKeywords()\n {\n Dictionary d = new Dictionary(StringComparer.OrdinalIgnoreCase);\n d.Add(\"true\", trueLiteral);\n d.Add(\"false\", falseLiteral);\n d.Add(\"null\", nullLiteral);\n //d.Add(keywordIt, keywordIt);\n //d.Add(keywordIif, keywordIif);\n //d.Add(keywordNew, keywordNew);\n foreach (Type type in predefinedTypes) d.Add(type.Name, type);\n return d;\n }\n }\n\n static class Res\n {\n public const string DuplicateIdentifier = \"The identifier '{0}' was defined more than once\";\n public const string ExpressionTypeMismatch = \"Expression of type '{0}' expected\";\n public const string ExpressionExpected = \"Expression expected\";\n public const string InvalidCharacterLiteral = \"Character literal must contain exactly one character\";\n public const string InvalidIntegerLiteral = \"Invalid integer literal '{0}'\";\n public const string InvalidRealLiteral = \"Invalid real literal '{0}'\";\n public const string UnknownIdentifier = \"Unknown identifier '{0}'\";\n public const string NoItInScope = \"No 'it' is in scope\";\n public const string IifRequiresThreeArgs = \"The 'iif' function requires three arguments\";\n public const string FirstExprMustBeBool = \"The first expression must be of type 'Boolean'\";\n public const string BothTypesConvertToOther = \"Both of the types '{0}' and '{1}' convert to the other\";\n public const string NeitherTypeConvertsToOther = \"Neither of the types '{0}' and '{1}' converts to the other\";\n public const string MissingAsClause = \"Expression is missing an 'as' clause\";\n public const string ArgsIncompatibleWithLambda = \"Argument list incompatible with lambda expression\";\n public const string TypeHasNoNullableForm = \"Type '{0}' has no nullable form\";\n public const string NoMatchingConstructor = \"No matching constructor in type '{0}'\";\n public const string AmbiguousConstructorInvocation = \"Ambiguous invocation of '{0}' constructor\";\n public const string CannotConvertValue = \"A value of type '{0}' cannot be converted to type '{1}'\";\n public const string NoApplicableMethod = \"No applicable method '{0}' exists in type '{1}'\";\n public const string MethodsAreInaccessible = \"Methods on type '{0}' are not accessible\";\n public const string MethodIsVoid = \"Method '{0}' in type '{1}' does not return a value\";\n public const string AmbiguousMethodInvocation = \"Ambiguous invocation of method '{0}' in type '{1}'\";\n public const string UnknownPropertyOrField = \"No property or field '{0}' exists in type '{1}'\";\n public const string NoApplicableAggregate = \"No applicable aggregate method '{0}' exists\";\n public const string CannotIndexMultiDimArray = \"Indexing of multi-dimensional arrays is not supported\";\n public const string InvalidIndex = \"Array index must be an integer expression\";\n public const string NoApplicableIndexer = \"No applicable indexer exists in type '{0}'\";\n public const string AmbiguousIndexerInvocation = \"Ambiguous invocation of indexer in type '{0}'\";\n public const string IncompatibleOperand = \"Operator '{0}' incompatible with operand type '{1}'\";\n public const string IncompatibleOperands = \"Operator '{0}' incompatible with operand types '{1}' and '{2}'\";\n public const string UnterminatedStringLiteral = \"Unterminated string literal\";\n public const string InvalidCharacter = \"Syntax error '{0}'\";\n public const string DigitExpected = \"Digit expected\";\n public const string SyntaxError = \"Syntax error\";\n public const string TokenExpected = \"{0} expected\";\n public const string ParseExceptionFormat = \"{0} (at index {1})\";\n public const string ColonExpected = \"':' expected\";\n public const string OpenParenExpected = \"'(' expected\";\n public const string CloseParenOrOperatorExpected = \"')' or operator expected\";\n public const string CloseParenOrCommaExpected = \"')' or ',' expected\";\n public const string DotOrOpenParenExpected = \"'.' or '(' expected\";\n public const string OpenBracketExpected = \"'[' expected\";\n public const string CloseBracketOrCommaExpected = \"']' or ',' expected\";\n public const string IdentifierExpected = \"Identifier expected\";\n }\n}\n" }, { "path": "RaptorDB/Views/LINQQuery.cs", "content": "using System;\nusing System.Collections;\nusing System.Collections.Generic;\nusing System.Linq;\nusing System.Linq.Expressions;\nusing System.Reflection;\n\nnamespace RaptorDB.Views\n{\n delegate MGRB QueryFromTo(string colname, object from, object to);\n delegate MGRB QueryExpression(string colname, RDBExpression exp, object from);\n\n internal class QueryVisitor : ExpressionVisitor\n {\n public QueryVisitor(QueryExpression express, QueryFromTo fromto)\n {\n qexpression = express;\n qfromto = fromto;\n }\n public Stack _stack = new Stack();\n public Stack _bitmap = new Stack();\n QueryFromTo qfromto;\n QueryExpression qexpression;\n private bool _leftmode = true;\n\n protected override Expression VisitBinary(BinaryExpression b)\n {\n _leftmode = true;\n var m = this.Visit(b.Left);\n if (m == null) // VB.net sty;e linq for string compare\n return b.Right;\n ExpressionType t = b.NodeType;\n\n if (t == ExpressionType.Equal || t == ExpressionType.NotEqual ||\n t == ExpressionType.LessThan || t == ExpressionType.LessThanOrEqual ||\n t == ExpressionType.GreaterThan || t == ExpressionType.GreaterThanOrEqual)\n _stack.Push(b.NodeType);\n\n _leftmode = false;\n this.Visit(b.Right);\n t = b.NodeType;\n if (t == ExpressionType.Equal || t == ExpressionType.NotEqual ||\n t == ExpressionType.LessThanOrEqual || t == ExpressionType.LessThan ||\n t == ExpressionType.GreaterThanOrEqual || t == ExpressionType.GreaterThan\n )\n {\n // binary expression \n object lval = _stack.Pop();\n ExpressionType lop = (ExpressionType)_stack.Pop();\n string lname = (string)_stack.Pop();\n if (_stack.Count > 0)\n {\n lname += \"_\" + (string)_stack.Pop();\n }\n RDBExpression exp = RDBExpression.Equal;\n if (lop == ExpressionType.LessThan) exp = RDBExpression.Less;\n else if (lop == ExpressionType.LessThanOrEqual) exp = RDBExpression.LessEqual;\n else if (lop == ExpressionType.GreaterThan) exp = RDBExpression.Greater;\n else if (lop == ExpressionType.GreaterThanOrEqual) exp = RDBExpression.GreaterEqual;\n else if (lop == ExpressionType.NotEqual) exp = RDBExpression.NotEqual;\n\n _bitmap.Push(qexpression(lname, exp, lval));\n }\n\n if (t == ExpressionType.And || t == ExpressionType.AndAlso ||\n t == ExpressionType.Or || t == ExpressionType.OrElse)\n {\n if (_bitmap.Count > 1)\n {\n // do bitmap operations \n MGRB right = (MGRB)_bitmap.Pop();\n MGRB left = (MGRB)_bitmap.Pop();\n\n if (t == ExpressionType.And || t == ExpressionType.AndAlso)\n _bitmap.Push(right.And(left));\n if (t == ExpressionType.Or || t == ExpressionType.OrElse)\n _bitmap.Push(right.Or(left));\n }\n else\n {\n // single bitmap operation\n }\n }\n return b;\n }\n\n protected override Expression VisitMethodCall(MethodCallExpression m)\n {\n string s = m.ToString();\n // FEATURE : add contains , startswith ??\n\n // VB.net : e.g. CompareString(x.NoCase, \"Me 4\", False)\n if (s.StartsWith(\"CompareString\"))\n {\n var left = m.Arguments[0];\n // Removes dot if any\n var leftStr = left.ToString().Substring(left.ToString().IndexOf('.') + 1);\n var right = m.Arguments[1].ToString().Replace(\"\\\"\", String.Empty);\n RDBExpression exp = RDBExpression.Equal;\n _bitmap.Push(qexpression(\"\" + leftStr, exp, right));\n return null;\n }\n string mc = s.Substring(s.IndexOf('.') + 1);\n if (mc.Contains(\"Between\"))\n {\n Type datatype = m.Arguments[0].Type;\n var ss = m.Arguments[0].ToString().Split('.');\n string name = \"\";\n if (ss.Length > 2)\n {\n // handle datetype.year etc\n name = ss[1] + \"_$\" + ss[2];\n }\n else\n name = ss[1];\n\n if (datatype == typeof(DateTime))\n {\n DateTime from = DateTime.Now;\n DateTime to = DateTime.Now;\n if (m.Arguments[1].Type == typeof(string))\n {\n from = DateTime.Parse((string)GetValueForMember(m.Arguments[1]));\n to = DateTime.Parse((string)GetValueForMember(m.Arguments[2]));\n }\n else\n {\n from = (DateTime)GetValueForMember(m.Arguments[1]);\n to = (DateTime)GetValueForMember(m.Arguments[2]);\n }\n _bitmap.Push(qfromto(name, from, to));\n }\n else\n {\n var from = GetValueForMember(m.Arguments[1]);\n var to = GetValueForMember(m.Arguments[2]);\n _bitmap.Push(qfromto(name, from, to));\n }\n }\n else if (mc.Contains(\"In\"))\n {\n var ss = m.Arguments[0].ToString().Split('.');\n string name = \"\";\n if (ss.Length > 2)\n {\n // handle datetype.year etc\n name = ss[1] + \"_$\" + ss[2];\n }\n else\n name = ss[1];\n _InCommand = name;\n Visit(m.Arguments[1]);\n _bitmap.Push(_inBmp);\n _inBmp = null;\n _InCommand = \"\";\n }\n else\n _stack.Push(mc);\n\n return m;\n }\n\n private MGRB _inBmp = null;\n private string _InCommand = \"\";\n private int _count = 0;\n public override Expression Visit(Expression node)\n {\n if (node.NodeType == ExpressionType.NewArrayInit)\n {\n var a = node as NewArrayExpression;\n _count = a.Expressions.Count;\n return base.Visit(node);\n }\n else if (node.NodeType == ExpressionType.MemberAccess)\n {\n var v = base.Visit(node);\n if (_InCommand != \"\")\n {\n var a = _stack.Pop() as IList;\n foreach (var c in a)\n {\n if (_inBmp == null)\n _inBmp = qexpression(_InCommand, RDBExpression.Equal, c);\n else\n _inBmp = _inBmp.Or(qexpression(_InCommand, RDBExpression.Equal, c));\n }\n }\n\n return v;\n }\n else\n return base.Visit(node);\n }\n\n private object GetValueForMember(object m)\n {\n object val = null;\n var f = m as ConstantExpression;\n if (f != null)\n return f.Value;\n\n var mm = m as MemberExpression;\n if (mm.NodeType == ExpressionType.MemberAccess)\n {\n Type tt = mm.Expression.Type;\n val = tt.InvokeMember(mm.Member.Name, BindingFlags.GetField |\n BindingFlags.GetProperty |\n BindingFlags.Public |\n BindingFlags.NonPublic |\n BindingFlags.Instance |\n BindingFlags.Static, null, (mm.Expression as ConstantExpression).Value, null);\n }\n return val;\n }\n\n protected override Expression VisitMember(MemberExpression m)\n {\n var n = m.Member.Name;\n if (n != null && m.Expression.Type == typeof(DateTime))\n {\n _stack.Push(\"$\" + n);\n }\n var e = base.VisitMember(m);\n var c = m.Expression as ConstantExpression;\n if (c != null)\n {\n Type t = c.Value.GetType();\n var x = t.InvokeMember(m.Member.Name, BindingFlags.GetField |\n BindingFlags.GetProperty |\n BindingFlags.Public |\n BindingFlags.NonPublic |\n BindingFlags.Instance |\n BindingFlags.Static, null, c.Value, null);\n _stack.Push(x);\n }\n\n if (m.Expression != null)\n {\n if (m.Expression.NodeType == ExpressionType.Parameter) // property\n _stack.Push(m.Member.Name);\n else if (m.Expression.NodeType == ExpressionType.MemberAccess) // obj.property\n {\n if (_leftmode == false)\n {\n Type t = m.Expression.Type;\n var val = t.InvokeMember(m.Member.Name, BindingFlags.GetField |\n BindingFlags.GetProperty |\n BindingFlags.Public |\n BindingFlags.NonPublic |\n BindingFlags.Instance |\n BindingFlags.Static, null, _stack.Pop(), null);\n _stack.Push(val);\n }\n }\n }\n\n return e;\n }\n\n protected override Expression VisitConstant(ConstantExpression c)\n {\n IQueryable q = c.Value as IQueryable;\n if (q != null)\n {\n _stack.Push(q.ElementType.Name);\n }\n else if (c.Value == null)\n {\n _stack.Push(null);\n }\n else\n {\n Type t = c.Value.GetType();\n if (t.IsValueType || t == typeof(string))\n {\n if (_InCommand != \"\")\n {\n if (_inBmp == null)\n _inBmp = qexpression(_InCommand, RDBExpression.Equal, c.Value);\n else\n _inBmp = _inBmp.Or(qexpression(_InCommand, RDBExpression.Equal, c.Value));\n }\n else\n _stack.Push(c.Value);\n }\n }\n return c;\n }\n }\n}\n" }, { "path": "RaptorDB/Views/TaskQueue.cs", "content": "using System;\nusing System.Timers;\nusing System.Threading.Tasks;\nusing System.Collections.Concurrent;\n\nnamespace RaptorDB.Views\n{\n internal class TaskQueue\n {\n public TaskQueue()\n {\n _timer = new Timer();\n _timer.Interval = Global.TaskCleanupTimerSeconds * 1000;\n _timer.Elapsed += new ElapsedEventHandler(_timer_Elapsed);\n _timer.Enabled = true;\n _log.Debug(\"TaskQueue starting\");\n }\n\n private ILog _log = LogManager.GetLogger(typeof(TaskQueue));\n private object _lock = new object();\n private bool _shuttingdown = false;\n private Timer _timer;\n private ConcurrentQueue _que = new ConcurrentQueue();\n\n void _timer_Elapsed(object sender, ElapsedEventArgs e)\n {\n lock (_lock)\n {\n while (_que.Count > 0)\n {\n //_log.Debug(\"in queue cleanup, count = \" + _que.Count);\n if (_shuttingdown)\n break;\n Task t = null;\n if (_que.TryPeek(out t))\n {\n if (t.IsCompleted || t.IsCanceled || t.IsFaulted)\n _que.TryDequeue(out t);\n else\n break;\n }\n else\n break;\n }\n }\n }\n\n public void AddTask(Action action)\n {\n if (_shuttingdown == false)\n _que.Enqueue(Task.Factory.StartNew(action));\n }\n\n public void Shutdown()\n {\n _log.Debug(\"TaskQueue shutdown\");\n // wait for tasks to finish\n _shuttingdown = true;\n _timer.Enabled = false;\n if (_que.Count > 0)\n Task.WaitAll(_que.ToArray());\n }\n }\n}\n" }, { "path": "RaptorDB/Views/ViewHandler.cs", "content": "using fastBinaryJSON;\nusing fastJSON;\nusing RaptorDB.Common;\nusing System;\nusing System.Collections.Generic;\nusing System.IO;\nusing System.Linq.Expressions;\nusing System.Reflection;\nusing System.Reflection.Emit;\nusing System.Text;\nusing System.Threading;\nusing System.Threading.Tasks;\n\nnamespace RaptorDB.Views\n{\n public class ViewRowDefinition\n {\n public ViewRowDefinition()\n {\n Columns = new List>();\n }\n public List> Columns { get; set; }\n\n public void Add(string name, Type type)\n {\n Columns.Add(new KeyValuePair(name, type));\n }\n }\n\n internal class tran_data\n {\n public Guid docid;\n public Dictionary> rows;\n }\n\n internal class ViewHandler\n {\n private ILog _log = LogManager.GetLogger(typeof(ViewHandler));\n\n public ViewHandler(string path, ViewManager manager)\n {\n _Path = path;\n _viewmanager = manager;\n }\n\n private string _S = Path.DirectorySeparatorChar.ToString();\n private string _Path = \"\";\n private ViewManager _viewmanager;\n internal ViewBase _view;\n private Dictionary _indexes = new Dictionary();\n private StorageFile _viewData;\n private BoolIndex _deletedRows;\n private string _docid = \"docid\";\n private List _colnames = new List();\n private RowFill _rowfiller;\n private ViewRowDefinition _schema;\n private ViewRowDefinition _datecolumns;\n private SafeDictionary _transactions = new SafeDictionary();\n private SafeDictionary _nocase = new SafeDictionary();\n private Dictionary _idxlen = new Dictionary();\n\n private System.Timers.Timer _saveTimer;\n Type basetype; // used for mapper\n dynamic _mapper;\n bool _isDirty = false;\n private string _dirtyFilename = \"temp.$\";\n private bool _stsaving = false;\n private int _RaptorDBVersion = 5; // used for engine changes to views\n private string _RaptorDBVersionFilename = \"RaptorDB.version\";\n\n // TODO : showing incorrect results\n //private SafeDictionary _queryCache = new SafeDictionary();\n\n private object _savetimerlock = new object();\n void _saveTimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)\n {\n lock (_savetimerlock)\n {\n _stsaving = true;\n foreach (var i in _indexes)\n i.Value.SaveIndex();\n\n _deletedRows.SaveIndex();\n _stsaving = false;\n }\n }\n\n public Type GetFireOnType()\n {\n return basetype;\n }\n\n internal void SetView(View view, IDocStorage docs)\n {\n bool rebuild = false;\n _view = view;\n // generate schemacolumns from schema\n GenerateSchemaColumns(_view);\n\n if (_Path.EndsWith(_S) == false) _Path += _S;\n _Path += view.Name + _S;\n if (Directory.Exists(_Path) == false)\n {\n Directory.CreateDirectory(_Path);\n rebuild = true;\n }\n else\n {\n // read version file and check with view\n int version = 0;\n if (File.Exists(_Path + _view.Name + \".version\"))\n {\n int.TryParse(File.ReadAllText(_Path + _view.Name + \".version\"), out version);\n if (version != view.Version)\n {\n _log.Debug(\"Newer view version detected\");\n rebuild = true;\n }\n }\n }\n\n if (File.Exists(_Path + _dirtyFilename))\n {\n _log.Debug(\"Last shutdown failed, rebuilding view : \" + _view.Name);\n rebuild = true;\n }\n\n if (File.Exists(_Path + _RaptorDBVersionFilename))\n {\n // check view engine version\n string s = File.ReadAllText(_Path + _RaptorDBVersionFilename);\n int version = 0;\n int.TryParse(s, out version);\n if (version != _RaptorDBVersion)\n {\n _log.Debug(\"RaptorDB view engine upgrade, rebuilding view : \" + _view.Name);\n rebuild = true;\n }\n }\n else\n {\n _log.Debug(\"RaptorDB view engine upgrade, rebuilding view : \" + _view.Name);\n rebuild = true;\n }\n\n if (rebuild)\n {\n _log.Debug(\"Deleting old view data folder = \" + view.Name);\n Directory.Delete(_Path, true);\n Directory.CreateDirectory(_Path);\n }\n // load indexes here\n CreateLoadIndexes();\n\n _deletedRows = new BoolIndex(_Path, _view.Name, \".deleted\");\n\n _viewData = new StorageFile(_Path + view.Name + \".mgdat\");\n\n CreateResultRowFiller();\n\n _mapper = view.Mapper;\n // looking for the T in View\n if (view.GetType().GetGenericArguments().Length == 1) // HACK : kludge change when possible \n basetype = view.GetType().GetGenericArguments()[0];\n else\n {\n // or recurse until found\n basetype = view.GetType().BaseType.GetGenericArguments()[0];\n }\n\n if (rebuild)\n Task.Factory.StartNew(() => RebuildFromScratch(docs));\n\n _saveTimer = new System.Timers.Timer();\n _saveTimer.AutoReset = true;\n _saveTimer.Elapsed += new System.Timers.ElapsedEventHandler(_saveTimer_Elapsed);\n _saveTimer.Interval = Global.SaveIndexToDiskTimerSeconds * 1000;\n _saveTimer.Start();\n }\n\n internal void FreeMemory()\n {\n _log.Debug(\"free memory : \" + _view.Name);\n foreach (var i in _indexes)\n i.Value.FreeMemory();\n\n _deletedRows.FreeMemory();\n InvalidateCaches();\n }\n\n internal void Commit(int ID)\n {\n tran_data data = null;\n // save data to indexes\n if (_transactions.TryGetValue(ID, out data))\n {\n // delete any items with docid in view\n if (_view.DeleteBeforeInsert)\n DeleteRowsWith(data.docid);\n SaveAndIndex(data.rows);\n }\n // remove in memory data\n _transactions.Remove(ID);\n }\n\n internal void RollBack(int ID)\n {\n // remove in memory data\n _transactions.Remove(ID);\n }\n\n internal void Insert(Guid guid, T doc)\n {\n apimapper api = new apimapper(_viewmanager, this);\n\n if (basetype == doc.GetType())\n {\n View view = (View)_view;\n if (view.Mapper != null)\n view.Mapper(api, guid, doc);\n }\n else if (_mapper != null)\n _mapper(api, guid, doc);\n\n // map objects to rows\n foreach (var d in api.emitobj)\n api.emit.Add(d.Key, ExtractRows(d.Value));\n\n // delete any items with docid in view\n if (_view.DeleteBeforeInsert)\n DeleteRowsWith(guid);\n\n SaveAndIndex(api.emit);\n }\n\n private void SaveAndIndex(Dictionary> rows)\n {\n foreach (var d in rows)\n {\n // insert new items into view\n InsertRowsWithIndexUpdate(d.Key, d.Value);\n }\n InvalidateCaches();\n }\n\n internal bool InsertTransaction(Guid docid, T doc)\n {\n apimapper api = new apimapper(_viewmanager, this);\n if (basetype == doc.GetType())\n {\n View view = (View)_view;\n\n try\n {\n if (view.Mapper != null)\n view.Mapper(api, docid, doc);\n }\n catch (Exception ex)\n {\n _log.Error(ex);\n return false;\n }\n }\n else if (_mapper != null)\n _mapper(api, docid, doc);\n\n if (api._RollBack == true)\n return false;\n\n // map emitobj -> rows\n foreach (var d in api.emitobj)\n api.emit.Add(d.Key, ExtractRows(d.Value));\n\n tran_data data = new tran_data();\n if (_transactions.TryGetValue(Thread.CurrentThread.ManagedThreadId, out data))\n {\n // TODO : exists -> merge data??\n }\n else\n {\n data = new tran_data();\n data.docid = docid;\n data.rows = api.emit;\n _transactions.Add(Thread.CurrentThread.ManagedThreadId, data);\n }\n\n return true;\n }\n\n internal Result Query(string filter, int start, int count, string orderby)\n {\n filter = filter.Trim();\n if (filter == \"\")\n return Query(start, count, orderby);\n\n DateTime dt = FastDateTime.Now;\n _log.Debug(\"query : \" + _view.Name);\n _log.Debug(\"query : \" + filter);\n if (orderby != \"\")\n _log.Debug(\"orderby : \" + orderby);\n\n var ba = GenerateBitmap(filter);\n var order = SortBy(orderby);\n bool desc = false;\n if (orderby.ToLower().Contains(\" desc\"))\n desc = true;\n _log.Debug(\"query bitmap done (ms) : \" + FastDateTime.Now.Subtract(dt).TotalMilliseconds);\n dt = FastDateTime.Now;\n // exec query return rows\n return ReturnRowsTyped(ba, null, start, count, order, desc);\n }\n\n internal Result Query(int start, int count)\n {\n return Query(start, count, \"\");\n }\n\n internal Result Query(int start, int count, string orderby)\n {\n // no filter query -> just show all the data\n DateTime dt = FastDateTime.Now;\n _log.Debug(\"query : \" + _view.Name);\n int totalviewrows = _viewData.Count();\n List rows = new List();\n Result ret = new Result();\n int skip = start;\n int cc = 0;\n MGRB del = _deletedRows.GetBits();\n ret.TotalCount = totalviewrows - (int)del.CountOnes();\n\n var order = SortBy(orderby);\n bool desc = false;\n if (orderby.ToLower().Contains(\" desc\"))\n desc = true;\n if (order.Count == 0)\n for (int i = 0; i < totalviewrows; i++)\n order.Add(i);\n\n if (count == -1)\n count = totalviewrows;\n int len = order.Count;\n if (desc == false)\n {\n for (int idx = 0; idx < len; idx++)\n {\n extractrowobject(count, rows, ref skip, ref cc, del, order, idx);\n if (cc == count) break;\n }\n }\n else\n {\n for (int idx = len - 1; idx >= 0; idx--)\n {\n extractrowobject(count, rows, ref skip, ref cc, del, order, idx);\n if (cc == count) break;\n }\n }\n\n _log.Debug(\"query rows fetched (ms) : \" + FastDateTime.Now.Subtract(dt).TotalMilliseconds);\n _log.Debug(\"query rows count : \" + rows.Count.ToString(\"#,0\"));\n ret.OK = true;\n ret.Count = rows.Count;\n ret.Rows = rows;\n return ret;\n }\n\n private void extractrowobject(int count, List rows, ref int skip, ref int cc, MGRB del, List order, int idx)\n {\n int i = order[idx];\n if (del.Get(i) == false)\n {\n if (skip > 0)\n skip--;\n else\n {\n bool b = OutputRow(rows, i);\n if (b && count > 0)\n cc++;\n }\n }\n }\n\n internal void Shutdown()\n {\n try\n {\n lock (_savetimerlock)\n _saveTimer.Enabled = false;\n while (_stsaving)\n Thread.Sleep(1);\n\n if (_rebuilding)\n _log.Debug(\"Waiting for view rebuild to finish... : \" + _view.Name);\n\n while (_rebuilding)\n Thread.Sleep(50);\n\n _log.Debug(\"Shutting down Viewhandler\");\n // shutdown indexes\n List tasks = new List();\n foreach (var v in _indexes)\n {\n tasks.Add(Task.Factory.StartNew(() =>\n {\n _log.Debug(\"Shutting down view index : \" + v.Key);\n v.Value.Shutdown();\n }));\n }\n Task.WaitAll(tasks.ToArray());\n // save deletedbitmap\n _deletedRows.Shutdown();\n\n _viewData.Shutdown();\n\n // write view version\n File.WriteAllText(_Path + _view.Name + \".version\", _view.Version.ToString());\n\n File.WriteAllText(_Path + _RaptorDBVersionFilename, _RaptorDBVersion.ToString());\n // remove dirty file\n if (File.Exists(_Path + _dirtyFilename))\n File.Delete(_Path + _dirtyFilename);\n _log.Debug(\"Viewhandler shutdown done.\");\n }\n catch (Exception ex)\n {\n _log.Error(ex);\n }\n }\n\n internal void Delete(Guid docid)\n {\n DeleteRowsWith(docid);\n\n InvalidateCaches();\n }\n\n #region [ private methods ]\n\n private void CreateResultRowFiller()\n {\n _rowfiller = CreateRowFillerDelegate(_view.Schema, _schema);\n // init the row create \n _createrow = null;\n FastCreateObject(_view.Schema);\n }\n\n public delegate object RowFill(object o, object[] data);\n public static RowFill CreateRowFillerDelegate(Type objtype, ViewRowDefinition schema)\n {\n DynamicMethod dynMethod = new DynamicMethod(\"rowfill\", typeof(object), new Type[] { typeof(object), typeof(object[]) });\n ILGenerator il = dynMethod.GetILGenerator();\n var row = il.DeclareLocal(objtype);\n il.Emit(OpCodes.Ldarg_0);\n il.Emit(OpCodes.Castclass, objtype);\n il.Emit(OpCodes.Stloc, row);\n int i = 1;\n var val = il.DeclareLocal(typeof(object));\n var fields = objtype.GetFields();\n var properties = objtype.GetProperties();\n foreach (var col in schema.Columns)\n {\n FieldInfo c = null;\n PropertyInfo p = null;\n if (isField(fields, col.Key, out c))\n {\n var end = il.DefineLabel();\n\n il.Emit(OpCodes.Ldarg_1);\n if (col.Key != \"docid\")\n il.Emit(OpCodes.Ldc_I4, i);\n else\n il.Emit(OpCodes.Ldc_I4, 0);\n\n il.Emit(OpCodes.Ldelem_Ref);\n // check if value is not null\n il.Emit(OpCodes.Stloc, val);\n il.Emit(OpCodes.Ldloc, val);\n il.Emit(OpCodes.Brfalse_S, end);\n il.Emit(OpCodes.Ldloc, row);\n il.Emit(OpCodes.Ldloc, val);\n\n il.Emit(OpCodes.Unbox_Any, c.FieldType);\n il.Emit(OpCodes.Stfld, c);\n il.MarkLabel(end);\n i++;\n }\n else if (isProperty(properties, col.Key, out p))\n {\n var end = il.DefineLabel();\n MethodInfo setMethod = p.GetSetMethod();\n il.Emit(OpCodes.Ldarg_1);\n if (col.Key != \"docid\")\n il.Emit(OpCodes.Ldc_I4, i);\n else\n il.Emit(OpCodes.Ldc_I4, 0);\n\n il.Emit(OpCodes.Ldelem_Ref);\n // check if value is not null\n il.Emit(OpCodes.Stloc, val);\n il.Emit(OpCodes.Ldloc, val);\n il.Emit(OpCodes.Brfalse_S, end);\n il.Emit(OpCodes.Ldloc, row);\n il.Emit(OpCodes.Ldloc, val);\n\n il.Emit(OpCodes.Unbox_Any, p.PropertyType);\n if (!p.DeclaringType.IsValueType)\n il.EmitCall(OpCodes.Callvirt, setMethod, null);\n else\n il.EmitCall(OpCodes.Call, setMethod, null);\n il.MarkLabel(end);\n i++;\n }\n }\n il.Emit(OpCodes.Ldloc, row);\n il.Emit(OpCodes.Ret);\n\n return (RowFill)dynMethod.CreateDelegate(typeof(RowFill));\n }\n\n private static bool isProperty(PropertyInfo[] properties, string key, out PropertyInfo p)\n {\n foreach (var i in properties)\n if (i.Name == key)\n {\n p = i;\n return true;\n }\n p = null;\n return false;\n }\n\n private static bool isField(FieldInfo[] fields, string key, out FieldInfo c)\n {\n foreach (var i in fields)\n if (i.Name == key)\n {\n c = i;\n return true;\n }\n c = null;\n return false;\n }\n\n //private Result ReturnRowsObject(MGRB ba, List trows, int start, int count, List orderby, bool descending)\n //{\n // DateTime dt = FastDateTime.Now;\n // List rows = new List();\n // Result ret = new Result();\n // int skip = start;\n // int c = 0;\n // ret.TotalCount = (int)ba.CountOnes();\n // if (count == -1) count = ret.TotalCount;\n // if (count > 0)\n // {\n // int len = orderby.Count;\n // if (len > 0)\n // {\n // if (descending == false)\n // {\n // for (int idx = 0; idx < len; idx++)\n // {\n // extractsortrowobject(ba, count, orderby, rows, ref skip, ref c, idx);\n // if (c == count) break;\n // }\n // }\n // else\n // {\n // for (int idx = len - 1; idx >= 0; idx--)\n // {\n // extractsortrowobject(ba, count, orderby, rows, ref skip, ref c, idx);\n // if (c == count) break;\n // }\n // }\n // }\n // foreach (int i in ba.GetBitIndexes())\n // {\n // if (c < count)\n // {\n // if (skip > 0)\n // skip--;\n // else\n // {\n // bool b = OutputRow(rows, i);\n // if (b && count > 0)\n // c++;\n // }\n // if (c == count) break;\n // }\n // }\n // }\n // if (trows != null) // TODO : move to start and decrement in count\n // foreach (var o in trows)\n // rows.Add(o);\n // _log.Debug(\"query rows fetched (ms) : \" + FastDateTime.Now.Subtract(dt).TotalMilliseconds);\n // _log.Debug(\"query rows count : \" + rows.Count.ToString(\"#,0\"));\n // ret.OK = true;\n // ret.Count = rows.Count;\n // ret.Rows = rows;\n // return ret;\n //}\n\n private void extractsortrowobject(MGRB ba, int count, List orderby, List rows, ref int skip, ref int c, int idx)\n {\n int i = orderby[idx];\n if (ba.Get(i))\n {\n if (skip > 0)\n skip--;\n else\n {\n bool b = OutputRow(rows, i);\n if (b && count > 0)\n c++;\n }\n ba.Set(i, false);\n }\n }\n\n private bool OutputRow(List rows, int i)\n {\n byte[] b = _viewData.ViewReadRawBytes(i);\n if (b != null)\n {\n object o = FastCreateObject(_view.Schema);\n object[] data = (object[])BJSON.ToObject(b);\n rows.Add((T)_rowfiller(o, data));\n return true;\n }\n return false;\n }\n\n private Result ReturnRowsTyped(MGRB ba, List trows, int start, int count, List orderby, bool descending)\n {\n DateTime dt = FastDateTime.Now;\n List rows = new List();\n Result ret = new Result();\n int skip = start;\n int c = 0;\n ret.TotalCount = (int)ba.CountOnes();\n if (count == -1) count = ret.TotalCount;\n if (count > 0)\n {\n int len = orderby.Count;\n if (len > 0)\n {\n if (descending == false)\n {\n for (int idx = 0; idx < len; idx++) //foreach (int i in orderby)\n {\n extractsortrowT(ba, count, orderby, rows, ref skip, ref c, idx);\n if (c == count) break;\n }\n }\n else\n {\n for (int idx = len - 1; idx >= 0; idx--) //foreach (int i in orderby)\n {\n extractsortrowT(ba, count, orderby, rows, ref skip, ref c, idx);\n if (c == count) break;\n }\n }\n }\n foreach (int i in ba.GetBitIndexes())\n {\n if (c < count)\n {\n if (skip > 0)\n skip--;\n else\n {\n bool b = OutputRow(rows, i);\n if (b && count > 0)\n c++;\n }\n if (c == count) break;\n }\n }\n }\n if (trows != null)// TODO : move to start and decrement in count\n foreach (var o in trows)\n rows.Add(o);\n _log.Debug(\"query rows fetched (ms) : \" + FastDateTime.Now.Subtract(dt).TotalMilliseconds);\n _log.Debug(\"query rows count : \" + rows.Count.ToString(\"#,0\"));\n ret.OK = true;\n ret.Count = rows.Count;\n ret.Rows = rows;\n return ret;\n }\n\n private void extractsortrowT(MGRB ba, int count, List orderby, List rows, ref int skip, ref int c, int idx)\n {\n int i = orderby[idx];\n if (ba.Get(i))\n {\n if (skip > 0)\n skip--;\n else\n {\n bool b = OutputRow(rows, i);\n if (b && count > 0)\n c++;\n }\n ba.Set(i, false);\n }\n }\n\n private CreateRow _createrow = null;\n private delegate object CreateRow();\n private object FastCreateObject(Type objtype)\n {\n try\n {\n if (_createrow != null)\n return _createrow();\n else\n {\n DynamicMethod dynMethod = new DynamicMethod(\"_\", objtype, null);\n ILGenerator ilGen = dynMethod.GetILGenerator();\n\n ilGen.Emit(OpCodes.Newobj, objtype.GetConstructor(Type.EmptyTypes));\n ilGen.Emit(OpCodes.Ret);\n _createrow = (CreateRow)dynMethod.CreateDelegate(typeof(CreateRow));\n return _createrow();\n }\n }\n catch (Exception exc)\n {\n throw new Exception(string.Format(\"Failed to fast create instance for type '{0}' from assemebly '{1}'\",\n objtype.FullName, objtype.AssemblyQualifiedName), exc);\n }\n }\n\n MethodInfo _insertmethod = null;\n bool _rebuilding = false;\n private void RebuildFromScratch(IDocStorage docs)\n {\n _rebuilding = true;\n try\n {\n _insertmethod = this.GetType().GetMethod(\"Insert\", BindingFlags.Instance | BindingFlags.NonPublic);\n _log.Debug(\"Rebuilding view from scratch...\");\n _log.Debug(\"View = \" + _view.Name);\n DateTime dt = FastDateTime.Now;\n\n int c = docs.RecordCount();\n int dc = 0;\n\n for (int i = 0; i < c; i++)\n {\n StorageItem meta = null;\n object b = docs.GetObject(i, out meta);\n if (meta != null && meta.isDeleted)\n Delete(meta.key);\n else\n {\n if (b != null)\n {\n object obj = b;\n Type t = obj.GetType();\n if (t == typeof(View_delete))\n {\n View_delete vd = (View_delete)obj;\n if (vd.Viewname.ToLower() == this._view.Name.ToLower())\n ViewDelete(vd.Filter);\n }\n else if (t == typeof(View_insert))\n {\n View_insert vi = (View_insert)obj;\n if (vi.Viewname.ToLower() == this._view.Name.ToLower())\n ViewInsert(vi.ID, vi.RowObject);\n }\n else if (t.IsSubclassOf(basetype) || t == basetype)\n {\n var m = _insertmethod.MakeGenericMethod(new Type[] { obj.GetType() });\n m.Invoke(this, new object[] { meta.key, obj });\n dc++;\n }\n }\n else\n _log.Error(\"Doc is null : \" + meta.key);\n }\n }\n _log.Debug(\"Documents processed = \" + dc);\n _log.Debug(\"rebuild view '\" + _view.Name + \"' done (s) = \" + FastDateTime.Now.Subtract(dt).TotalSeconds);\n\n // write version.dat file when done\n File.WriteAllText(_Path + _view.Name + \".version\", _view.Version.ToString());\n }\n catch (Exception ex)\n {\n _log.Error(\"Rebuilding View failed : \" + _view.Name, ex);\n }\n _rebuilding = false;\n InvalidateCaches();\n }\n\n private object CreateObject(byte[] b)\n {\n if (b[0] < 32)\n return BJSON.ToObject(b);\n else\n return JSON.ToObject(Encoding.ASCII.GetString(b));\n }\n\n private void CreateLoadIndexes()\n {\n int i = 0;\n _indexes.Add(_docid, new TypeIndexes(_Path, _docid, 16));\n // load indexes\n foreach (var c in _schema.Columns)\n {\n if (c.Key != \"docid\")\n _indexes.Add(_schema.Columns[i].Key,\n CreateIndex(\n _schema.Columns[i].Key,\n _schema.Columns[i].Value));\n i++;\n }\n i = 0;\n foreach (var c in _datecolumns.Columns)\n {\n _indexes.Add(_datecolumns.Columns[i].Key,\n CreateIndex(\n _datecolumns.Columns[i].Key,\n _datecolumns.Columns[i].Value));\n i++;\n }\n }\n\n private void GenerateSchemaColumns(ViewBase _view)\n {\n // generate schema columns from schema\n _schema = new ViewRowDefinition();\n _datecolumns = new ViewRowDefinition();\n\n foreach (var p in _view.Schema.GetProperties())\n {\n Type t = p.PropertyType;\n\n if (_view.NoIndexingColumns.Contains(p.Name) || _view.NoIndexingColumns.Contains(p.Name.ToLower()))\n {\n _schema.Add(p.Name, typeof(NoIndexing));\n }\n else\n {\n if (p.GetCustomAttributes(typeof(FullTextAttribute), true).Length > 0)\n t = typeof(FullTextString);\n if (_view.FullTextColumns.Contains(p.Name) || _view.FullTextColumns.Contains(p.Name.ToLower()))\n t = typeof(FullTextString);\n if (p.Name != \"docid\")\n _schema.Add(p.Name, t);\n\n if (p.GetCustomAttributes(typeof(CaseInsensitiveAttribute), true).Length > 0)\n _nocase.Add(p.Name, 0);\n if (_view.CaseInsensitiveColumns.Contains(p.Name) || _view.CaseInsensitiveColumns.Contains(p.Name.ToLower()))\n _nocase.Add(p.Name, 0);\n if (t != typeof(FullTextString))\n {\n var a = p.GetCustomAttributes(typeof(StringIndexLength), false);\n if (a.Length > 0)\n {\n byte l = (a[0] as StringIndexLength).Length;\n _idxlen.Add(p.Name, l);\n }\n if (_view.StringIndexLength.ContainsKey(p.Name) || _view.StringIndexLength.ContainsKey(p.Name.ToLower()))\n {\n byte b = 0;\n if (_view.StringIndexLength.TryGetValue(p.Name, out b))\n _idxlen.Add(p.Name, b);\n if (_view.StringIndexLength.TryGetValue(p.Name.ToLower(), out b))\n _idxlen.Add(p.Name, b);\n }\n }\n if (t == typeof(DateTime))\n {\n _datecolumns.Add(p.Name + \"_$Year\", typeof(int));\n _datecolumns.Add(p.Name + \"_$Month\", typeof(int));\n _datecolumns.Add(p.Name + \"_$Day\", typeof(int));\n _datecolumns.Add(p.Name + \"_$Hour\", typeof(int));\n _datecolumns.Add(p.Name + \"_$Minute\", typeof(int));\n }\n }\n }\n\n foreach (var f in _view.Schema.GetFields())\n {\n Type t = f.FieldType;\n if (_view.NoIndexingColumns.Contains(f.Name) || _view.NoIndexingColumns.Contains(f.Name.ToLower()))\n {\n _schema.Add(f.Name, typeof(NoIndexing));\n }\n else\n {\n if (f.GetCustomAttributes(typeof(FullTextAttribute), true).Length > 0)\n t = typeof(FullTextString);\n if (_view.FullTextColumns.Contains(f.Name) || _view.FullTextColumns.Contains(f.Name.ToLower()))\n t = typeof(FullTextString);\n if (f.Name != \"docid\")\n _schema.Add(f.Name, t);\n\n if (f.GetCustomAttributes(typeof(CaseInsensitiveAttribute), true).Length > 0)\n _nocase.Add(f.Name, 0);\n if (_view.CaseInsensitiveColumns.Contains(f.Name) || _view.CaseInsensitiveColumns.Contains(f.Name.ToLower()))\n _nocase.Add(f.Name, 0);\n if (t != typeof(FullTextString))\n {\n var a = f.GetCustomAttributes(typeof(StringIndexLength), false);\n if (a.Length > 0)\n {\n byte l = (a[0] as StringIndexLength).Length;\n _idxlen.Add(f.Name, l);\n }\n if (_view.StringIndexLength.ContainsKey(f.Name) || _view.StringIndexLength.ContainsKey(f.Name.ToLower()))\n {\n byte b = 0;\n if (_view.StringIndexLength.TryGetValue(f.Name, out b))\n _idxlen.Add(f.Name, b);\n if (_view.StringIndexLength.TryGetValue(f.Name.ToLower(), out b))\n _idxlen.Add(f.Name, b);\n }\n }\n if (t == typeof(DateTime))\n {\n _datecolumns.Add(f.Name + \"_$Year\", typeof(int));\n _datecolumns.Add(f.Name + \"_$Month\", typeof(int));\n _datecolumns.Add(f.Name + \"_$Day\", typeof(int));\n _datecolumns.Add(f.Name + \"_$Hour\", typeof(int));\n _datecolumns.Add(f.Name + \"_$Minute\", typeof(int));\n }\n }\n }\n _schema.Add(\"docid\", typeof(Guid));\n\n foreach (var s in _schema.Columns)\n _colnames.Add(s.Key);\n\n // set column index for nocase\n for (int i = 0; i < _colnames.Count; i++)\n {\n int j = 0;\n if (_nocase.TryGetValue(_colnames[i], out j))\n _nocase[_colnames[i]] = i;\n }\n }\n\n private void InsertRowsWithIndexUpdate(Guid guid, List rows)\n {\n if (_isDirty == false)\n WriteDirtyFile();\n\n foreach (var row in rows)\n {\n object[] r = new object[row.Length + 1];\n r[0] = guid;\n Array.Copy(row, 0, r, 1, row.Length);\n byte[] b = BJSON.ToBJSON(r, new BJSONParameters { UseExtensions = false, UseTypedArrays = false });\n\n int rownum = (int)_viewData.WriteRawData(b);\n\n // case insensitve columns here\n foreach (var kv in _nocase)\n row[kv.Value] = (\"\" + row[kv.Value]).ToLowerInvariant();\n\n IndexRow(guid, row, rownum);\n }\n }\n\n private List ExtractRows(List rows)\n {\n List output = new List();\n // reflection match object properties to the schema row\n\n int colcount = _schema.Columns.Count;\n\n foreach (var obj in rows)\n {\n object[] r = new object[colcount];\n Getters[] getters = Reflection.Instance.GetGetters(obj.GetType(), /*true,*/ null);\n\n for (int i = 0; i < colcount; i++)\n {\n var c = _schema.Columns[i];\n foreach (var g in getters)\n {\n if (g.Name == c.Key)\n {\n r[i] = g.Getter(obj);\n break;\n }\n }\n }\n output.Add(r);\n }\n\n return output;\n }\n\n private void IndexRow(Guid docid, object[] row, int rownum)\n {\n int c = _colnames.Count - 1; // skip last docid \n _indexes[_docid].Set(docid, rownum);\n\n for (int i = 0; i < c; i++)\n {\n string colname = _colnames[i];\n object d = row[i];\n var idx = _indexes[colname];\n if (idx != null)\n idx.Set(d, rownum);\n foreach (var dc in _datecolumns.Columns)\n {\n if (dc.Key.ToLower().StartsWith(colname.ToLower() + \"_$\"))\n {\n DateTime dt = (DateTime)d;\n _indexes[colname + \"_$Year\"].Set(dt.Year, rownum);\n _indexes[colname + \"_$Month\"].Set(dt.Month, rownum);\n _indexes[colname + \"_$Day\"].Set(dt.Day, rownum);\n _indexes[colname + \"_$Hour\"].Set(dt.Hour, rownum);\n _indexes[colname + \"_$Minute\"].Set(dt.Minute, rownum);\n break;\n }\n }\n }\n }\n\n private IIndex CreateIndex(string name, Type type)\n {\n if (type == typeof(NoIndexing))\n return new NoIndex();\n\n if (type == typeof(FullTextString))\n return new FullTextIndex(_Path, name, false, true, _viewmanager._tokenizer);\n\n else if (type == typeof(string))\n {\n byte len = Global.DefaultStringKeySize;\n if (_idxlen.TryGetValue(name, out len) == false)\n len = Global.DefaultStringKeySize;\n return new TypeIndexes(_Path, name, len);\n }\n\n else if (type == typeof(bool) || type == typeof(bool?))\n return new BoolIndex(_Path, name, \".idx\");\n\n else if (type.IsEnum)\n return (IIndex)Activator.CreateInstance(\n typeof(EnumIndex<>).MakeGenericType(type),\n new object[] { _Path, name });\n\n else\n return (IIndex)Activator.CreateInstance(\n typeof(TypeIndexes<>).MakeGenericType(type),\n new object[] { _Path, name, Global.DefaultStringKeySize });\n }\n\n private void DeleteRowsWith(Guid guid)\n {\n // find bitmap for guid column\n MGRB gc = QueryColumnExpression(_docid, RDBExpression.Equal, guid);\n _deletedRows.InPlaceOR(gc);\n }\n\n private MGRB QueryColumnExpression(string colname, RDBExpression exp, object from)\n {\n int i = 0;\n if (_nocase.TryGetValue(colname, out i)) // no case query\n return _indexes[colname].Query(exp, (\"\" + from).ToLowerInvariant(), _viewData.Count());\n else\n return _indexes[colname].Query(exp, from, _viewData.Count());\n }\n\n private MGRB QueryColumnExpressionFromTo(string colname, object from, object to)\n {\n int i = 0;\n if (_nocase.TryGetValue(colname, out i)) // no case query\n return _indexes[colname].Query((\"\" + from).ToLowerInvariant(), (\"\" + to).ToLowerInvariant(), _viewData.Count());\n else\n return _indexes[colname].Query(from, to, _viewData.Count());\n }\n #endregion\n\n internal int Count(Expression> filter)\n {\n int totcount = 0;\n DateTime dt = FastDateTime.Now;\n if (filter == null)\n totcount = internalCount();\n else\n {\n var ba = GenerateBitmap(filter);\n totcount = (int)ba.CountOnes();\n }\n _log.Debug(\"Count items = \" + totcount);\n _log.Debug(\"Count time (ms) : \" + FastDateTime.Now.Subtract(dt).TotalMilliseconds);\n return totcount;\n }\n\n internal int Count(string filter)\n {\n int totcount = 0;\n DateTime dt = FastDateTime.Now;\n filter = filter.Trim();\n if (filter == null || filter == \"\")\n totcount = internalCount();\n else\n {\n _log.Debug(\"Count filter : \" + filter);\n var ba = GenerateBitmap(filter);\n totcount = (int)ba.CountOnes();\n }\n _log.Debug(\"Count items = \" + totcount);\n _log.Debug(\"Count time (ms) : \" + FastDateTime.Now.Subtract(dt).TotalMilliseconds);\n return totcount;\n }\n\n private MGRB GenerateBitmap(Expression> filter)\n {\n MGRB ba = null;\n //new MGRB();\n // check query cache\n //_queryCache.TryGetValue(filter, out ba);// FIX : showing incorrect results\n if (ba == null)\n {\n ba = new MGRB();\n QueryVisitor qv = new QueryVisitor(QueryColumnExpression, QueryColumnExpressionFromTo);\n qv.Visit(filter);\n var delbits = _deletedRows.GetBits();\n if (qv._bitmap.Count > 0)\n {\n MGRB qbits = (MGRB)qv._bitmap.Pop();\n ba = qbits.AndNot(delbits);\n }\n else if (qv._stack.Count > 0)\n {\n var val = Convert.ToBoolean(qv._stack.Pop());\n if (val == true)\n ba = new MGRB().Not(_viewData.Count()).AndNot(delbits);\n }\n //_queryCache.Add(filter, ba);\n }\n else\n {\n _log.Debug(\" found in cache\");\n ba = ba.Copy();\n }\n\n return ba;\n }\n\n private MGRB GenerateBitmap(string filter)\n {\n MGRB ba = null;\n //new MGRB();\n // check query cache\n //_queryCache.TryGetValue(filter, out ba);// FIX : showing incorrect results\n if (ba == null)\n {\n ba = new MGRB();\n LambdaExpression le = null;\n //if (_lambdacache.TryGetValue(filter, out le) == false)\n {\n le = System.Linq.Dynamic.DynamicExpression.ParseLambda(_view.Schema, typeof(bool), filter, null);\n // _lambdacache.Add(filter, le);\n }\n QueryVisitor qv = new QueryVisitor(QueryColumnExpression, QueryColumnExpressionFromTo);\n qv.Visit(le.Body);\n var delbits = _deletedRows.GetBits();\n if (qv._bitmap.Count > 0)\n {\n MGRB qbits = (MGRB)qv._bitmap.Pop();\n ba = qbits.AndNot(delbits);\n }\n else if (qv._stack.Count > 0)\n {\n var val = Convert.ToBoolean(qv._stack.Pop());\n if (val == true)\n ba = new MGRB().Not(_viewData.Count()).AndNot(delbits);\n }\n //_queryCache.Add(filter, ba);\n }\n else\n {\n _log.Debug(\" found in cache\");\n ba = ba.Copy();\n }\n\n return ba;\n }\n\n private int internalCount() // note : don't use for Not() since _deletedRows are excluded\n {\n if (_rebuilding)\n while (_rebuilding)\n Thread.Sleep(10); // wait for rebuild to finish\n int c = _viewData.Count();\n int cc = (int)_deletedRows.GetBits().CountOnes();\n return c - cc;\n }\n\n internal Result QueryWithTypedResult(Expression> filter, int start, int count, string orderby)\n {\n DateTime dt = FastDateTime.Now;\n _log.Debug(\"query : \" + _view.Name);\n\n var ba = GenerateBitmap(filter);\n\n List trows = null;\n if (_viewmanager.inTransaction())\n {\n // query from transactions own data\n tran_data data = null;\n if (_transactions.TryGetValue(Thread.CurrentThread.ManagedThreadId, out data))\n {\n List rrows = new List();\n foreach (var kv in data.rows)\n {\n foreach (var r in kv.Value)\n {\n object o = FastCreateObject(_view.Schema);\n rrows.Add((T)_rowfiller(o, r));\n }\n }\n trows = rrows.FindAll(filter.Compile());\n }\n }\n var order = SortBy(orderby);\n bool desc = false;\n if (orderby.ToLower().Contains(\" desc\"))\n desc = true;\n _log.Debug(\"query bitmap done (ms) : \" + FastDateTime.Now.Subtract(dt).TotalMilliseconds);\n dt = FastDateTime.Now;\n // exec query return rows\n return ReturnRowsTyped(ba, trows, start, count, order, desc);\n }\n\n internal Result QueryWithTypedResult(string filter, int start, int count, string orderby)\n {\n DateTime dt = FastDateTime.Now;\n _log.Debug(\"query : \" + _view.Name);\n _log.Debug(\"query : \" + filter);\n if (orderby != \"\")\n _log.Debug(\"order by : \" + orderby);\n\n MGRB ba = new MGRB();\n var delbits = _deletedRows.GetBits();\n\n if (filter != \"\")\n ba = GenerateBitmap(filter);\n else\n ba = MGRB.Fill(_viewData.Count()).AndNot(delbits);\n\n var order = SortBy(orderby);\n bool desc = false;\n if (orderby.ToLower().Contains(\" desc\"))\n desc = true;\n _log.Debug(\"query bitmap done (ms) : \" + FastDateTime.Now.Subtract(dt).TotalMilliseconds);\n dt = FastDateTime.Now;\n // exec query return rows\n return ReturnRowsTyped(ba, null, start, count, order, desc);\n }\n\n private SafeDictionary> _sortcache = new SafeDictionary>();\n\n internal List SortBy(string sortcol)\n {\n List sortlist = new List();\n if (sortcol == \"\")\n return sortlist;\n string col = \"\";\n foreach (var c in _schema.Columns)\n if (sortcol.ToLower().Contains(c.Key.ToLower()))\n {\n col = c.Key;\n break;\n }\n if (col == \"\")\n {\n _log.Debug(\"sort column not recognized : \" + sortcol);\n return sortlist;\n }\n\n DateTime dt = FastDateTime.Now;\n\n if (_sortcache.TryGetValue(col, out sortlist) == false)\n {\n sortlist = new List();\n int count = _viewData.Count();\n IIndex idx = _indexes[col];\n object[] keys = idx.GetKeys();\n Array.Sort(keys);\n\n foreach (var k in keys)\n {\n var bi = idx.Query(RDBExpression.Equal, k, count).GetBitIndexes();\n foreach (var i in bi)\n sortlist.Add(i);\n }\n _sortcache.Add(col, sortlist);\n }\n _log.Debug(\"Sort column = \" + col + \", time (ms) = \" + FastDateTime.Now.Subtract(dt).TotalMilliseconds);\n return sortlist;\n }\n\n internal object GetAssembly(out string typename)\n {\n typename = _view.Schema.AssemblyQualifiedName;\n return File.ReadAllBytes(_view.Schema.Assembly.Location);\n }\n\n public ViewRowDefinition GetSchema()\n {\n return _schema;\n }\n\n int _lastrownumber = -1;\n object _rowlock = new object();\n internal int NextRowNumber()\n {\n lock (_rowlock)\n {\n if (_lastrownumber == -1)\n _lastrownumber = internalCount();\n return ++_lastrownumber;\n }\n }\n\n internal int ViewDelete(Expression> filter)\n {\n _log.Debug(\"delete : \" + _view.Name);\n\n var ba = GenerateBitmap(filter);\n long count = ba.CountOnes();\n if (count > 0)\n {\n _deletedRows.InPlaceOR(ba);\n _log.Debug(\"Deleted rows = \" + count);\n InvalidateCaches();\n if (_isDirty == false)\n WriteDirtyFile();\n }\n\n return (int)count;\n }\n\n private object _dfile = new object();\n private void WriteDirtyFile()\n {\n lock (_dfile)\n {\n _isDirty = true;\n if (File.Exists(_Path + _dirtyFilename) == false)\n File.WriteAllText(_Path + _dirtyFilename, \"dirty\");\n }\n }\n\n internal int ViewDelete(string filter)\n {\n _log.Debug(\"delete : \" + _view.Name);\n\n long count = 0;\n if (filter != \"\")\n {\n var ba = GenerateBitmap(filter);\n count = ba.CountOnes();\n if (count > 0)\n {\n _deletedRows.InPlaceOR(ba);\n\n InvalidateCaches();\n if (_isDirty == false)\n WriteDirtyFile();\n }\n }\n\n return (int)count;\n }\n\n internal bool ViewInsert(Guid id, object row)\n {\n List l = new List();\n l.Add(row);\n\n var r = ExtractRows(l);\n InsertRowsWithIndexUpdate(id, r);\n if (l.Count > 0)\n InvalidateCaches();\n return true;\n }\n\n private void InvalidateCaches()\n {\n // invalidate sort cache\n _sortcache = new SafeDictionary>();\n // inavidate query cache\n //_queryCache = new SafeDictionary(); // FIX : showing incorrect results\n }\n\n #region [ removed ]\n //SafeDictionary _lambdacache = new SafeDictionary();\n //internal Result Query(string filter, int start, int count)\n //{\n // return Query(filter, start, count, \"\");\n //}\n\n //internal Result Query(Expression> filter, int start, int count)\n //{\n // return Query(filter, start, count, \"\");\n //}\n\n //internal Result Query(Expression> filter, int start, int count, string orderby)\n //{\n // if (filter == null)\n // return Query(start, count);\n\n // DateTime dt = FastDateTime.Now;\n // _log.Debug(\"query : \" + _view.Name);\n\n // MGRB ba = new MGRB();\n\n // _queryCache.TryGetValue(filter, out ba);\n // if (ba == null)\n // {\n // ba = new MGRB();\n // QueryVisitor qv = new QueryVisitor(QueryColumnExpression, QueryColumnExpFromTo);\n // qv.Visit(filter);\n // var delbits = _deletedRows.GetBits();\n // if (qv._bitmap.Count > 0)\n // {\n // MGRB qbits = (MGRB)qv._bitmap.Pop();\n // ba = qbits.AndNot(delbits);\n // }\n // else if (qv._stack.Count > 0)\n // {\n // var val = Convert.ToBoolean(qv._stack.Pop());\n // if (val == true)\n // ba = new MGRB().Not(this.internalCount()).AndNot(delbits);\n // }\n // _queryCache.Add(filter, ba);\n // }\n // else\n // _log.Debug(\" found in cache\");\n\n // List trows = null;\n // if (_viewmanager.inTransaction())\n // {\n // // query from transaction own data\n // tran_data data = null;\n // if (_transactions.TryGetValue(Thread.CurrentThread.ManagedThreadId, out data))\n // {\n // List rrows = new List();\n // foreach (var kv in data.rows)\n // {\n // foreach (var r in kv.Value)\n // {\n // object o = FastCreateObject(_view.Schema);\n // rrows.Add((T)_rowfiller(o, r));\n // }\n // }\n // trows = rrows.FindAll(filter.Compile());\n // }\n // }\n\n // var order = SortBy(orderby);\n // bool desc = false;\n // if (orderby.ToLower().Contains(\" desc\"))\n // desc = true;\n // _log.Debug(\"query bitmap done (ms) : \" + FastDateTime.Now.Subtract(dt).TotalMilliseconds);\n // dt = FastDateTime.Now;\n // // exec query return rows\n // return ReturnRowsObject(ba, trows, start, count, order, desc);\n //}\n\n //internal Result Query2(Expression> filter, int start, int count)\n //{\n // return Query2(filter, start, count, \"\");\n //}\n\n //internal Result Query2(string filter, int start, int count)\n //{\n // return Query2(filter, start, count, \"\");\n //}\n #endregion\n }\n}\n" }, { "path": "RaptorDB/Views/ViewManager.cs", "content": "using RaptorDB.Common;\nusing System;\nusing System.Collections.Generic;\nusing System.Linq.Expressions;\nusing System.Threading;\nusing System.Threading.Tasks;\n\nnamespace RaptorDB.Views\n{\n internal class ViewManager\n {\n public ViewManager(string viewfolder, IDocStorage objstore, IKeyStoreHF kvhf, ITokenizer tokenizer)\n {\n _Path = viewfolder;\n _objectStore = objstore;\n _kvhf = kvhf;\n _tokenizer = tokenizer;\n }\n\n private IKeyStoreHF _kvhf;\n private IDocStorage _objectStore;\n private ILog _log = LogManager.GetLogger(typeof(ViewManager));\n private string _Path = \"\";\n // list of views\n private SafeDictionary _views = new SafeDictionary();\n // primary view list\n private SafeDictionary _primaryView = new SafeDictionary();\n // like primary view list \n private SafeDictionary _otherViewTypes = new SafeDictionary();\n // consistent views\n private SafeDictionary> _consistentViews = new SafeDictionary>();\n // other views type->list of view names to call\n private SafeDictionary> _otherViews = new SafeDictionary>();\n private TaskQueue _que = new TaskQueue();\n private SafeDictionary _transactions = new SafeDictionary();\n internal ITokenizer _tokenizer;\n\n internal int Count(string viewname, string filter)\n {\n ViewHandler view = null;\n // find view from name\n if (_views.TryGetValue(viewname.ToLower(), out view))\n return view.Count(filter);\n\n _log.Error(\"view not found\", viewname);\n return 0;\n }\n\n internal Result Query(string viewname, string filter, int start, int count)\n {\n return Query(viewname, filter, start, count, \"\");\n }\n\n internal Result Query(string viewname, int start, int count)\n {\n ViewHandler view = null;\n // find view from name\n if (_views.TryGetValue(viewname.ToLower(), out view))\n return view.Query(start, count);\n\n _log.Error(\"view not found\", viewname);\n return new Result(false, new Exception(\"view not found : \" + viewname));\n }\n\n internal void Insert(string viewname, Guid docid, T data)\n {\n ViewHandler vman = null;\n // find view from name\n if (_views.TryGetValue(viewname.ToLower(), out vman))\n {\n if (vman._view.isActive == false)\n {\n _log.Debug(\"view is not active, skipping insert : \" + viewname);\n return;\n }\n if (vman._view.BackgroundIndexing)\n _que.AddTask(() => vman.Insert(docid, data));\n else\n vman.Insert(docid, data);\n\n return;\n }\n _log.Error(\"view not found\", viewname);\n }\n\n internal bool InsertTransaction(string viewname, Guid docid, T data)\n {\n ViewHandler vman = null;\n // find view from name\n if (_views.TryGetValue(viewname.ToLower(), out vman))\n {\n if (vman._view.isActive == false)\n {\n _log.Debug(\"view is not active, skipping insert : \" + viewname);\n return false;\n }\n\n return vman.InsertTransaction(docid, data);\n }\n _log.Error(\"view not found\", viewname);\n return false;\n }\n\n internal object Fetch(Guid guid)\n {\n object b = null;\n _objectStore.GetObject(guid, out b);\n\n return b;\n }\n\n internal string GetPrimaryViewForType(Type type)\n {\n string vn = \"\";\n if (type == null || type == typeof(object)) // reached the end\n return vn;\n // find direct\n if (_primaryView.TryGetValue(type, out vn))\n return vn;\n // recurse basetype\n return GetPrimaryViewForType(type.BaseType);\n }\n\n internal List GetOtherViewsList(Type type)\n {\n List list = new List();\n _otherViews.TryGetValue(type, out list);\n return list;\n }\n\n internal string GetViewName(Type type) // used for queries\n {\n string viewname = null;\n // find view from name\n\n viewname = GetPrimaryViewForType(type);\n if (viewname != \"\")\n return viewname;\n\n // search for viewtype here\n if (_otherViewTypes.TryGetValue(type, out viewname))\n return viewname;\n\n return \"\";\n }\n\n internal void RegisterView(View view)\n {\n view.Verify();\n\n ViewHandler vh = null;\n if (_views.TryGetValue(view.Name.ToLower(), out vh))\n {\n _log.Error(\"View already added and exists : \" + view.Name);\n }\n else\n {\n vh = new ViewHandler(_Path, this);\n vh.SetView(view, _objectStore);\n _views.Add(view.Name.ToLower(), vh);\n _otherViewTypes.Add(view.GetType(), view.Name.ToLower());\n\n // add view schema mapping \n _otherViewTypes.Add(view.Schema, view.Name.ToLower());\n\n Type basetype = vh.GetFireOnType();\n if (view.isPrimaryList)\n {\n _primaryView.Add(basetype, view.Name.ToLower());\n }\n else\n {\n if (view.ConsistentSaveToThisView)\n AddToViewList(_consistentViews, basetype, view.Name);\n else\n AddToViewList(_otherViews, basetype, view.Name);\n }\n }\n }\n\n internal void ShutDown()\n {\n _log.Debug(\"View Manager shutdown\");\n _que.Shutdown();\n List tasks = new List();\n // shutdown views\n foreach (var v in _views)\n {\n try\n {\n tasks.Add(Task.Factory.StartNew(() =>\n {\n _log.Debug(\" shutting down view : \" + v.Value._view.Name);\n v.Value.Shutdown();\n }));\n }\n catch (Exception ex)\n {\n _log.Error(ex);\n }\n }\n Task.WaitAll(tasks.ToArray());\n }\n\n internal List GetConsistentViews(Type type)\n {\n List list = new List();\n _consistentViews.TryGetValue(type, out list);\n return list;\n }\n\n private void AddToViewList(SafeDictionary> diclist, Type fireontype, string viewname)\n {\n //foreach (var tn in view.FireOnTypes)\n {\n List list = null;\n Type t = fireontype;// Type.GetType(tn);\n if (diclist.TryGetValue(t, out list))\n list.Add(viewname);\n else\n {\n list = new List();\n list.Add(viewname);\n diclist.Add(t, list);\n }\n }\n }\n\n internal void Delete(Guid docid)\n {\n // remove from all views\n foreach (var v in _views)\n v.Value.Delete(docid);\n }\n\n internal void Rollback(int ID)\n {\n _log.Debug(\"ROLLBACK\");\n // rollback all views with tran id\n foreach (var v in _views)\n v.Value.RollBack(ID);\n\n _transactions.Remove(ID);\n }\n\n internal void Commit(int ID)\n {\n _log.Debug(\"COMMIT\");\n // commit all data in vews with tran id\n foreach (var v in _views)\n v.Value.Commit(ID);\n\n _transactions.Remove(ID);\n }\n\n internal bool isTransaction(string viewname)\n {\n return _views[viewname.ToLower()]._view.TransactionMode;\n }\n\n internal bool inTransaction()\n {\n bool b = false;\n return _transactions.TryGetValue(Thread.CurrentThread.ManagedThreadId, out b);\n }\n\n internal void StartTransaction()\n {\n _transactions.Add(Thread.CurrentThread.ManagedThreadId, false);\n }\n\n internal Result Query(Expression> filter, int start, int count)\n {\n return Query(filter, start, count, \"\");\n }\n\n internal Result Query(Expression> filter, int start, int count, string orderby)\n {\n string view = GetViewName(typeof(T));\n\n ViewHandler vman = null;\n // find view from name\n if (_views.TryGetValue(view.ToLower(), out vman))\n {\n return vman.QueryWithTypedResult(filter, start, count, orderby);\n }\n return new Result(false, new Exception(\"View not found\"));\n }\n\n internal Result Query(string filter, int start, int count)\n {\n return Query(filter, start, count, \"\");\n }\n\n internal Result Query(string filter, int start, int count, string orderby)\n {\n string view = GetViewName(typeof(T));\n\n ViewHandler vman = null;\n // find view from name\n if (_views.TryGetValue(view.ToLower(), out vman))\n {\n return vman.QueryWithTypedResult(filter, start, count, orderby);\n }\n return new Result(false, new Exception(\"View not found\"));\n }\n\n internal int Count(Expression> filter)\n {\n string view = GetViewName(typeof(T));\n\n ViewHandler vman = null;\n // find view from name\n if (_views.TryGetValue(view.ToLower(), out vman))\n {\n return vman.Count(filter);\n }\n return 0;\n }\n\n internal void FreeMemory()\n {\n foreach (var v in _views)\n v.Value.FreeMemory();\n }\n\n internal object GetAssemblyForView(string viewname, out string typename)\n {\n ViewHandler view = null;\n typename = \"\";\n // find view from name\n if (_views.TryGetValue(viewname.ToLower(), out view))\n {\n return view.GetAssembly(out typename);\n }\n return null;\n }\n\n internal List GetViews()\n {\n List o = new List();\n foreach (var i in _views)\n o.Add(i.Value._view);\n return o;\n }\n\n internal ViewRowDefinition GetSchema(string view)\n {\n ViewHandler v = null;\n if (_views.TryGetValue(view.ToLower(), out v))\n {\n return v.GetSchema();\n }\n return null;\n }\n\n internal Result Query(string viewname, string filter, int start, int count, string orderby)\n {\n ViewHandler view = null;\n // find view from name\n if (_views.TryGetValue(viewname.ToLower(), out view))\n return view.Query(filter, start, count, orderby);\n\n _log.Error(\"view not found\", viewname);\n return new Result(false, new Exception(\"view not found : \" + viewname));\n }\n\n internal int ViewDelete(Expression> filter)\n {\n string view = GetViewName(typeof(T));\n\n ViewHandler vman = null;\n // find view from name\n if (_views.TryGetValue(view.ToLower(), out vman))\n {\n return vman.ViewDelete(filter);\n }\n return -1;\n }\n\n internal int ViewDelete(string viewname, string filter)\n {\n ViewHandler view = null;\n // find view from name\n if (_views.TryGetValue(viewname.ToLower(), out view))\n return view.ViewDelete(filter);\n return -1;\n }\n\n internal bool ViewInsert(Guid id, T row)\n {\n string view = GetViewName(typeof(T));\n\n ViewHandler vman = null;\n // find view from name\n if (_views.TryGetValue(view.ToLower(), out vman))\n {\n return vman.ViewInsert(id, row);\n }\n return false;\n }\n\n internal bool ViewInsert(string viewname, Guid id, object row)\n {\n ViewHandler vman = null;\n // find view from name\n if (_views.TryGetValue(viewname.ToLower(), out vman))\n {\n return vman.ViewInsert(id, row);\n }\n return false;\n }\n\n internal IKeyStoreHF GetKVHF()\n {\n return _kvhf;\n }\n }\n}\n" }, { "path": "RaptorDB/Views/apimapper.cs", "content": "using System;\nusing System.Collections.Generic;\nusing System.Linq.Expressions;\n\nnamespace RaptorDB.Views\n{\n internal class apimapper : IMapAPI\n {\n public apimapper(ViewManager man, ViewHandler vhandler)\n {\n _viewmanager = man;\n _viewhandler = vhandler;\n }\n\n ViewManager _viewmanager;\n ViewHandler _viewhandler;\n private ILog _log = LogManager.GetLogger(typeof(apimapper));\n internal Dictionary> emit = new Dictionary>();\n internal Dictionary> emitobj = new Dictionary>();\n internal bool _RollBack = false;\n\n public void Log(string message)\n {\n _log.Debug(message);\n }\n\n public object Fetch(Guid guid)\n {\n return _viewmanager.Fetch(guid);\n }\n\n public void Emit(Guid docid, params object[] data)\n {\n if (data == null)\n return;\n List d = null;\n if (emit.Count == 0)\n {\n d = new List();\n d.Add(data);\n emit.Add(docid, d);\n }\n else\n {\n if (emit.TryGetValue(docid, out d))\n {\n d.Add(data);\n }\n else\n {\n d = new List();\n d.Add(data);\n emit.Add(docid, d);\n }\n }\n }\n\n public void EmitObject(Guid docid, T doc)\n {\n if (doc == null)\n return;\n List d = null;\n if (emitobj.Count == 0)\n {\n d = new List();\n d.Add(doc);\n emitobj.Add(docid, d);\n }\n else\n {\n if (emitobj.TryGetValue(docid, out d))\n {\n d.Add(doc);\n }\n else\n {\n d = new List();\n d.Add(doc);\n emitobj.Add(docid, d);\n }\n }\n }\n\n public void RollBack()\n {\n _RollBack = true;\n }\n\n public int Count(string viewname)\n {\n return _viewmanager.Count(viewname, \"\");\n }\n\n public int Count(string ViewName, string Filter)\n {\n return _viewmanager.Count(ViewName, Filter);\n }\n\n public Result Query(Expression> Filter)\n {\n return _viewmanager.Query(Filter, 0, -1);\n }\n\n public Result Query(Expression> Filter, int start, int count)\n {\n return _viewmanager.Query(Filter, start, count);\n }\n\n public Result Query(string Filter)\n {\n return _viewmanager.Query(Filter, 0, -1);\n }\n\n public Result Query(string Filter, int start, int count)\n {\n return _viewmanager.Query(Filter, start, count);\n }\n\n public int Count(Expression> Filter)\n {\n return _viewmanager.Count(Filter);\n }\n\n public int NextRowNumber()\n {\n return _viewhandler.NextRowNumber();\n }\n\n public Common.IKeyStoreHF GetKVHF()\n {\n return _viewmanager.GetKVHF();\n }\n\n public T Fetch(Guid guid) where T : class\n {\n return (T)Fetch(guid);\n }\n }\n}\n" }, { "path": "RaptorDB/WEB/bundle.css", "content": ".Container.svelte-obficw{width:100%;backface-visibility:hidden;will-change:overflow}.Left.svelte-obficw,.Middle.svelte-obficw{overflow:auto;height:auto;-webkit-overflow-scrolling:touch;-ms-overflow-style:none}.Left.svelte-obficw::-webkit-scrollbar,.Middle.svelte-obficw::-webkit-scrollbar{display:none}.Left.svelte-obficw{width:220px;float:left}.tab-links.svelte-obficw:after{display:block;clear:both;content:\"\"}.tab-links.svelte-obficw li.svelte-obficw{margin:0px 1px;float:left;list-style:none}.tab-links.svelte-obficw{-webkit-padding-start:5px;border-radius:5px 5px 0px 0px;margin-bottom:0px;margin-top:10px}.tab-links.svelte-obficw div.svelte-obficw{padding:9px 15px;display:inline-block;border-radius:5px 5px 0px 0px;background:#333333;font-size:16px;font-weight:600;color:#999999;transition:all linear 0.15s;cursor:pointer}.tab-links.svelte-obficw div #close.svelte-obficw{margin-left:10px;padding:2px}.tab-links.svelte-obficw div #close.svelte-obficw:hover{background:red;color:white;text-align:center;-webkit-border-radius:15px;-moz-border-radius:15px;border-radius:15px;-moz-box-shadow:1px 1px 3px #000;-webkit-box-shadow:1px 1px 3px #000;box-shadow:1px 1px 3px #000}.tab-links.svelte-obficw div.svelte-obficw:hover{background:white;text-decoration:none}li.active.svelte-obficw div.svelte-obficw,li.active.svelte-obficw{background:white;border-radius:5px 5px 0px 0px;border:1px;color:#333}.closeall.svelte-obficw{margin-left:-35px;color:white;padding:8px;float:right !important;font-weight:bolder;text-align:center}.closeall.svelte-obficw:hover{background:red;-webkit-border-radius:15px;-moz-border-radius:15px;border-radius:15px}\nbutton.svelte-g32zaw,a.svelte-g32zaw{font:inherit;border:1px solid #cf0056;background:#cf0056;padding:0.5rem 1rem;color:white;border-radius:5px;box-shadow:1px 1px 3px rgba(0, 0, 0, 0.26);cursor:pointer;text-decoration:none}button.svelte-g32zaw:focus{outline:none}button.svelte-g32zaw:hover,button.svelte-g32zaw:active,a.svelte-g32zaw:hover,a.svelte-g32zaw:active{background:#e40763;border-color:#e40763;box-shadow:1px 1px 8px rgba(77, 51, 51, 0.26)}button.svelte-g32zaw:disabled,button.svelte-g32zaw:disabled:hover,button.svelte-g32zaw:disabled:active{background:#ccc;border-color:#ccc;color:#959595;box-shadow:none;cursor:not-allowed}.success.svelte-g32zaw{background:#01a129;border-color:#01a129}.success.svelte-g32zaw:hover,.success.svelte-g32zaw:active{background:#1ac745;border-color:#1ac745}.outline.svelte-g32zaw{background:transparent;color:#cf0056;box-shadow:none}.outline.svelte-g32zaw:hover,.outline.svelte-g32zaw:active{background:#ffc7de;box-shadow:none}.outline.svelte-g32zaw:disabled,.outline.svelte-g32zaw:disabled:hover,.outline.svelte-g32zaw:disabled:active{background:transparent;color:#ccc}.outline.success.svelte-g32zaw{border-color:#01a129;color:#01a129}.outline.success.svelte-g32zaw:hover,.outline.success.svelte-g32zaw:active{background:#c2ffd1}\nbutton.accordion.svelte-k0merd{background-color:#333333;color:#bbb;cursor:pointer;padding:18px;width:97%;border:none;text-align:left;outline:none;font-size:15px;transition:0.3s;margin-bottom:5px}button.accordion.active.svelte-k0merd{background-color:#666666;color:#eee;margin-bottom:0}button.accordion.svelte-k0merd:hover{background-color:#999999}button.accordion.svelte-k0merd:after{content:\"\\02795\";font-size:13px;color:#666666;float:right;margin-left:5px}button.accordion.active.svelte-k0merd:after{content:\"\\2796\"}div.panel.svelte-k0merd{padding:0 18px;background-color:white;max-height:0;overflow:hidden;transition:0.1s ease-out;opacity:0}div.panel.show.svelte-k0merd{opacity:1;max-height:500px;min-height:100px;padding:5px;width:203px}div.panel.show.svelte-k0merd label.svelte-k0merd{cursor:pointer;display:block;padding:5px}div.panel.show.svelte-k0merd label.svelte-k0merd:hover{background-color:#999999}\n.modal-backdrop.svelte-f1bfsy{position:fixed;top:0;left:0;width:100%;height:100vh;background:rgba(0, 0, 0, 0.55);z-index:10}.modal.svelte-f1bfsy{position:absolute;top:10vh;width:80%;max-height:80vh;background:white;border-radius:5px;z-index:100;box-shadow:0 2px 8px rgba(0, 0, 0, 0.96)}h1.svelte-f1bfsy{padding:1rem;margin:0;border-bottom:1px solid #ccc;font-family:\"Roboto Slab\", sans-serif}.content.svelte-f1bfsy{padding:1rem}footer.svelte-f1bfsy{padding:1rem}@media(min-width: 768px){.modal.svelte-f1bfsy{width:40rem;left:calc(50% - 20rem)}}\n.link.svelte-1bl9b36{color:blue;text-decoration:underline;cursor:pointer}.ListBox.svelte-1bl9b36{border-style:solid;border-color:gray;background-color:white;padding:5px;height:500px;overflow:scroll}.ListBox.svelte-1bl9b36 label.svelte-1bl9b36{color:blue;text-decoration:underline;display:block;padding:2px}.ListBox.svelte-1bl9b36 label.svelte-1bl9b36:hover{background-color:#c0c0c0}\n.link.svelte-m3ntri{color:blue;text-decoration:underline;cursor:pointer;display:block}.link2.svelte-m3ntri{color:blue;text-decoration:underline;cursor:pointer;margin-left:20px}.ListBox.svelte-m3ntri{border-style:solid;border-color:gray;background-color:white;padding:5px;height:500px;overflow:scroll}\n.bold.svelte-yogi1w{font-weight:700}button.svelte-yogi1w{margin-right:20px}\n.link.svelte-k8kdho{color:blue;text-decoration:underline;cursor:pointer;margin-left:20px}\n.link.svelte-6usscg{color:blue;text-decoration:underline;cursor:pointer;display:block}.ListBox.svelte-6usscg{border-style:solid;border-color:gray;background-color:white;padding:5px;height:500px;overflow:scroll}.ListBox.svelte-6usscg label.svelte-6usscg{color:blue;text-decoration:underline;display:block;padding:2px}.ListBox.svelte-6usscg label.svelte-6usscg:hover{background-color:#c0c0c0}\n.link.svelte-15b96z{color:blue;text-decoration:underline;cursor:pointer}.ListBox.svelte-15b96z{border-style:solid;border-color:gray;background-color:white;padding:5px;height:500px;overflow:scroll}.ListBox.svelte-15b96z label.svelte-15b96z{display:block;padding:2px}.ListBox.svelte-15b96z label.svelte-15b96z:hover{background-color:#c0c0c0}\n.bold.svelte-mdntjp{font-weight:700}.ListBox.svelte-mdntjp{border-style:solid;border-color:gray;background-color:white;padding:5px;height:500px;overflow:scroll;height:150px}.collist.svelte-mdntjp{display:block;font-weight:bold}.schema_def.svelte-mdntjp th.svelte-mdntjp{background-color:#cccccc}.schema_def.svelte-mdntjp td.svelte-mdntjp:nth-child(odd){font-weight:bold;text-align:right}.schema_def.svelte-mdntjp td.svelte-mdntjp:nth-child(even){color:green}\n.link.svelte-1qrhxsw{color:blue;text-decoration:underline;cursor:pointer}.cssTbl.svelte-1qrhxsw{margin:0px;padding:3px;font-family:Arial;font-weight:normal}.cssTbl.svelte-1qrhxsw table.svelte-1qrhxsw{border-spacing:0;width:100%;margin:0px;padding:0px}.cssTbl.svelte-1qrhxsw th.svelte-1qrhxsw{background-color:#333333;padding:5px;cursor:pointer;font-weight:normal;color:white}.cssTbl.svelte-1qrhxsw th.svelte-1qrhxsw:hover{color:red}.cssTbl.svelte-1qrhxsw tr.svelte-1qrhxsw:nth-child(odd){background-color:#c0c0c0}.cssTbl.svelte-1qrhxsw tr.svelte-1qrhxsw:nth-child(even){background-color:#ffffff}.cssTbl.svelte-1qrhxsw td.svelte-1qrhxsw{vertical-align:middle;border:1px solid #666666;text-align:left;padding:5px;color:#000000}\n" }, { "path": "RaptorDB/WEB/bundle.js", "content": "var app=function(){\"use strict\";function t(){}const e=t=>t;function n(t,e){for(const n in e)t[n]=e[n];return t}function o(t){return t()}function c(){return Object.create(null)}function l(t){t.forEach(o)}function a(t){return\"function\"==typeof t}function i(t,e){return t!=t?e==e:t!==e||t&&\"object\"==typeof t||\"function\"==typeof t}function r(t,e,n){if(t){const o=s(t,e,n);return t[0](o)}}function s(t,e,o){return t[1]?n({},n(e.$$scope.ctx,t[1](o?o(e):{}))):e.$$scope.ctx}function d(t,e,o,c){return t[1]?n({},n(e.$$scope.changed||{},t[1](c?c(o):{}))):e.$$scope.changed||{}}let u=\"undefined\"!=typeof window?()=>window.performance.now():()=>Date.now(),v=t=>requestAnimationFrame(t);const f=new Set;let p,h=!1;function m(){f.forEach(t=>{t[0](u())||(f.delete(t),t[1]())}),(h=f.size>0)&&v(m)}function g(t,e){t.appendChild(e)}function b(t,e,n){t.insertBefore(e,n||null)}function w(t){t.parentNode.removeChild(t)}function $(t,e){for(let n=0;nt.removeEventListener(e,n,o)}function S(t,e,n){null==n?t.removeAttribute(e):t.setAttribute(e,n)}function j(t,e){e=\"\"+e,t.data!==e&&(t.data=e)}function O(t,e,n){t.style.setProperty(e,n)}function D(t,e){for(let n=0;n>>0}(d)}_${i}`;if(!M[u]){if(!p){const t=x(\"style\");document.head.appendChild(t),p=t.sheet}M[u]=!0,p.insertRule(`@keyframes ${u} ${d}`,p.cssRules.length)}const v=t.style.animation||\"\";return t.style.animation=`${v?`${v}, `:\"\"}${u} ${o}ms linear ${c}ms 1 both`,T+=1,u}function E(t,e){t.style.animation=(t.style.animation||\"\").split(\", \").filter(e?t=>t.indexOf(e)<0:t=>-1===t.indexOf(\"__svelte\")).join(\", \"),e&&!--T&&v(()=>{if(T)return;let t=p.cssRules.length;for(;t--;)p.deleteRule(t);M={}})}function q(t){I=t}function R(t){(function(){if(!I)throw new Error(\"Function called outside component initialization\");return I})().$$.on_mount.push(t)}function F(){const t=I;return(e,n)=>{const o=t.$$.callbacks[e];if(o){const c=V(e,n);o.slice().forEach(e=>{e.call(t,c)})}}}function A(t,e){const n=t.$$.callbacks[e.type];n&&n.slice().forEach(t=>t(e))}const G=[],H=[],z=[],U=[],J=Promise.resolve();let P,K=!1;function X(){K||(K=!0,J.then(Z))}function Q(){return X(),J}function W(t){z.push(t)}function Y(t){U.push(t)}function Z(){const t=new Set;do{for(;G.length;){const t=G.shift();q(t),tt(t.$$)}for(;H.length;)H.pop()();for(let e=0;e{nt.delete(t),o&&(n&&t.d(1),o())}),t.o(e)}}function rt(n,o,c,i){let r=o(n,c),s=i?0:1,d=null,p=null,g=null;function b(){g&&E(n,g)}function w(t,e){const n=t.b-s;return e*=Math.abs(n),{a:s,b:t.b,d:n,duration:e,start:t.start,end:t.start+e,group:t.group}}function $(o){const{delay:c=0,duration:a=300,easing:i=e,tick:$=t,css:x}=r,y={start:u()+c,b:o};o||(y.group=ot,ot.r+=1),d?p=y:(x&&(b(),g=B(n,s,o,a,c,i,x)),o&&$(0,1),d=w(y,a),W(()=>et(n,o,\"start\")),function(t){let e;h||(h=!0,v(m)),new Promise(n=>{f.add(e=[t,n])})}(t=>{if(p&&t>p.start&&(d=w(p,a),p=null,et(n,d.b,\"start\"),x&&(b(),g=B(n,s,d.b,d.duration,0,i,r.css))),d)if(t>=d.end)$(s=d.b,1-s),et(n,d.b,\"end\"),p||(d.b?b():--d.group.r||l(d.group.c)),d=null;else if(t>=d.start){const e=t-d.start;s=d.a+d.d*i(e/d.duration),$(s,1-s)}return!(!d&&!p)}))}return{run(t){a(r)?(P||(P=Promise.resolve()).then(()=>{P=null}),P).then(()=>{r=r(),$(t)}):$(t)},end(){b(),d=p=null}}}function st(t,e){t.d(1),e.delete(t.key)}function dt(t,e,n,o,c,l,a,i,r,s,d,u){let v=t.length,f=l.length,p=v;const h={};for(;p--;)h[t[p].key]=p;const m=[],g=new Map,b=new Map;for(p=f;p--;){const t=u(c,l,p),i=n(t);let r=a.get(i);r?o&&r.p(e,t):(r=s(i,t)).c(),g.set(i,m[p]=r),i in h&&b.set(i,Math.abs(p-h[i]))}const w=new Set,$=new Set;function x(t){at(t,1),t.m(i,d),a.set(t.key,t),d=t.first,f--}for(;v&&f;){const e=m[f-1],n=t[v-1],o=e.key,c=n.key;e===n?(d=e.first,v--,f--):g.has(c)?!a.has(o)||w.has(o)?x(e):$.has(c)?v--:b.get(o)>b.get(c)?($.add(o),x(e)):(w.add(c),v--):(r(n,a),v--)}for(;v--;){const e=t[v];g.has(e.key)||r(e,a)}for(;f;)x(m[f-1]);return m}function ut(t,e,n){-1!==t.$$.props.indexOf(e)&&(t.$$.bound[e]=n,n(t.$$.ctx[e]))}function vt(t,e,n){const{fragment:c,on_mount:i,on_destroy:r,after_update:s}=t.$$;c.m(e,n),W(()=>{const e=i.map(o).filter(a);r?r.push(...e):l(e),t.$$.on_mount=[]}),s.forEach(W)}function ft(t,e){t.$$.fragment&&(l(t.$$.on_destroy),t.$$.fragment.d(e),t.$$.on_destroy=t.$$.fragment=null,t.$$.ctx={})}function pt(e,n,o,a,i,r){const s=I;q(e);const d=n.props||{},u=e.$$={fragment:null,ctx:null,props:r,update:t,not_equal:i,bound:c(),on_mount:[],on_destroy:[],before_update:[],after_update:[],context:new Map(s?s.$$.context:[]),callbacks:c(),dirty:null};let v=!1;var f;u.ctx=o?o(e,d,(t,n)=>{u.ctx&&i(u.ctx[t],u.ctx[t]=n)&&(u.bound[t]&&u.bound[t](n),v&&function(t,e){t.$$.dirty||(G.push(t),X(),t.$$.dirty=c()),t.$$.dirty[e]=!0}(e,t))}):d,u.update(),v=!0,l(u.before_update),u.fragment=a(u.ctx),n.target&&(n.hydrate?u.fragment.l((f=n.target,Array.from(f.childNodes))):u.fragment.c(),n.intro&&at(e.$$.fragment),vt(e,n.target,n.anchor),Z()),q(s)}class ht{$destroy(){ft(this,1),this.$destroy=t}$on(t,e){const n=this.$$.callbacks[t]||(this.$$.callbacks[t]=[]);return n.push(e),()=>{const t=n.indexOf(e);-1!==t&&n.splice(t,1)}}$set(){}}function mt(t,e,n){const o=Object.create(t);return o.panel=e[n],o}function gt(t,e,n){const o=Object.create(t);return o.name=e[n],o}function bt(t){var e,n,o,c=t.panel.description;function l(){return t.click_handler_1(t)}return{c(){e=x(\"label\"),n=y(c),S(e,\"id\",t.panel.name),S(e,\"class\",\"svelte-k0merd\"),o=C(e,\"click\",l)},m(t,o){b(t,e,o),g(e,n)},p(e,n){t=n},d(t){t&&w(e),o()}}}function wt(t){var e,n,o,c,l,a,i=t.name;function r(){return t.click_handler(t)}for(var s=t.navdata[t.name],d=[],u=0;uWelcome to the RaptorDB Web Studio\\n\\t\\t\\t You can use this Web interface to query and see configuration information.\\n\\t\\t\\t \",S(e,\"class\",\"tab-content\"),N(e,\"active\",t.active)},m(t,n){b(t,e,n)},p(t,n){t.active&&N(e,\"active\",n.active)},d(t){t&&w(e)}}}function _t(e){var n,o=e.active&&kt(e);return{c(){o&&o.c(),n=_()},m(t,e){o&&o.m(t,e),b(t,n,e)},p(t,e){e.active?o?o.p(t,e):((o=kt(e)).c(),o.m(n.parentNode,n)):o&&(o.d(1),o=null)},i:t,o:t,d(t){o&&o.d(t),t&&w(n)}}}function Ct(t,e,n){let{active:o=!1}=e;return t.$set=(t=>{\"active\"in t&&n(\"active\",o=t.active)}),{active:o}}class St extends ht{constructor(t){super(),pt(this,t,Ct,_t,i,[\"active\"])}get active(){return this.$$.ctx.active}set active(t){this.$set({active:t}),Z()}}function jt(t,e,n){const o=Object.create(t);return o.r=e[n],o}function Ot(t,e,n){const o=Object.create(t);return o.row=e[n],o}function Dt(t,e,n){const o=Object.create(t);return o.col=e[n],o}function Lt(e){var n;return{c(){(n=x(\"p\")).textContent=\"No data\"},m(t,e){b(t,n,e)},p:t,d(t){t&&w(n)}}}function Nt(t){var e,n,o,c,a,i,r,s,d,u,v,f,p,h,m,_,D,L,N,V,I,T,M,B,E,q,R,F,A,G,H,z,U,J,P,K,X,Q,W,Y,Z,tt=t.rows.length,et=t.totalrows.toLocaleString(),nt=t.totalpages.toLocaleString(),ot=[],ct=new Map,lt=t.totalpages.toLocaleString(),at=Object.keys(t.rows[0]);const it=t=>t.col;for(var rt=0;rt\",_=k(),(D=x(\"label\")).textContent=\"Rows per Page :\",L=k(),(N=x(\"label\")).textContent=\"10\",V=k(),(I=x(\"label\")).textContent=\"25\",T=k(),(M=x(\"label\")).textContent=\"50\",B=k(),(E=x(\"label\")).textContent=\"100\",q=k(),R=x(\"div\"),F=x(\"table\"),A=x(\"tr\"),l=0;l\",S(s,\"class\",\"link svelte-1qrhxsw\"),S(m,\"class\",\"link svelte-1qrhxsw\"),O(D,\"padding-left\",\"100px\"),S(N,\"class\",\"link svelte-1qrhxsw\"),O(N,\"padding-left\",\"10px\"),S(I,\"class\",\"link svelte-1qrhxsw\"),O(I,\"padding-left\",\"10px\"),S(M,\"class\",\"link svelte-1qrhxsw\"),O(M,\"padding-left\",\"10px\"),S(E,\"class\",\"link svelte-1qrhxsw\"),O(E,\"padding-left\",\"10px\"),S(r,\"class\",\"pager\"),S(A,\"class\",\"svelte-1qrhxsw\"),S(F,\"class\",\"svelte-1qrhxsw\"),S(R,\"class\",\"cssTbl svelte-1qrhxsw\"),S(U,\"class\",\"link svelte-1qrhxsw\"),S(Y,\"class\",\"link svelte-1qrhxsw\"),S(z,\"class\",\"pager\"),Z=[C(s,\"click\",t.prevpage),C(m,\"click\",t.nextpage),C(N,\"click\",t.page10),C(I,\"click\",t.page25),C(M,\"click\",t.page50),C(E,\"click\",t.page100),C(U,\"click\",t.prevpage),C(Y,\"click\",t.nextpage)]},m(t,l){for(b(t,e,l),g(e,n),g(e,o),g(e,c),g(e,a),b(t,i,l),b(t,r,l),g(r,s),g(r,d),g(r,u),g(u,v),g(u,f),g(u,p),g(r,h),g(r,m),g(r,_),g(r,D),g(r,L),g(r,N),g(r,V),g(r,I),g(r,T),g(r,M),g(r,B),g(r,E),b(t,q,l),b(t,R,l),g(R,F),g(F,A),w=0;wt.r;for(var i=0;i0?Nt:Lt}var c=o(e),l=c(e);return{c(){l.c(),n=_()},m(t,e){l.m(t,e),b(t,n,e)},p(t,e){c===(c=o(e))&&l?l.p(t,e):(l.d(1),(l=c(e))&&(l.c(),l.m(n.parentNode,n)))},i:t,o:t,d(t){l.d(t),t&&w(n)}}}function qt(t,e,n){let{rows:o=[],totalrows:c=1,page:l=1}=e;const a=F();let i,r=10,s=\"\",d=\"\";function u(){a(\"refresh\",{start:(l-1)*r,count:r,sort:s+\" \"+d,page:l})}function v(t){s!==t?(s=t,d=\"ASC\"):d=\"ASC\"===d?\"DESC\":\"ASC\",u()}function f(t){a(\"showdoc\",t)}return t.$set=(t=>{\"rows\"in t&&n(\"rows\",o=t.rows),\"totalrows\"in t&&n(\"totalrows\",c=t.totalrows),\"page\"in t&&n(\"page\",l=t.page)}),t.$$.update=((t={totalrows:1,rowspp:1})=>{(t.totalrows||t.rowspp)&&n(\"totalpages\",i=Math.ceil(c/r))}),{rows:o,totalrows:c,page:l,nextpage:function(){l1&&n(\"page\",l-=1),u()},sortby:v,showdoc:f,page10:function(){n(\"rowspp\",r=10),n(\"page\",l=1),u()},page25:function(){n(\"rowspp\",r=25),n(\"page\",l=1),u()},page50:function(){n(\"rowspp\",r=50),n(\"page\",l=1),u()},page100:function(){n(\"rowspp\",r=100),n(\"page\",l=1),u()},totalpages:i,click_handler:function({col:t}){return v(t)},click_handler_1:function({row:t,r:e}){return f(t[e])}}}class Rt extends ht{constructor(t){super(),pt(this,t,qt,Et,i,[\"rows\",\"totalrows\",\"page\"])}}function Ft(t,e,n){const o=Object.create(t);return o.col=e[n],o}function At(t,e,n){const o=Object.create(t);return o.vn=e[n],o}function Gt(t){var e,n,o,c,a,i,r,s,d,u,v,f,p,h,m,$,y,_,j,L,V,I,T,M,B,E,q,R,F,A,G,H,z,U,J,P,K,X,Q,Y,Z,tt,et,nt,ot,rt,ut,vt=[],ft=new Map,pt=[],ht=new Map,mt=t.viewnames;const gt=t=>t.vn;for(var bt=0;btt.col;for(bt=0;btt.select_change_handler.call(c)),O(c,\"width\",\"200px\"),O(c,\"font-size\",\"larger\"),S(u,\"align\",\"top\"),S(m,\"id\",\"columns\"),O(m,\"height\",\"150px\"),O(m,\"width\",\"150px\"),O(m,\"display\",\"inline-block\"),S(m,\"class\",\"ListBox svelte-m3ntri\"),S(_,\"class\",\"link2 svelte-m3ntri\"),S(L,\"class\",\"link2 svelte-m3ntri\"),S(I,\"class\",\"link2 svelte-m3ntri\"),S(M,\"class\",\"link2 svelte-m3ntri\"),S(E,\"class\",\"link2 svelte-m3ntri\"),S(R,\"class\",\"link2 svelte-m3ntri\"),S(A,\"class\",\"link2 svelte-m3ntri\"),S(H,\"class\",\"link2 svelte-m3ntri\"),O(H,\"float\",\"right\"),S(P,\"id\",\"filter\"),S(P,\"cols\",\"80\"),O(P,\"height\",\"156px\"),O(P,\"resize\",\"none\"),O(P,\"border-color\",\"grey\"),O(P,\"border-width\",\"medium\"),O(X,\"width\",\"200px\"),O(X,\"height\",\"40px\"),O(Y,\"width\",\"200px\"),O(Y,\"height\",\"40px\"),S(e,\"class\",\"tab-content\"),N(e,\"active\",t.active),ut=[C(c,\"change\",t.select_change_handler),C(c,\"change\",t.schemachanged),C(_,\"click\",t.click_handler_1),C(L,\"click\",t.click_handler_2),C(I,\"click\",t.click_handler_3),C(M,\"click\",t.click_handler_4),C(E,\"click\",t.click_handler_5),C(R,\"click\",t.click_handler_6),C(A,\"click\",t.click_handler_7),C(H,\"click\",t.click_handler_8),C(P,\"input\",t.textarea_input_handler),C(X,\"click\",t.click_handler_9),C(Y,\"click\",t.excel)]},m(l,w){for(b(l,e,w),g(e,n),g(e,o),g(e,c),bt=0;bt{yt[a]=null}),lt(),(ot=yt[nt])||(ot=yt[nt]=xt[nt](n)).c(),at(ot,1),ot.m(e,null)),t.active&&N(e,\"active\",n.active)},i(t){rt||(at(ot),rt=!0)},o(t){it(ot),rt=!1},d(t){for(t&&w(e),bt=0;btn=!1)}function r(e){t.datatable_totalrows_binding.call(null,e),o=!0,Y(()=>o=!1)}function s(e){t.datatable_page_binding.call(null,e),c=!0,Y(()=>c=!1)}let d={};void 0!==t.data&&(d.rows=t.data),void 0!==t.totalrows&&(d.totalrows=t.totalrows),void 0!==t.page&&(d.page=t.page);var u=new Rt({props:d});return H.push(()=>ut(u,\"rows\",i)),H.push(()=>ut(u,\"totalrows\",r)),H.push(()=>ut(u,\"page\",s)),u.$on(\"refresh\",t.refresh_handler),u.$on(\"showdoc\",t.showdoc_handler),{c(){a&&a.c(),e=k(),u.$$.fragment.c()},m(t,n){a&&a.m(t,n),b(t,e,n),vt(u,t,n),l=!0},p(t,l){null!=l.data?a?a.p(t,l):((a=Pt(l)).c(),a.m(e.parentNode,e)):a&&(a.d(1),a=null);var i={};!n&&t.data&&(i.rows=l.data),!o&&t.totalrows&&(i.totalrows=l.totalrows),!c&&t.page&&(i.page=l.page),u.$set(i)},i(t){l||(at(u.$$.fragment,t),l=!0)},o(t){it(u.$$.fragment,t),l=!1},d(t){a&&a.d(t),t&&w(e),ft(u,t)}}}function Pt(t){var e,n,o;return{c(){e=x(\"pre\"),n=y(\"Query time (+render) : \"),o=y(t.qtime)},m(t,c){b(t,e,c),g(e,n),g(e,o)},p(t,e){t.qtime&&j(o,e.qtime)},d(t){t&&w(e)}}}function Kt(t){var e,n,o=t.active&&Gt(t);return{c(){o&&o.c(),e=_()},m(t,c){o&&o.m(t,c),b(t,e,c),n=!0},p(t,n){n.active?o?(o.p(t,n),at(o,1)):((o=Gt(n)).c(),at(o,1),o.m(e.parentNode,e)):o&&(ct(),it(o,1,1,()=>{o=null}),lt())},i(t){n||(at(o),n=!0)},o(t){it(o),n=!1},d(t){o&&o.d(t),t&&w(e)}}}function Xt(t,e,n){let{active:o=!1}=e,c=\"\",l=[],a=[],i=null,r=0,s=10,d=\"\",u=\"\",v=1,f=1,p=\"\",h=\"\";function m(){n(\"page\",f=1),n(\"start\",r=0),window.GET(\"RaptorDB/GetSchema?view=\"+c,function(t){var e=[];t.Rows.forEach(t=>e.push(t.ColumnName)),n(\"colnames\",a=e)})}function g(){n(\"errormsg\",p=\"\");var t=\"raptordb/views/\"+c+(\"\"!==d?\"?\"+d:\"\")+\"?start=\"+r+\"?count=\"+s+(\" \"!==u?\"?orderby=\"+u:\"\"),e=(new Date).getTime();window.GET(t,function(t){n(\"totalrows\",v=t.TotalCount),n(\"data\",i=t.Rows),n(\"qtime\",h=(new Date).getTime()-e+\"ms\")},function(t){n(\"errormsg\",p=t),n(\"data\",i=[])})}function b(t){n(\"start\",r=t.detail.start),s=t.detail.count,u=t.detail.sort,g()}function w(t){n(\"filter\",d+=t)}return R(()=>{n(\"errormsg\",p=\"\"),window.GET(\"RaptorDB/GetViews\",function(t){var e=[];t.Rows.forEach(t=>e.push(t.Name)),n(\"viewnames\",l=e),e.length>0&&(n(\"viewname\",c=e[0]),m())})}),t.$set=(t=>{\"active\"in t&&n(\"active\",o=t.active)}),{active:o,viewname:c,viewnames:l,colnames:a,data:i,start:r,filter:d,totalrows:v,page:f,errormsg:p,qtime:h,schemachanged:m,run:g,excel:function(){var t=window.ServerURL+\"RaptorDB/ExcelExport/\"+c+\"?\"+d;window.open(t)},refresh:b,addcolumn:w,showdoc_handler:function(e){A(t,e)},select_change_handler:function(){c=L(this),n(\"viewname\",c),n(\"viewnames\",l)},click_handler:function({col:t}){return w(\" \"+t)},click_handler_1:function(){return w(\" AND\")},click_handler_2:function(){return w(\" OR\")},click_handler_3:function(){return w(\".between(,)\")},click_handler_4:function(){return w(\".in(,)\")},click_handler_5:function(){return w(\".year\")},click_handler_6:function(){return w(\".month\")},click_handler_7:function(){return w(\".day\")},click_handler_8:function(){const t=d=\"\";return n(\"filter\",d),t},textarea_input_handler:function(){d=this.value,n(\"filter\",d)},click_handler_9:function(){n(\"page\",f=1),n(\"start\",r=0),g()},datatable_rows_binding:function(t){n(\"data\",i=t)},datatable_totalrows_binding:function(t){n(\"totalrows\",v=t)},datatable_page_binding:function(t){n(\"page\",f=t)},refresh_handler:function(t){return b(t)}}}class Qt extends ht{constructor(t){super(),pt(this,t,Xt,Kt,i,[\"active\"])}get active(){return this.$$.ctx.active}set active(t){this.$set({active:t}),Z()}}function Wt(t,e,n){const o=Object.create(t);return o.l=e[n],o}function Yt(t){var e,n,o,c,l,a,i,r,s,d,u,v,f,p,h,m,$,_,O,D,L,V,I,T,M,B,E,q,R,F,A,G,H,z,U,J,P,K,X,Q,W,Y,Z,tt,et,nt,ot,ct,lt,at,it,rt,ut,vt,ft,pt,ht,mt,gt,bt,wt,$t,xt,yt,kt,_t,Ct,St,jt,Ot,Dt,Lt,Nt,Vt,It,Tt=t.DataFolderSize.toLocaleString()+\" bytes\",Mt=t.DocumentCount.toLocaleString(),Bt=t.FileCount.toLocaleString(),Et=t.HighFrequncyItems.toLocaleString(),qt=[],Rt=new Map,Ft=t.logs;const At=t=>t.l;for(var Gt=0;Gt{p()}),t.$set=(t=>{\"active\"in t&&n(\"active\",o=t.active)}),{active:o,DataFolderSize:c,DocumentCount:l,FileCount:a,HighFrequncyItems:i,MemoryUsage:r,NumberOfViews:s,OSVersion:d,RaptorDBVersion:u,Uptime:v,logs:f,refresh:p}}class ne extends ht{constructor(t){super(),pt(this,t,ee,te,i,[\"active\"])}get active(){return this.$$.ctx.active}set active(t){this.$set({active:t}),Z()}}function oe(t,e,n){const o=Object.create(t);return o.v=e[n],o}function ce(t){var e,n,o,c,a,i,r,s,d,u,v,f,p,h,m,$,_,D,L,V=t.total>0&&le(t);return{c(){e=x(\"div\"),(n=x(\"label\")).textContent=\"Search:\",o=k(),c=x(\"input\"),a=k(),(i=x(\"button\")).textContent=\"Get\",r=k(),s=x(\"br\"),d=k(),u=x(\"br\"),v=k(),(f=x(\"label\")).textContent=\"Documents found :\",p=k(),h=x(\"label\"),m=y(t.total),$=k(),_=x(\"label\"),D=k(),V&&V.c(),S(n,\"align\",\"top\"),S(c,\"id\",\"search\"),c.autofocus=!0,O(c,\"width\",\"400px\"),O(i,\"width\",\"50px\"),S(f,\"align\",\"top\"),O(h,\"font-weight\",\"700\"),O(_,\"color\",\"red\"),S(_,\"id\",\"err\"),S(e,\"class\",\"tab-content\"),N(e,\"active\",t.active),L=[C(c,\"input\",t.input_input_handler),C(c,\"keypress\",t.keypress_handler),C(i,\"click\",t.docsearch)]},m(l,w){b(l,e,w),g(e,n),g(e,o),g(e,c),c.value=t.find,g(e,a),g(e,i),g(e,r),g(e,s),g(e,d),g(e,u),g(e,v),g(e,f),g(e,p),g(e,h),g(h,m),g(e,$),g(e,_),g(e,D),V&&V.m(e,null),c.focus()},p(t,n){t.find&&c.value!==n.find&&(c.value=n.find),t.total&&j(m,n.total),n.total>0?V?V.p(t,n):((V=le(n)).c(),V.m(e,null)):V&&(V.d(1),V=null),t.active&&N(e,\"active\",n.active)},d(t){t&&w(e),V&&V.d(),l(L)}}}function le(t){var e,n,o,c,a,i,r,s,d,u,v,f,p,h,m,$,_,D,L,N,V,I,T,M,B=[],E=new Map,q=t.versions;const R=t=>t.v;for(var F=0;F\",p=k(),(h=x(\"td\")).innerHTML='
',m=k(),$=x(\"tr\"),_=x(\"td\"),D=x(\"div\"),F=0;F{\"active\"in t&&n(\"active\",c=t.active)}),t.$$.update=((t={total:1,count:1})=>{(t.total||t.count)&&n(\"pages\",o=Math.ceil(s/re))}),{active:c,find:l,page:a,versions:i,jsondata:r,total:s,docsearch:u,next:function(){a1&&n(\"page\",a-=1),u()},showver:v,pages:o,input_input_handler:function(){l=this.value,n(\"find\",l)},keypress_handler:function(t){13==(t.keyCode||t.which)&&u()},click_handler:function({v:t}){return v(t)}}}class de extends ht{constructor(t){super(),pt(this,t,se,ie,i,[\"active\"])}get active(){return this.$$.ctx.active}set active(t){this.$set({active:t}),Z()}}function ue(t){var e,n,o,c,a,i,r,s,d,u,v,f,p=\"\"!==t.doc&&ve(t);return{c(){e=x(\"div\"),(n=x(\"label\")).textContent=\"docid:\",o=k(),c=x(\"input\"),a=k(),(i=x(\"button\")).textContent=\"Get\",r=k(),s=x(\"br\"),d=k(),u=x(\"br\"),v=k(),p&&p.c(),S(n,\"align\",\"top\"),S(c,\"id\",\"docid\"),c.autofocus=!0,O(c,\"width\",\"400px\"),O(i,\"width\",\"50px\"),S(e,\"class\",\"tab-content\"),N(e,\"active\",t.active),f=[C(c,\"input\",t.input_input_handler),C(c,\"keypress\",t.keypress_handler),C(i,\"click\",t.docview)]},m(l,f){b(l,e,f),g(e,n),g(e,o),g(e,c),c.value=t.docid,g(e,a),g(e,i),g(e,r),g(e,s),g(e,d),g(e,u),g(e,v),p&&p.m(e,null),c.focus()},p(t,n){t.docid&&c.value!==n.docid&&(c.value=n.docid),\"\"!==n.doc?p?p.p(t,n):((p=ve(n)).c(),p.m(e,null)):p&&(p.d(1),p=null),t.active&&N(e,\"active\",n.active)},d(t){t&&w(e),p&&p.d(),l(f)}}}function ve(t){var e,n,o,c,l,a,i,r,s,d,u,v,f,p,h;return{c(){e=x(\"div\"),(n=x(\"label\")).textContent=\"Document Revisions :\",o=k(),c=x(\"label\"),l=k(),(a=x(\"label\")).textContent=\"See Revisions\",i=k(),r=x(\"br\"),s=k(),(d=x(\"label\")).textContent=\"Document JSON :\",u=k(),v=x(\"div\"),f=x(\"div\"),p=y(t.doc),S(n,\"align\",\"top\"),O(c,\"font-weight\",\"700\"),S(a,\"class\",\"link svelte-k8kdho\"),O(a,\"margin-left\",\"5px\"),S(f,\"class\",\"JSON\"),S(f,\"id\",\"data\"),S(v,\"class\",\"JSONArea\"),S(e,\"class\",\"disp\"),h=C(a,\"click\",t.showrevs)},m(t,h){b(t,e,h),g(e,n),g(e,o),g(e,c),g(e,l),g(e,a),g(e,i),g(e,r),g(e,s),g(e,d),g(e,u),g(e,v),g(v,f),g(f,p)},p(t,e){t.doc&&j(p,e.doc)},d(t){t&&w(e),h()}}}function fe(e){var n,o=e.active&&ue(e);return{c(){o&&o.c(),n=_()},m(t,e){o&&o.m(t,e),b(t,n,e)},p(t,e){e.active?o?o.p(t,e):((o=ue(e)).c(),o.m(n.parentNode,n)):o&&(o.d(1),o=null)},i:t,o:t,d(t){o&&o.d(t),t&&w(n)}}}function pe(t,e,n){let{active:o=!1,docid:c=null}=e,l=\"\";const a=F();function i(){n(\"doc\",l=\"\"),window.LOAD(\"/raptordb/docget?\"+c,function(t){n(\"doc\",l=t)})}return R(()=>{null!=c&&i()}),t.$set=(t=>{\"active\"in t&&n(\"active\",o=t.active),\"docid\"in t&&n(\"docid\",c=t.docid)}),{active:o,docid:c,doc:l,docview:i,showrevs:function(){a(\"showrevs\",c)},input_input_handler:function(){c=this.value,n(\"docid\",c)},keypress_handler:function(t){13==(t.keyCode||t.which)&&i()}}}class he extends ht{constructor(t){super(),pt(this,t,pe,fe,i,[\"active\",\"docid\"])}get active(){return this.$$.ctx.active}set active(t){this.$set({active:t}),Z()}get docid(){return this.$$.ctx.docid}set docid(t){this.$set({docid:t}),Z()}}function me(t,e,n){const o=Object.create(t);return o.v=e[n],o}function ge(t){var e,n,o,c,a,i,r,s,d,u,v,f,p,h,m,$,_,D=t.versions.length,L=t.versions.length>0&&be(t);return{c(){e=x(\"div\"),(n=x(\"label\")).textContent=\"docid:\",o=k(),c=x(\"input\"),a=k(),(i=x(\"button\")).textContent=\"Get\",r=k(),s=x(\"br\"),d=k(),u=x(\"br\"),v=k(),(f=x(\"label\")).textContent=\"Document Revisions :\",p=k(),h=x(\"label\"),m=y(D),$=k(),L&&L.c(),S(n,\"align\",\"top\"),S(c,\"id\",\"docid\"),c.autofocus=!0,O(c,\"width\",\"400px\"),O(i,\"width\",\"50px\"),S(f,\"align\",\"top\"),O(h,\"font-weight\",\"700\"),S(e,\"class\",\"tab-content\"),N(e,\"active\",t.active),_=[C(c,\"input\",t.input_input_handler),C(c,\"keypress\",t.fillhistory),C(i,\"keypress\",t.keypress_handler)]},m(l,w){b(l,e,w),g(e,n),g(e,o),g(e,c),c.value=t.docid,g(e,a),g(e,i),g(e,r),g(e,s),g(e,d),g(e,u),g(e,v),g(e,f),g(e,p),g(e,h),g(h,m),g(e,$),L&&L.m(e,null),c.focus()},p(t,n){t.docid&&c.value!==n.docid&&(c.value=n.docid),t.versions&&D!==(D=n.versions.length)&&j(m,D),n.versions.length>0?L?L.p(t,n):((L=be(n)).c(),L.m(e,null)):L&&(L.d(1),L=null),t.active&&N(e,\"active\",n.active)},d(t){t&&w(e),L&&L.d(),l(_)}}}function be(t){var e,n,o,c,l,a,i,r,s,d=[],u=new Map,v=t.versions;const f=t=>t.v;for(var p=0;p{null!=c&&i()}),t.$set=(t=>{\"active\"in t&&n(\"active\",o=t.active),\"docid\"in t&&n(\"docid\",c=t.docid)}),{active:o,docid:c,jsondata:l,versions:a,fillhistory:i,showver:r,input_input_handler:function(){c=this.value,n(\"docid\",c)},keypress_handler:function(t){13==(t.keyCode||t.which)&&i()},click_handler:function({v:t}){return r(t.Version)}}}class ye extends ht{constructor(t){super(),pt(this,t,xe,$e,i,[\"active\",\"docid\"])}get active(){return this.$$.ctx.active}set active(t){this.$set({active:t}),Z()}get docid(){return this.$$.ctx.docid}set docid(t){this.$set({docid:t}),Z()}}function ke(t,e,n){const o=Object.create(t);return o.v=e[n],o}function _e(t){var e,n,o,c,a,i,r,s,d,u,v,f,p,h,m,$,_,D,L,V,I,T,M,B,E,q,R,F,A,G,H,z,U,J,P,K,X,Q,W,Y,Z,tt,et,nt,ot,ct,lt,at,it,rt=[],ut=new Map,vt=t.keys;const ft=t=>t.v;for(var pt=0;pt\",Q=k(),(W=x(\"td\")).innerHTML='
',Y=k(),Z=x(\"tr\"),tt=x(\"td\"),et=x(\"div\"),pt=0;pt{u()}),t.$set=(t=>{\"active\"in t&&n(\"active\",c=t.active)}),t.$$.update=((t={total:1,count:1})=>{(t.total||t.count)&&n(\"pages\",o=Math.ceil(s/je))}),{active:c,key:l,keys:a,json:i,total:s,page:d,getall:u,showkey:v,next:function(){d1&&n(\"page\",d-=1),u()},pages:o,input_input_handler:function(){l=this.value,n(\"key\",l)},keypress_handler:function(t){13==(t.keyCode||t.which)&&v(l)},click_handler:function(){return v(l)},click_handler_1:function({v:t}){return v(t)}}}class De extends ht{constructor(t){super(),pt(this,t,Oe,Se,i,[\"active\"])}get active(){return this.$$.ctx.active}set active(t){this.$set({active:t}),Z()}}function Le(t,e,n){const o=Object.create(t);return o.fc=e[n],o}function Ne(t,e,n){const o=Object.create(t);return o.fc=e[n],o}function Ve(t,e,n){const o=Object.create(t);return o.fc=e[n],o}function Ie(t,e,n){const o=Object.create(t);return o.fc=e[n],o}function Te(t,e,n){const o=Object.create(t);return o.sc=e[n],o}function Me(t,e,n){const o=Object.create(t);return o.vn=e[n],o}function Be(t){var e,n,o,c,a,i,r,s,d,u,v,f,p,h,m,$,_,L,V,I,T,M,B,E,q,R,F,A,G,H,z,U,J,P,K,X,Q,Y,Z,tt,et,nt,ot,ct,lt,at,it,rt,ut,vt,ft,pt,ht,mt,gt,bt,wt,$t,xt,yt,kt,_t,Ct,St,jt,Ot,Dt,Lt,Nt,Vt,It,Tt,Mt,Bt,Et,qt,Rt,Ft,At,Gt,Ht,zt,Ut,Jt,Pt,Kt,Xt,Qt,Wt,Yt,Zt,te,ee,ne,oe,ce,le,ae=[],ie=new Map,re=[],se=new Map,de=[],ue=new Map,ve=[],fe=new Map,pe=[],he=new Map,me=[],ge=new Map,be=t.viewnames;const we=t=>t.vn;for(var $e=0;$et.sc;for($e=0;$et.fc;for($e=0;$et.fc;for($e=0;$et.fc;for($e=0;$et.fc;for($e=0;$eFull Text Columns : Case Insensitive Columns : String Index Length : No Indexing Columns :\",Jt=k(),Pt=x(\"tr\"),Kt=x(\"td\"),Xt=x(\"div\"),$e=0;$et.select_change_handler.call(c)),O(c,\"width\",\"200px\"),O(c,\"font-size\",\"larger\"),S(p,\"class\",\"bold svelte-mdntjp\"),S(I,\"class\",\"bold svelte-mdntjp\"),S(F,\"class\",\"bold svelte-mdntjp\"),S(P,\"class\",\"bold svelte-mdntjp\"),S(et,\"class\",\"bold svelte-mdntjp\"),S(rt,\"class\",\"bold svelte-mdntjp\"),S(gt,\"class\",\"bold svelte-mdntjp\"),S(_t,\"class\",\"bold svelte-mdntjp\"),S(Nt,\"class\",\"bold svelte-mdntjp\"),S(Rt,\"class\",\"svelte-mdntjp\"),S(At,\"class\",\"svelte-mdntjp\"),S(qt,\"class\",\"schema_def svelte-mdntjp\"),S(Xt,\"class\",\"ListBox svelte-mdntjp\"),S(Kt,\"width\",\"25%\"),S(Yt,\"class\",\"ListBox svelte-mdntjp\"),S(Wt,\"width\",\"25%\"),S(ee,\"class\",\"ListBox svelte-mdntjp\"),S(te,\"width\",\"25%\"),S(ce,\"class\",\"ListBox svelte-mdntjp\"),S(oe,\"width\",\"25%\"),S(zt,\"width\",\"100%\"),S(u,\"class\",\"View\"),S(e,\"class\",\"tab-content\"),N(e,\"active\",t.active),le=[C(c,\"change\",t.select_change_handler),C(c,\"change\",t.schemachanged)]},m(l,w){for(b(l,e,w),g(e,n),g(e,o),g(e,c),$e=0;$e{window.GET(\"RaptorDB/GetViews\",function(t){var e=[];t.Rows.forEach(t=>e.push(t.Name)),n(\"viewnames\",l=e),e.length>0&&(n(\"viewname\",c=e[0]),$())})}),t.$set=(t=>{\"active\"in t&&n(\"active\",o=t.active)}),{active:o,viewname:c,viewnames:l,schema:a,FullTextColumns:i,CaseInsensitiveColumns:r,StringIndexLength:s,NoIndexingColumns:d,Name:u,Description:v,Version:f,isPrimaryList:p,isActive:h,DeleteBeforeInsert:m,BackgroundIndexing:g,ConsistentSaveToThisView:b,TransactionMode:w,schemachanged:$,select_change_handler:function(){c=L(this),n(\"viewname\",c),n(\"viewnames\",l)}}}class Ue extends ht{constructor(t){super(),pt(this,t,ze,He,i,[\"active\"])}get active(){return this.$$.ctx.active}set active(t){this.$set({active:t}),Z()}}function Je(t){var e,n,o,c,l,a;return{c(){e=x(\"div\"),(n=x(\"h2\")).textContent=\"raptordb.config\",o=k(),c=x(\"div\"),l=x(\"pre\"),a=y(t.configs),S(l,\"class\",\"JSON\"),S(c,\"class\",\"JSONArea\"),S(e,\"class\",\"tab-content\"),N(e,\"active\",t.active)},m(t,i){b(t,e,i),g(e,n),g(e,o),g(e,c),g(c,l),g(l,a)},p(t,n){t.configs&&j(a,n.configs),t.active&&N(e,\"active\",n.active)},d(t){t&&w(e)}}}function Pe(e){var n,o=e.active&&Je(e);return{c(){o&&o.c(),n=_()},m(t,e){o&&o.m(t,e),b(t,n,e)},p(t,e){e.active?o?o.p(t,e):((o=Je(e)).c(),o.m(n.parentNode,n)):o&&(o.d(1),o=null)},i:t,o:t,d(t){o&&o.d(t),t&&w(n)}}}function Ke(t,e,n){let{active:o=!1}=e,c=\"\";return R(()=>{window.LOAD(\"/raptordb/action?getconfigs\",function(t){n(\"configs\",c=t)})}),t.$set=(t=>{\"active\"in t&&n(\"active\",o=t.active)}),{active:o,configs:c}}class Xe extends ht{constructor(t){super(),pt(this,t,Ke,Pe,i,[\"active\"])}get active(){return this.$$.ctx.active}set active(t){this.$set({active:t}),Z()}}function Qe(t){const e=t-1;return e*e*e+1}function We(t,{delay:e=0,duration:n=400}){const o=+getComputedStyle(t).opacity;return{delay:e,duration:n,css:t=>`opacity: ${t*o}`}}function Ye(t,{delay:e=0,duration:n=400,easing:o=Qe,x:c=0,y:l=0,opacity:a=0}){const i=getComputedStyle(t),r=+i.opacity,s=\"none\"===i.transform?\"\":i.transform,d=r*(1-a);return{delay:e,duration:n,easing:o,css:(t,e)=>`\\n\\t\\t\\ttransform: ${s} translate(${(1-t)*c}px, ${(1-t)*l}px);\\n\\t\\t\\topacity: ${r-d*e}`}}function Ze(t){var e,n,o,c;const l=t.$$slots.default,a=r(l,t,null);return{c(){e=x(\"button\"),a&&a.c(),S(e,\"class\",n=t.mode+\" \"+t.color+\" svelte-g32zaw\"),S(e,\"type\",t.type),e.disabled=t.disabled,c=C(e,\"click\",t.click_handler)},l(t){a&&a.l(button_nodes)},m(t,n){b(t,e,n),a&&a.m(e,null),o=!0},p(t,c){a&&a.p&&t.$$scope&&a.p(d(l,c,t,null),s(l,c,null)),o&&!t.mode&&!t.color||n===(n=c.mode+\" \"+c.color+\" svelte-g32zaw\")||S(e,\"class\",n),o&&!t.type||S(e,\"type\",c.type),o&&!t.disabled||(e.disabled=c.disabled)},i(t){o||(at(a,t),o=!0)},o(t){it(a,t),o=!1},d(t){t&&w(e),a&&a.d(t),c()}}}function tn(t){var e,n;const o=t.$$slots.default,c=r(o,t,null);return{c(){e=x(\"a\"),c&&c.c(),S(e,\"href\",t.href),S(e,\"class\",\"svelte-g32zaw\")},l(t){c&&c.l(a_nodes)},m(t,o){b(t,e,o),c&&c.m(e,null),n=!0},p(t,l){c&&c.p&&t.$$scope&&c.p(d(o,l,t,null),s(o,l,null)),n&&!t.href||S(e,\"href\",l.href)},i(t){n||(at(c,t),n=!0)},o(t){it(c,t),n=!1},d(t){t&&w(e),c&&c.d(t)}}}function en(t){var e,n,o,c,l=[tn,Ze],a=[];function i(t){return t.href?0:1}return e=i(t),n=a[e]=l[e](t),{c(){n.c(),o=_()},m(t,n){a[e].m(t,n),b(t,o,n),c=!0},p(t,c){var r=e;(e=i(c))===r?a[e].p(t,c):(ct(),it(a[r],1,1,()=>{a[r]=null}),lt(),(n=a[e])||(n=a[e]=l[e](c)).c(),at(n,1),n.m(o.parentNode,o))},i(t){c||(at(n),c=!0)},o(t){it(n),c=!1},d(t){a[e].d(t),t&&w(o)}}}function nn(t,e,n){let{type:o=\"button\",href:c=null,mode:l=null,color:a=null,disabled:i=!1}=e,{$$slots:r={},$$scope:s}=e;return t.$set=(t=>{\"type\"in t&&n(\"type\",o=t.type),\"href\"in t&&n(\"href\",c=t.href),\"mode\"in t&&n(\"mode\",l=t.mode),\"color\"in t&&n(\"color\",a=t.color),\"disabled\"in t&&n(\"disabled\",i=t.disabled),\"$$scope\"in t&&n(\"$$scope\",s=t.$$scope)}),{type:o,href:c,mode:l,color:a,disabled:i,click_handler:function(e){A(t,e)},$$slots:r,$$scope:s}}class on extends ht{constructor(t){super(),pt(this,t,nn,en,i,[\"type\",\"href\",\"mode\",\"color\",\"disabled\"])}}const cn=()=>({}),ln=()=>({});function an(t){var e;return{c(){e=y(\"Close\")},m(t,n){b(t,e,n)},d(t){t&&w(e)}}}function rn(t){var e,n,o,c,l,a,i,u,v,f,p,h,m;const $=t.$$slots.default,_=r($,t,null),O=t.$$slots.footer,D=r(O,t,ln);var L=new on({props:{$$slots:{default:[an]},$$scope:{ctx:t}}});return L.$on(\"click\",t.closeModal),{c(){e=x(\"div\"),o=k(),c=x(\"div\"),l=x(\"h1\"),a=y(t.title),i=k(),u=x(\"div\"),_&&_.c(),v=k(),f=x(\"footer\"),D||L.$$.fragment.c(),D&&D.c(),S(e,\"class\",\"modal-backdrop svelte-f1bfsy\"),S(l,\"class\",\"svelte-f1bfsy\"),S(u,\"class\",\"content svelte-f1bfsy\"),S(f,\"class\",\"svelte-f1bfsy\"),S(c,\"class\",\"modal svelte-f1bfsy\"),m=C(e,\"click\",t.closeModal)},l(t){_&&_.l(div1_nodes),D&&D.l(footer_nodes)},m(t,n){b(t,e,n),b(t,o,n),b(t,c,n),g(c,l),g(l,a),g(c,i),g(c,u),_&&_.m(u,null),g(c,v),g(c,f),D?D.m(f,null):vt(L,f,null),h=!0},p(t,e){if(h&&!t.title||j(a,e.title),_&&_.p&&t.$$scope&&_.p(d($,e,t,null),s($,e,null)),!D){var n={};t.$$scope&&(n.$$scope={changed:t,ctx:e}),L.$set(n)}D&&D.p&&t.$$scope&&D.p(d(O,e,t,cn),s(O,e,ln))},i(t){h||(W(()=>{n||(n=rt(e,We,{},!0)),n.run(1)}),at(_,t),at(L.$$.fragment,t),at(D,t),W(()=>{p||(p=rt(c,Ye,{y:300,duration:200},!0)),p.run(1)}),h=!0)},o(t){n||(n=rt(e,We,{},!1)),n.run(0),it(_,t),it(L.$$.fragment,t),it(D,t),p||(p=rt(c,Ye,{y:300,duration:200},!1)),p.run(0),h=!1},d(t){t&&(w(e),n&&n.end(),w(o),w(c)),_&&_.d(t),D||ft(L),D&&D.d(t),t&&p&&p.end(),m()}}}function sn(t,e,n){let{title:o}=e;const c=F();let{$$slots:l={},$$scope:a}=e;return t.$set=(t=>{\"title\"in t&&n(\"title\",o=t.title),\"$$scope\"in t&&n(\"$$scope\",a=t.$$scope)}),{title:o,closeModal:function(){c(\"cancel\")},$$slots:l,$$scope:a}}class dn extends ht{constructor(t){super(),pt(this,t,sn,rn,i,[\"title\"])}}function un(t,e,n){const o=Object.create(t);return o.tab=e[n],o}function vn(t){var e,n;function o(){return t.click_handler(t)}return{c(){(e=x(\"label\")).textContent=\"X\",S(e,\"id\",\"close\"),S(e,\"class\",\"svelte-obficw\"),n=C(e,\"click\",o)},m(t,n){b(t,e,n)},p(e,n){t=n},d(t){t&&w(e),n()}}}function fn(t,e){var n,o,c,l,a,i,r,s,d=e.tab.title,u=\"Help\"!==e.tab.title&&vn(e);function v(){return e.click_handler_1(e)}return{key:t,first:null,c(){n=x(\"li\"),o=x(\"div\"),c=x(\"label\"),l=y(d),a=k(),u&&u.c(),i=k(),S(o,\"class\",\"svelte-obficw\"),S(n,\"id\",r=e.tab.id),S(n,\"class\",\"svelte-obficw\"),N(n,\"active\",e.tab.id===e.activetabid),s=C(n,\"click\",v),this.first=n},m(t,e){b(t,n,e),g(n,o),g(o,c),g(c,l),g(o,a),u&&u.m(o,null),g(n,i)},p(t,c){e=c,t.tabs&&d!==(d=e.tab.title)&&j(l,d),\"Help\"!==e.tab.title?u||((u=vn(e)).c(),u.m(o,null)):u&&(u.d(1),u=null),t.tabs&&r!==(r=e.tab.id)&&S(n,\"id\",r),(t.tabs||t.activetabid)&&N(n,\"active\",e.tab.id===e.activetabid)},d(t){t&&w(n),u&&u.d(),s()}}}function pn(t){var e,n=new dn({props:{title:t.title,$$slots:{default:[wn],footer:[bn]},$$scope:{ctx:t}}});return n.$on(\"cancel\",t.cancel_handler),{c(){n.$$.fragment.c()},m(t,o){vt(n,t,o),e=!0},p(t,e){var o={};t.title&&(o.title=e.title),(t.$$scope||t.modalyes)&&(o.$$scope={changed:t,ctx:e}),n.$set(o)},i(t){e||(at(n.$$.fragment,t),e=!0)},o(t){it(n.$$.fragment,t),e=!1},d(t){ft(n,t)}}}function hn(t){var e,n=new on({props:{type:\"button\",$$slots:{default:[mn]},$$scope:{ctx:t}}});return n.$on(\"click\",t.click_handler_2),{c(){n.$$.fragment.c()},m(t,o){vt(n,t,o),e=!0},i(t){e||(at(n.$$.fragment,t),e=!0)},o(t){it(n.$$.fragment,t),e=!1},d(t){ft(n,t)}}}function mn(t){var e;return{c(){e=y(\"Yes\")},m(t,n){b(t,e,n)},d(t){t&&w(e)}}}function gn(t){var e;return{c(){e=y(\"Cancel\")},m(t,n){b(t,e,n)},d(t){t&&w(e)}}}function bn(t){var e,n,o,c=t.modalyes&&hn(t),l=new on({props:{type:\"button\",mode:\"outline\",$$slots:{default:[gn]},$$scope:{ctx:t}}});return l.$on(\"click\",t.click_handler_3),{c(){e=x(\"div\"),c&&c.c(),n=k(),l.$$.fragment.c(),S(e,\"slot\",\"footer\")},m(t,a){b(t,e,a),c&&c.m(e,null),g(e,n),vt(l,e,null),o=!0},p(t,o){o.modalyes?c?at(c,1):((c=hn(o)).c(),at(c,1),c.m(e,n)):c&&(ct(),it(c,1,1,()=>{c=null}),lt());var a={};t.$$scope&&(a.$$scope={changed:t,ctx:o}),l.$set(a)},i(t){o||(at(c),at(l.$$.fragment,t),o=!0)},o(t){it(c),it(l.$$.fragment,t),o=!1},d(t){t&&w(e),c&&c.d(),ft(l)}}}function wn(e){return{c:t,m:t,p:t,i:t,o:t,d:t}}function $n(t){var e,n,o,c,l,a,i,r,s,d,u,v,f,p=[],h=new Map,m=new yt({});m.$on(\"navclick\",t.navclick_handler);var $=t.tabs;const y=t=>t.tab.id;for(var j=0;j<$.length;j+=1){let e=un(t,$,j),n=y(e);h.set(n,p[j]=fn(n,e))}var O=t.showmsg&&pn(t);return{c(){for(e=x(\"div\"),n=x(\"div\"),m.$$.fragment.c(),o=k(),c=x(\"div\"),l=x(\"ul\"),(a=x(\"span\")).textContent=\"X\",i=k(),j=0;j{O=null}),lt())},i(t){v||(at(m.$$.fragment,t),at(O),v=!0)},o(t){it(m.$$.fragment,t),it(O),v=!1},d(t){for(t&&w(e),ft(m),j=0;j{var o=function(t,e,n){var o=null;switch(t){case\"help\":o=new St({target:document.getElementById(e)});break;case\"query\":(o=new Qt({target:document.getElementById(e)})).$on(\"showdoc\",t=>{r({name:\"docview\",description:\"View\"},t.detail)});break;case\"sysinfo\":o=new ne({target:document.getElementById(e)});break;case\"docsearch\":o=new de({target:document.getElementById(e)});break;case\"docview\":(o=new he({target:document.getElementById(e),props:{docid:n||null}})).$on(\"showrevs\",t=>{r({name:\"dochistory\",description:\"History\"},t.detail)});break;case\"dochistory\":o=new ye({target:document.getElementById(e),props:{docid:n||null}});break;case\"hfbrowser\":o=new De({target:document.getElementById(e)});break;case\"schema\":o=new Ue({target:document.getElementById(e)});break;case\"sysconfig\":o=new Xe({target:document.getElementById(e)})}return o}(t.name,i,e);n(\"tabs\",c=[...c,{title:t.description,id:i,active:!0,obj:o}]),d(i)})}function s(t){r(t)}function d(t){n(\"activetabid\",o=t),c.forEach(e=>{e.active=e.id==t,e.obj.active=e.active}),n(\"tabs\",c)}function u(t){var e=c.findIndex(e=>e.id===t);d(c[--e].id),function(t){var e=c.find(e=>e.id===t);null!==e&&null!==e.obj&&(e.obj.$destroy(),n(\"tabs\",c=c.filter(e=>e.id!==t)),document.querySelector(\"#mainid #\"+t).remove())}(t)}function v(){n(\"showmsg\",l=!1),c.forEach(t=>{\"Help\"!==t.title&&u(t.id)})}return R(()=>{r({name:\"help\",description:\"Help\"}),Q().then(()=>d(c[0].id))}),{activetabid:o,tabs:c,showmsg:l,title:a,modalyes:i,showtab:s,changetab:d,closetab:u,closealltabs:v,closeall:function(){n(\"showmsg\",l=!0),n(\"modalyes\",i=!1),1==c.length?n(\"title\",a=\"No Tabs to close\"):(n(\"title\",a=\"Do you want to close all the tabs?\"),n(\"modalyes\",i=!0))},navclick_handler:function(t){return s(t.detail)},click_handler:function({tab:t}){return u(t.id)},click_handler_1:function({tab:t}){return d(t.id)},click_handler_2:function(){return v()},click_handler_3:function(){const t=l=!1;return n(\"showmsg\",l),t},cancel_handler:function(){const t=l=!1;return n(\"showmsg\",l),t}}}const yn=new class extends ht{constructor(t){super(),pt(this,t,xn,$n,i,[])}}({target:document.body});return window.ServerURL=document.location.protocol+\"//\"+document.location.hostname+\":\"+document.location.port+\"/\",window.GET=function(t,e,n){var o=new XMLHttpRequest;o.open(\"GET\",window.ServerURL+t),o.onreadystatechange=function(){if(1!==o.readyState&&4===o.readyState)if(o.status<400){var t=null;t=\"\"!==o.responseText?JSON.parse(o.responseText):\"\",e&&e(t)}else 401===o.status?(o.abort(),e&&e(\"\")):null!==n&&n(o.responseText)},o.onerror=function(t){console.log(t)},o.withCredentials=!1,o.send()},window.LOAD=function(t,e,n){var o=new XMLHttpRequest;o.open(\"GET\",window.ServerURL+t),o.onreadystatechange=function(){if(1!==o.readyState&&4===o.readyState)if(o.status<400){var t=null;t=\"\"!==o.responseText?o.responseText:\"\",e&&e(t)}else 401===o.status?(o.abort(),e&&e(\"\")):null!==n&&n(o.responseText)},o.onerror=function(t){console.log(t)},o.withCredentials=!1,o.send()},yn}();" }, { "path": "RaptorDB/WEB/global.css", "content": "body {\n font-family: sans-serif;\n color: #222;\n background: #414141;\n}\n\n.tab-content {\n background-color: white;\n padding: 5px;\n padding-top: 15px;\n min-height: 90vh;\n}\n\n.LogArea {\n border-style: solid;\n border-color: gray;\n background-color: #e0e0e0;\n padding: 5px;\n}\n\n.LogItems {\n font-family: \"Courier New\", monospace;\n white-space: pre-wrap;\n font-size: smaller;\n}\n\n.JSONArea {\n border-style: solid;\n border-color: gray;\n background-color: white;\n padding: 5px;\n min-height: 500px;\n}\n\n.JSON {\n font-family: \"Courier New\", monospace;\n white-space: pre-wrap;\n font-size: smaller;\n}" }, { "path": "RaptorDB/WEB/index.html", "content": "\n\n\t\n\t\n\n\tRaptorDB WebStudio\n\n\t\n\t\n\t\n\n\n\n\t\n\n" }, { "path": "RaptorDB/cron/CronDaemon.cs", "content": "using System;\nusing System.Collections.Generic;\nusing System.Timers;\nusing System.Threading;\n\nnamespace RaptorDB\n{\n public class CronDaemon\n {\n private readonly System.Timers.Timer timer = new System.Timers.Timer(30000);\n private readonly List cron_jobs = new List();\n private DateTime _last= DateTime.Now;\n\n public CronDaemon()\n {\n timer.AutoReset = true;\n timer.Elapsed += timer_elapsed;\n timer.Start();\n }\n\n public void AddJob(string schedule, ThreadStart action)\n {\n var cj = new CronJob(schedule, action);\n cron_jobs.Add(cj);\n }\n\n //public void RemoveJob(string schedule)//, ThreadStart action)\n //{\n // var f = cron_jobs.Find((x) => { return x._cron_schedule._expression == schedule; });\n // if(f!=null)\n // cron_jobs.Remove(f);\n //}\n //public void Start()\n //{\n // timer.Start();\n //}\n\n public void Stop()\n {\n timer.Stop();\n\n foreach (CronJob job in cron_jobs)\n job.abort();\n }\n\n private void timer_elapsed(object sender, ElapsedEventArgs e)\n {\n if (DateTime.Now.Minute != _last.Minute)\n {\n _last = DateTime.Now;\n foreach (CronJob job in cron_jobs)\n job.execute(DateTime.Now);\n }\n }\n }\n}\n" }, { "path": "RaptorDB/cron/CronJob.cs", "content": "using System;\nusing System.Threading;\n\nnamespace RaptorDB\n{\n internal class CronJob\n {\n internal readonly CronSchedule _cron_schedule = new CronSchedule();\n private readonly ThreadStart _thread_start;\n internal Thread _thread;\n\n public CronJob(string schedule, ThreadStart thread_start)\n {\n _cron_schedule = new CronSchedule(schedule);\n _thread_start = thread_start;\n _thread = new Thread(thread_start);\n }\n\n private object _lock = new object();\n public void execute(DateTime date_time)\n {\n lock (_lock)\n {\n if (!_cron_schedule.isTime(date_time))\n return;\n\n if (_thread.ThreadState == ThreadState.Running)\n return;\n\n _thread = new Thread(_thread_start);\n _thread.Start();\n }\n }\n\n public void abort()\n {\n try\n {\n _thread.Abort();\n }\n catch { }\n }\n\n }\n}\n" }, { "path": "RaptorDB/cron/CronSchedule.cs", "content": "using System;\nusing System.Collections.Generic;\nusing System.Text.RegularExpressions;\n\nnamespace RaptorDB\n{\n internal class CronSchedule\n {\n #region Readonly Class Members\n\n readonly static Regex divided_regex = new Regex(@\"(\\*/\\d+)\");\n readonly static Regex range_regex = new Regex(@\"(\\d+\\-\\d+)\\/?(\\d+)?\");\n readonly static Regex wild_regex = new Regex(@\"(\\*)\");\n readonly static Regex list_regex = new Regex(@\"(((\\d+,)*\\d+)+)\");\n readonly static Regex validation_regex = new Regex(divided_regex + \"|\" + range_regex + \"|\" + wild_regex + \"|\" + list_regex);\n\n #endregion\n\n #region Private Instance Members\n\n internal readonly string _expression;\n public List minutes;\n public List hours;\n public List days_of_month;\n public List months;\n public List days_of_week;\n\n #endregion\n\n #region Public Constructors\n\n public CronSchedule()\n {\n }\n\n public CronSchedule(string expressions)\n {\n this._expression = expressions;\n generate();\n }\n\n #endregion\n\n #region Public Methods\n\n private bool isValid()\n {\n return isValid(this._expression);\n }\n\n public bool isValid(string expression)\n {\n MatchCollection matches = validation_regex.Matches(expression);\n return matches.Count > 0;//== 5;\n }\n\n public bool isTime(DateTime date_time)\n {\n return minutes.Contains(date_time.Minute) &&\n hours.Contains(date_time.Hour) &&\n days_of_month.Contains(date_time.Day) &&\n months.Contains(date_time.Month) &&\n days_of_week.Contains((int)date_time.DayOfWeek);\n }\n\n private void generate()\n {\n if (!isValid()) return;\n\n MatchCollection matches = validation_regex.Matches(this._expression);\n\n generate_minutes(matches[0].ToString());\n\n if (matches.Count > 1)\n generate_hours(matches[1].ToString());\n else\n generate_hours(\"*\");\n \n if (matches.Count > 2)\n generate_days_of_month(matches[2].ToString());\n else\n generate_days_of_month(\"*\");\n \n if (matches.Count > 3)\n generate_months(matches[3].ToString());\n else\n generate_months(\"*\");\n \n if (matches.Count > 4)\n generate_days_of_weeks(matches[4].ToString());\n else\n generate_days_of_weeks(\"*\");\n }\n\n private void generate_minutes(string match)\n {\n this.minutes = generate_values(match, 0, 60);\n }\n\n private void generate_hours(string match)\n {\n this.hours = generate_values(match, 0, 24);\n }\n\n private void generate_days_of_month(string match)\n {\n this.days_of_month = generate_values(match, 1, 32);\n }\n\n private void generate_months(string match)\n {\n this.months = generate_values(match, 1, 13);\n }\n\n private void generate_days_of_weeks(string match)\n {\n this.days_of_week = generate_values(match, 0, 7);\n }\n\n private List generate_values(string configuration, int start, int max)\n {\n if (divided_regex.IsMatch(configuration)) return divided_array(configuration, start, max);\n if (range_regex.IsMatch(configuration)) return range_array(configuration);\n if (wild_regex.IsMatch(configuration)) return wild_array(configuration, start, max);\n if (list_regex.IsMatch(configuration)) return list_array(configuration);\n\n return new List();\n }\n\n private List divided_array(string configuration, int start, int max)\n {\n if (!divided_regex.IsMatch(configuration))\n return new List();\n\n List ret = new List();\n string[] split = configuration.Split(\"/\".ToCharArray());\n int divisor = int.Parse(split[1]);\n\n for (int i = start; i < max; ++i)\n if (i % divisor == 0)\n ret.Add(i);\n\n return ret;\n }\n\n private List range_array(string configuration)\n {\n if (!range_regex.IsMatch(configuration))\n return new List();\n\n List ret = new List();\n string[] split = configuration.Split(\"-\".ToCharArray());\n int start = int.Parse(split[0]);\n int end = 0;\n if (split[1].Contains(\"/\"))\n {\n split = split[1].Split(\"/\".ToCharArray());\n end = int.Parse(split[0]);\n int divisor = int.Parse(split[1]);\n\n for (int i = start; i < end; ++i)\n if (i % divisor == 0)\n ret.Add(i);\n return ret;\n }\n else\n end = int.Parse(split[1]);\n\n for (int i = start; i <= end; ++i)\n ret.Add(i);\n\n return ret;\n }\n\n private List wild_array(string configuration, int start, int max)\n {\n if (!wild_regex.IsMatch(configuration))\n return new List();\n\n List ret = new List();\n\n for (int i = start; i < max; ++i)\n ret.Add(i);\n\n return ret;\n }\n\n private List list_array(string configuration)\n {\n if (!list_regex.IsMatch(configuration))\n return new List();\n\n List ret = new List();\n\n string[] split = configuration.Split(\",\".ToCharArray());\n\n foreach (string s in split)\n ret.Add(int.Parse(s));\n\n return ret;\n }\n\n #endregion\n }\n}\n" }, { "path": "RaptorDB.Common/DataTypes.cs", "content": "using System;\nusing System.Collections.Generic;\n\nnamespace RaptorDB\n{\n /// \n /// Result of queries\n /// OK : T = Query with data, F = EX has the exception\n /// Rows : query rows\n /// \n public class Result\n {\n public Result()\n {\n\n }\n public Result(bool ok)\n {\n OK = ok;\n }\n public Result(bool ok, Exception ex)\n {\n OK = ok;\n EX = ex;\n }\n /// \n /// T=Values return, F=exceptions occurred \n /// \n public bool OK { get; set; }\n public Exception EX { get; set; }\n /// \n /// Total number of rows of the query\n /// \n public int TotalCount { get; set; }\n /// \n /// Rows returned\n /// \n public int Count { get; set; }\n public List Rows { get; set; }\n\n public string Title { get; set; }\n // FEATURE : data pending in results\n ///// \n ///// Data is being indexed, so results will not reflect all documents\n ///// \n //public bool DataPending { get; set; }\n }\n\n /// \n /// Base for row schemas : implements a docid property and is bindable\n /// \n public abstract class RDBSchema : BindableFields\n {\n public Guid docid;\n }\n}\n\n// no namespace -> available to all\npublic static class RDBExtension\n{\n /// \n /// RDB Between checking for dates\n /// \n /// \n /// Must be yyyy-mm-dd\n /// Must be yyyy-mm-dd\n /// \n public static bool Between(this DateTime value, string fromdate, string todate)\n {\n return true;\n }\n\n public static bool Between(this T value, T from, T to)\n {\n return true;\n }\n\n public static bool In(this byte value, params byte[] values)\n {\n return true;\n }\n\n public static bool In(this T value, params T[] values)\n {\n return true;\n }\n}\n" }, { "path": "RaptorDB.Common/FieldDescriptor.cs", "content": "using System;\nusing System.Reflection;\nusing System.ComponentModel;\n\nnamespace RaptorDB\n{\n internal class FieldPropertyDescriptor : PropertyDescriptor\n {\n private FieldInfo _field;\n\n public FieldPropertyDescriptor(FieldInfo field) :\n base(field.Name, (Attribute[])field.GetCustomAttributes(typeof(Attribute), true))\n {\n _field = field;\n }\n\n public FieldInfo Field { get { return _field; } }\n\n public override bool Equals(object obj)\n {\n FieldPropertyDescriptor other = obj as FieldPropertyDescriptor;\n return other != null && other._field.Equals(_field);\n }\n\n public override int GetHashCode() { return _field.GetHashCode(); }\n\n public override bool IsReadOnly { get { return false; } }\n public override void ResetValue(object component) { }\n public override bool CanResetValue(object component) { return false; }\n public override bool ShouldSerializeValue(object component) { return true; }\n\n public override Type ComponentType { get { return _field.DeclaringType; } }\n public override Type PropertyType { get { return _field.FieldType; } }\n\n public override object GetValue(object component) { return _field.GetValue(component); }\n\n public override void SetValue(object component, object value)\n {\n _field.SetValue(component, value);\n OnValueChanged(component, EventArgs.Empty);\n }\n }\n\n public abstract class BindableFields : ICustomTypeDescriptor\n {\n object ICustomTypeDescriptor.GetPropertyOwner(PropertyDescriptor pd)\n {\n return this;\n }\n\n AttributeCollection ICustomTypeDescriptor.GetAttributes()\n {\n return TypeDescriptor.GetAttributes(this, true);\n }\n\n string ICustomTypeDescriptor.GetClassName()\n {\n return TypeDescriptor.GetClassName(this, true);\n }\n\n string ICustomTypeDescriptor.GetComponentName()\n {\n return TypeDescriptor.GetComponentName(this, true);\n }\n\n TypeConverter ICustomTypeDescriptor.GetConverter()\n {\n return TypeDescriptor.GetConverter(this, true);\n }\n\n EventDescriptor ICustomTypeDescriptor.GetDefaultEvent()\n {\n return TypeDescriptor.GetDefaultEvent(this, true);\n }\n\n PropertyDescriptor ICustomTypeDescriptor.GetDefaultProperty()\n {\n return TypeDescriptor.GetDefaultProperty(this, true);\n }\n\n object ICustomTypeDescriptor.GetEditor(Type editorBaseType)\n {\n return TypeDescriptor.GetEditor(this, editorBaseType, true);\n }\n\n EventDescriptorCollection ICustomTypeDescriptor.GetEvents(Attribute[] attributes)\n {\n return TypeDescriptor.GetEvents(this, attributes, true);\n }\n\n EventDescriptorCollection ICustomTypeDescriptor.GetEvents()\n {\n return TypeDescriptor.GetEvents(this, true);\n }\n\n private PropertyDescriptorCollection _propCache;\n private FilterCache _filterCache;\n\n PropertyDescriptorCollection ICustomTypeDescriptor.GetProperties()\n {\n return ((ICustomTypeDescriptor)this).GetProperties(null);\n }\n\n PropertyDescriptorCollection ICustomTypeDescriptor.GetProperties(Attribute[] attributes)\n {\n bool filtering = (attributes != null && attributes.Length > 0);\n PropertyDescriptorCollection props = _propCache;\n FilterCache cache = _filterCache;\n\n // Use a cached version if possible\n if (filtering && cache != null && cache.IsValid(attributes))\n {\n return cache.FilteredProperties;\n }\n else if (!filtering && props != null)\n {\n return props;\n }\n\n // Create the property collection and filter\n props = new PropertyDescriptorCollection(null);\n foreach (PropertyDescriptor prop in TypeDescriptor.GetProperties(this, attributes, true))\n {\n props.Add(prop);\n }\n foreach (FieldInfo field in GetType().GetFields())\n {\n FieldPropertyDescriptor fieldDesc = new FieldPropertyDescriptor(field);\n if (!filtering || fieldDesc.Attributes.Contains(attributes)) props.Add(fieldDesc);\n }\n\n // Store the computed properties\n if (filtering)\n {\n cache = new FilterCache();\n cache.Attributes = attributes;\n cache.FilteredProperties = props;\n _filterCache = cache;\n }\n else _propCache = props;\n\n return props;\n }\n\n private class FilterCache\n {\n public Attribute[] Attributes;\n public PropertyDescriptorCollection FilteredProperties;\n public bool IsValid(Attribute[] other)\n {\n if (other == null || Attributes == null) return false;\n if (Attributes.Length != other.Length) return false;\n for (int i = 0; i < other.Length; i++)\n {\n if (!Attributes[i].Match(other[i])) return false;\n }\n return true;\n }\n }\n }\n}\n" }, { "path": "RaptorDB.Common/IRaptorDB.cs", "content": "using System;\nusing System.Collections.Generic;\nusing System.Linq.Expressions;\n\nnamespace RaptorDB.Common\n{\n public delegate List ServerSideFunc(IRaptorDB rap, string filter);\n public delegate List ServerSideFuncWithArgs(IRaptorDB rap, string filter, params object[] args);\n\n public class HistoryInfo\n {\n public int Version;\n public DateTime ChangeDate;\n }\n\n /// \n /// High frequency mode Key/Value store with recycled storage file.\n /// Use for rapid saves of the same key.\n /// Views are not effected by saves in this storage.\n /// NOTE : You do not have history of changes in this storage.\n /// \n public interface IKeyStoreHF\n {\n object GetObjectHF(string key);\n bool SetObjectHF(string key, object obj);\n bool DeleteKeyHF(string key);\n int CountHF();\n bool ContainsHF(string key);\n string[] GetKeysHF();\n void CompactStorageHF();\n int Increment(string key, int amount);\n decimal Increment(string key, decimal amount);\n int Decrement(string key, int amount);\n decimal Decrement(string key, decimal amount);\n //T Increment(string key, T amount);\n //T Decrement(string key, T amount);\n //IEnumerable EnumerateObjects();\n //string[] SearchKeys(string contains); // FIX : implement \n }\n\n public interface IRaptorDB\n {\n /// \n /// Save Bytes (files) to RptorDB storage\n /// \n /// \n /// \n /// \n bool SaveBytes(Guid fileID, byte[] bytes);\n\n /// \n /// Save a Document to RaptorDB\n /// \n /// \n /// \n /// \n /// \n bool Save(Guid docID, T document);\n\n /// \n /// Query all data in a view\n /// \n /// \n /// \n Result Query(string viewname);\n\n /// \n /// Query all data in a view with paging\n /// \n /// \n /// \n /// \n /// \n Result Query(string viewname, int start, int count);\n\n /// \n /// Query a View with a string filter\n /// \n /// \n /// \n /// \n Result Query(string viewname, string filter);\n\n /// \n /// Query a View with a string filter with paging\n /// \n /// \n /// \n /// \n /// \n /// \n Result Query(string viewname, string filter, int start, int count);\n\n /// \n /// Query a view with filter, paging and sorting\n /// \n /// \n /// \n /// \n /// \n /// \n Result Query(string viewname, string filter, int start, int count, string orderby);\n\n /// \n /// Count all data associated with View name\n /// \n /// \n /// \n int Count(string viewname);\n\n /// \n /// Count all data associated with View name and string filter\n /// \n /// \n /// \n /// \n int Count(string viewname, string filter);\n\n /// \n /// Fetch a Document\n /// \n /// \n /// \n object Fetch(Guid docID);\n\n /// \n /// Fetch a Document when you know the type\n /// \n /// \n /// \n T Fetch(Guid docID) where T : class;\n\n /// \n /// Fetch a file bytes\n /// \n /// \n /// \n byte[] FetchBytes(Guid fileID);\n\n ///// \n ///// Shutdown RaptorDB and flush all data to disk\n ///// \n //void Shutdown();\n\n /// \n /// Backup the document storage file incrementally to \"Backup\" folder\n /// \n /// True = done\n bool Backup();\n\n /// \n /// Start background restore of backups in the \"Restore\" folder\n /// \n void Restore();\n\n /// \n /// Delete a Document\n /// \n /// \n /// \n bool Delete(Guid docid);\n\n /// \n /// Delete a File\n /// \n /// \n /// \n bool DeleteBytes(Guid fileid);\n\n /// \n /// Add users\n /// \n /// \n /// \n /// \n /// \n bool AddUser(string username, string oldpassword, string newpassword);\n\n /// \n /// Do server side data aggregate queries, so you don't transfer large data rows to clients for processing \n /// \n /// \n /// \n object[] ServerSide(ServerSideFunc func, string filter);\n\n /// \n /// Do server side data aggregate queries, so you don't transfer large data rows to clients for processing \n /// \n /// \n /// \n object[] ServerSide(ServerSideFunc func, Expression> filter);\n\n\n /// \n /// Do server side data aggregate queries, so you don't transfer large data rows to clients for processing \n /// \n /// \n /// \n object[] ServerSide(ServerSideFuncWithArgs func, string filter, params object[] args);\n\n /// \n /// Do server side data aggregate queries, so you don't transfer large data rows to clients for processing \n /// \n /// \n /// \n object[] ServerSide(ServerSideFuncWithArgs func, Expression> filter, params object[] args);\n\n /// \n /// Full text search the entire original document\n /// \n /// \n /// \n int[] FullTextSearch(string filter);\n\n // new query model\n /// \n /// Query a view with linq filter\n /// \n /// Use the Row Schema type for your view\n /// \n /// \n Result Query(Expression> filter);\n\n /// \n /// Query a view with paging\n /// \n /// Use the Row Schema type for your view\n /// \n /// \n /// \n /// \n Result Query(Expression> filter, int start, int count);\n\n /// \n /// Query a view with linq filter, paging and sorting\n /// \n /// Use the Row Schema type for your view\n /// \n /// \n /// \n /// \n /// \n Result Query(Expression> filter, int start, int count, string orderby);\n\n /// \n /// Query a view with a string filter\n /// \n /// Use the Row Schema type for your view\n /// \n /// \n Result Query(string filter);\n\n /// \n /// Query a view with string filter and paging\n /// \n /// Use the Row Schema type for your view\n /// \n /// \n /// \n /// \n Result Query(string filter, int start, int count);\n\n /// \n /// Query a view with string filter, paging and sorting\n /// \n /// Use the Row Schema type for your view\n /// \n /// \n /// \n /// \n /// \n Result Query(string filter, int start, int count, string orderby);\n\n /// \n /// Count rows with a linq filter\n /// \n /// \n /// \n /// \n int Count(Expression> filter);\n\n /// \n /// Fetch the change history for a document\n /// \n /// \n /// \n int[] FetchHistory(Guid docid);\n\n /// \n /// Fetch the change history for a document with dates\n /// \n /// \n /// \n HistoryInfo[] FetchHistoryInfo(Guid docid);\n\n /// \n /// Fetch a change history for a file\n /// \n /// \n /// \n int[] FetchBytesHistory(Guid fileid);\n\n /// \n /// Fetch a change history for a file with dates\n /// \n /// \n /// \n HistoryInfo[] FetchBytesHistoryInfo(Guid docid);\n\n /// \n /// Fetch the specific document version \n /// \n /// \n /// \n object FetchVersion(int versionNumber);\n\n /// \n /// Fetch the specific file version\n /// \n /// \n /// \n byte[] FetchBytesVersion(int versionNumber);\n\n /// \n /// Delete rows from a view\n /// \n /// \n /// \n /// Number of rows deleted\n int ViewDelete(Expression> filter);\n\n /// \n /// Delete rows from a view\n /// \n /// \n /// \n /// Number of rows deleted\n int ViewDelete(string viewname, string filter);\n\n /// \n /// Insert directly into a view\n /// \n /// \n /// \n /// \n /// \n bool ViewInsert(Guid id, TRowSchema row);\n\n /// \n /// Insert directly into a view\n /// \n /// \n /// \n /// \n /// \n bool ViewInsert(string viewname, Guid id, object row);\n\n /// \n /// Get the number of documents in the storage file regardless of versions\n /// \n /// \n long DocumentCount();\n\n /// \n /// High frequency mode Key/Value store with recycled storage file.\n /// Use for rapid saves of the same key.\n /// Views are not effected by saves in this storage.\n /// NOTE : You do not have history of changes in this storage.\n /// \n IKeyStoreHF GetKVHF();\n\n void FreeMemory();\n\n void Shutdown();\n\n void SaveToDocsOnViewInsert(bool yes);\n }\n}\n" }, { "path": "RaptorDB.Common/Interfaces.cs", "content": "using System;\nusing System.Collections.Generic;\nusing System.Linq.Expressions;\nusing RaptorDB.Common;\n\nnamespace RaptorDB\n{\n //public static class RDBExtensions\n //{\n // ///// \n // ///// For RaptorDB optimized range queries\n // ///// \n // ///// \n // ///// \n // ///// \n // ///// \n // ///// \n // //public static bool Between(this T obj, T from, T to)\n // //{\n // // return true;\n // //}\n\n // ///// \n // ///// For RaptorDB full text search queries\n // ///// \n // ///// \n // ///// \n // ///// \n // //public static bool Contains(this string obj, string what)\n // //{\n // // return true;\n // //}\n //}\n\n /// \n /// Used for normal string columns \n /// \n [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property)]\n public class CaseInsensitiveAttribute : Attribute\n {\n }\n\n /// \n /// Used for the indexer -> hOOt full text indexing\n /// \n [AttributeUsage(AttributeTargets.Field | AttributeTargets.Property)]\n public class FullTextAttribute : Attribute\n {\n }\n\n /// \n /// Used for declaring view extensions DLL's\n /// \n [AttributeUsage(AttributeTargets.Class)]\n public class RegisterViewAttribute : Attribute\n {\n }\n\n /// \n /// Index file max string length size in UTF8 (Default = 60)\n /// \n [AttributeUsage(AttributeTargets.Property | AttributeTargets.Field)]\n public class StringIndexLength : Attribute\n {\n public StringIndexLength()\n {\n Length = 60; // default\n }\n public StringIndexLength(byte length)\n {\n Length = length;\n }\n public byte Length;\n }\n\n public interface IQueryInterface\n {\n /// \n /// Log messages\n /// \n /// \n void Log(string message);\n\n /// \n /// Count all data associated with the Documnet Type or the View Type with a string filter\n /// \n /// \n /// \n int Count(string viewname);\n\n /// \n /// Count all data associated with View name and string filter\n /// \n /// \n /// \n /// \n int Count(string ViewName, string Filter);\n\n /// \n /// Fetch a document by it's Guid\n /// \n /// \n /// \n object Fetch(Guid guid);\n\n /// \n /// Fetch a document by it's Guid\n /// \n /// \n /// \n T Fetch(Guid guid) where T : class;\n\n // new query model\n Result Query(Expression> Filter);\n Result Query(Expression> Filter, int start, int count);\n Result Query(string Filter);\n Result Query(string Filter, int start, int count);\n int Count(Expression> Filter);\n }\n\n public interface IMapAPI : IQueryInterface\n {\n /// \n /// Emit values, the ordering must match the view schema\n /// \n /// \n /// \n void Emit(Guid docid, params object[] data);\n\n /// \n /// Emits the object matching the view schema, you must make sure the object property names match the row schema\n /// \n /// \n /// \n /// \n void EmitObject(Guid docid, T doc);\n\n /// \n /// Roll back the transaction if the primary view is in transaction mode\n /// \n void RollBack();\n\n /// \n /// Get the next row number for this view\n /// \n /// \n int NextRowNumber();\n\n IKeyStoreHF GetKVHF();\n //void EmitRow(Guid docid, V row);\n }\n\n public interface IClientHandler\n {\n bool GenerateClientData(IQueryInterface api, string username, List DocsToSend);\n }\n\n public enum COMMANDS\n {\n Save,\n SaveBytes,\n QueryType,\n QueryStr,\n Fetch,\n FetchBytes,\n Backup,\n Delete,\n DeleteBytes,\n Restore,\n AddUser,\n ServerSide,\n FullText,\n CountType,\n CountStr,\n GCount,\n DocHistory,\n FileHistory,\n FetchVersion,\n FetchFileVersion,\n CheckAssembly,\n FetchHistoryInfo,\n FetchByteHistoryInfo,\n ViewDelete,\n ViewDelete_t,\n ViewInsert,\n ViewInsert_t,\n DocCount,\n GetObjectHF,\n SetObjectHF,\n DeleteKeyHF,\n CountHF,\n ContainsHF,\n GetKeysHF,\n CompactStorageHF,\n IncrementHF,\n DecrementHF,\n ServerSideWithArgs,\n FreeMemory\n }\n}\n" }, { "path": "RaptorDB.Common/LINQString.cs", "content": "using System;\nusing System.Collections;\nusing System.Collections.Generic;\nusing System.Linq;\nusing System.Linq.Expressions;\nusing System.Reflection;\nusing System.Text;\n\nnamespace RaptorDB\n{\n public class LINQString : ExpressionVisitor\n {\n public StringBuilder sb = new StringBuilder();\n\n protected override Expression VisitBinary(BinaryExpression b)\n {\n sb.Append(\"(\");\n this.Visit(b.Left);\n ExpressionType t = b.NodeType;\n\n switch (b.NodeType)\n {\n case ExpressionType.AndAlso:\n case ExpressionType.And:\n sb.Append(\" AND \");\n break;\n case ExpressionType.OrElse:\n case ExpressionType.Or:\n sb.Append(\" OR \");\n break;\n case ExpressionType.Equal:\n sb.Append(\" = \");\n break;\n case ExpressionType.NotEqual:\n sb.Append(\" != \");\n break;\n case ExpressionType.LessThan:\n sb.Append(\" < \");\n break;\n case ExpressionType.LessThanOrEqual:\n sb.Append(\" <= \");\n break;\n case ExpressionType.GreaterThan:\n sb.Append(\" > \");\n break;\n case ExpressionType.GreaterThanOrEqual:\n sb.Append(\" >= \");\n break;\n }\n\n this.Visit(b.Right);\n sb.Append(\")\");\n return b;\n }\n\n protected override Expression VisitMethodCall(MethodCallExpression m)\n {\n string s = m.ToString();\n string prop = m.Arguments[0].ToString();\n sb.Append(prop.Substring(prop.IndexOf(\".\") + 1) + \".\" + m.Method.Name + \"(\");\n\n Visit(m.Arguments[1]); \n\n sb.Append(\")\");\n return m;\n }\n\n Stack _stack = new Stack();\n protected override Expression VisitMember(MemberExpression m)\n {\n var e = base.VisitMember(m);\n var c = m.Expression as ConstantExpression;\n if (c != null)\n {\n Type t = c.Value.GetType();\n var x = t.InvokeMember(m.Member.Name, BindingFlags.GetField |\n BindingFlags.GetProperty |\n BindingFlags.Public |\n BindingFlags.NonPublic |\n BindingFlags.Instance |\n BindingFlags.Static, null, c.Value, null);\n\n if (x is string)\n sb.Append(\"\\\"\").Append(x).Append(\"\\\"\");\n else if (!x.GetType().IsClass)\n { // check for complex structs\n if (x is DateTime || x is Guid)\n sb.Append(\"\\\"\").Append(x).Append(\"\\\"\");\n else // numbers\n sb.Append(x);\n }\n else if (x.GetType().IsArray)\n {\n var a = x as IList;\n for (int i = 0; i < a.Count - 1; i++)\n {\n sb.Append(a[i]);\n sb.Append(\",\");\n }\n sb.Append(a[a.Count - 1]);\n }\n else\n _stack.Push(x);\n }\n\n if (m.Expression != null)\n {\n if (m.Expression.NodeType == ExpressionType.Parameter) // property\n sb.Append(m.Member.Name);\n else if (m.Expression.NodeType == ExpressionType.MemberAccess) // obj.property\n {\n Type t = m.Expression.Type;\n var f = m.Expression as MemberExpression;\n var val = t.InvokeMember(m.Member.Name, BindingFlags.GetField |\n BindingFlags.GetProperty |\n BindingFlags.Public |\n BindingFlags.NonPublic |\n BindingFlags.Instance |\n BindingFlags.Static, null, _stack.Pop(), null);\n\n if (val is string)\n sb.Append(\"\\\"\").Append(val).Append(\"\\\"\");\n else if (!val.GetType().IsClass)\n { // check for complex structs\n if (val is DateTime || val is Guid)\n sb.Append(\"\\\"\").Append(val).Append(\"\\\"\");\n else // numbers\n sb.Append(val);\n }\n else\n _stack.Push(val);\n }\n }\n return e;\n }\n\n protected override Expression VisitConstant(ConstantExpression c)\n {\n IQueryable q = c.Value as IQueryable;\n if (q != null)\n {\n sb.Append(q.ElementType.Name);\n }\n else if (c.Value == null)\n {\n sb.Append(\"NULL\");\n }\n else\n {\n //_stack.Push(c.Value);\n //if (Type.GetTypeCode(c.Value.GetType()) == TypeCode.Object)\n // _stack.Pop();\n\n switch (Type.GetTypeCode(c.Value.GetType()))\n {\n case TypeCode.Boolean:\n //sb.Append(((bool)c.Value) ? 1 : 0);\n sb.Append(((bool)c.Value) ? \"True\" : \"False\");\n break;\n case TypeCode.String:\n sb.Append(\"\\\"\");\n sb.Append(c.Value);\n sb.Append(\"\\\"\");\n break;\n case TypeCode.Object:\n break;\n default:\n sb.Append(c.Value);\n if (--_count > 0)\n sb.Append(\",\");\n break;\n }\n }\n return c;\n }\n\n int _count = 0;\n public override Expression Visit(Expression node)\n {\n if (node.NodeType == ExpressionType.NewArrayInit) \n {\n var a = node as NewArrayExpression;\n _count = a.Expressions.Count;\n return base.Visit(node);\n }\n else\n return base.Visit(node);\n }\n }\n}\n" }, { "path": "RaptorDB.Common/Logger.cs", "content": "using System;\nusing System.Collections.Generic;\nusing System.Text;\nusing System.Threading;\nusing System.IO;\nusing RaptorDB.Common;\n\nnamespace RaptorDB\n{\n public interface ILog\n {\n /// \n /// Fatal log = log level 5\n /// \n /// \n /// \n void Fatal(object msg, params object[] objs); // 5\n /// \n /// Error log = log level 4\n /// \n /// \n /// \n void Error(object msg, params object[] objs); // 4\n /// \n /// Warning log = log level 3\n /// \n /// \n /// \n void Warn(object msg, params object[] objs); // 3\n /// \n /// Debug log = log level 2 \n /// \n /// \n /// \n void Debug(object msg, params object[] objs); // 2\n /// \n /// Info log = log level 1\n /// \n /// \n /// \n void Info(object msg, params object[] objs); // 1\n }\n\n internal class FileLogger\n {\n // Sinlgeton pattern 4 from : http://csharpindepth.com/articles/general/singleton.aspx\n private static readonly FileLogger instance = new FileLogger();\n // Explicit static constructor to tell C# compiler\n // not to mark type as beforefieldinit\n static FileLogger()\n {\n }\n private FileLogger()\n {\n }\n public static FileLogger Instance { get { return instance; } }\n\n private Queue _que = new Queue();\n private Queue _log = new Queue();\n private StreamWriter _output;\n private string _filename;\n private int _sizeLimit = 0;\n private long _lastSize = 0;\n private DateTime _lastFileDate;\n private bool _showMethodName = false;\n private string _FilePath = \"\";\n private System.Timers.Timer _saveTimer;\n private int _lastLogsToKeep = 100;\n internal int _logabove = 1;\n private string _S = \"\" + Path.DirectorySeparatorChar;\n\n public bool ShowMethodNames\n {\n get { return _showMethodName; }\n }\n\n public void Init(string filename, int sizelimitKB, bool showmethodnames)\n {\n if (_output != null)\n return;\n _que = new Queue();\n _showMethodName = showmethodnames;\n _sizeLimit = sizelimitKB;\n _filename = filename;\n // handle folder names as well -> create dir etc.\n _S = Path.DirectorySeparatorChar.ToString();\n _FilePath = Path.GetDirectoryName(filename);\n if (_FilePath != \"\")\n {\n _FilePath = Directory.CreateDirectory(_FilePath).FullName;\n if (_FilePath.EndsWith(_S) == false)\n _FilePath += _S;\n }\n\n _output = new StreamWriter(filename, true);\n FileInfo fi = new FileInfo(filename);\n _lastSize = fi.Length;\n _lastFileDate = fi.LastWriteTime;\n // zip old logs\n ZipLogs(_FilePath, _lastFileDate);\n\n _saveTimer = new System.Timers.Timer(500);\n _saveTimer.Elapsed += new System.Timers.ElapsedEventHandler(_saveTimer_Elapsed);\n _saveTimer.Enabled = true;\n _saveTimer.AutoReset = true;\n }\n\n void _saveTimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)\n {\n WriteData();\n }\n\n public void ShutDown()\n {\n _saveTimer.Enabled = false;\n WriteData();\n if (_output != null)\n {\n _output.Flush();\n _output.Close();\n _output = null;\n }\n }\n\n private void WriteData()\n {\n if (_output == null)\n return;\n lock (_que)\n {\n while (_que.Count > 0)\n {\n object o = _que.Dequeue();\n if (_output != null && o != null)\n {\n if (_sizeLimit > 0)\n {\n // implement size limited logs\n // implement rolling logs\n #region [ rolling size limit ]\n _lastSize += (\"\" + o).Length;\n if (_lastSize > _sizeLimit * 1000)\n {\n _output.Flush();\n _output.Close();\n int count = 1;\n while (File.Exists(_FilePath + Path.GetFileNameWithoutExtension(_filename) + \".\" + count.ToString(\"0000\")))\n count++;\n\n File.Move(_filename,\n _FilePath +\n Path.GetFileNameWithoutExtension(_filename) +\n \".\" + count.ToString(\"0000\"));\n _output = new StreamWriter(_filename, true);\n _lastSize = 0;\n }\n #endregion\n }\n if (DateTime.Now.Subtract(_lastFileDate).Days > 0)\n {\n // implement date logs\n #region [ rolling dates ]\n _output.Flush();\n _output.Close();\n int count = 1;\n while (File.Exists(_FilePath + Path.GetFileNameWithoutExtension(_filename) + \".\" + count.ToString(\"0000\")))\n {\n File.Move(_FilePath + Path.GetFileNameWithoutExtension(_filename) + \".\" + count.ToString(\"0000\"),\n _FilePath +\n Path.GetFileNameWithoutExtension(_filename) +\n \".\" + count.ToString(\"0000\") +\n \".\" + _lastFileDate.ToString(\"yyyy-MM-dd\"));\n count++;\n }\n File.Move(_filename,\n _FilePath +\n Path.GetFileNameWithoutExtension(_filename) +\n \".\" + count.ToString(\"0000\") +\n \".\" + _lastFileDate.ToString(\"yyyy-MM-dd\"));\n // compress old logs here\n ZipLogs(_FilePath, _lastFileDate);\n\n _output = new StreamWriter(_filename, true);\n _lastFileDate = DateTime.Now;\n _lastSize = 0;\n #endregion\n }\n _output.Write(o);\n }\n }\n if (_output != null)\n _output.Flush();\n }\n lock (_log)\n {\n while (_log.Count > _lastLogsToKeep)\n _log.Dequeue();\n }\n }\n\n private void ZipLogs(string path, DateTime lastFileDate)\n {\n path = new DirectoryInfo(path).FullName;\n var prefix = path;\n var files = Directory.GetFiles(path, \"*-*\");\n if (files.Length > 0)\n {\n var fn = lastFileDate.ToString(\"yyyy-MM-dd\") + \".zip\";\n path += \"old\" + _S;\n if (Directory.Exists(path) == false)\n {\n fn = \"0000-00-00.zip\";\n Directory.CreateDirectory(path);\n }\n\n var zip = ZipStorer.Create(path + fn, \"\");\n foreach (var f in files)\n {\n zip.AddFile(ZipStorer.Compression.Deflate, f, f.Replace(prefix, \"\"), \"\");\n File.Delete(f);\n }\n zip.Close();\n }\n }\n\n private string FormatLog(string log, string type, string meth, string msg, object[] objs)\n {\n StringBuilder sb = new StringBuilder();\n sb.Append(DateTime.Now.ToString(\"yyyy-MM-dd HH:mm:ss\"));\n sb.Append(\"|\");\n sb.Append(log);\n sb.Append(\"|\");\n sb.Append(Thread.CurrentThread.ManagedThreadId.ToString());\n sb.Append(\"|\");\n sb.Append(type);\n sb.Append(\"|\");\n sb.Append(meth);\n sb.Append(\"| \");\n sb.AppendLine(msg);\n\n if (objs != null)\n foreach (object o in objs)\n sb.AppendLine(\"\" + o);\n\n return sb.ToString();\n }\n\n public void Log(string logtype, string type, string meth, string msg, params object[] objs)\n {\n var l = FormatLog(logtype, type, meth, msg, objs);\n lock (_que)\n _que.Enqueue(l);\n lock (_log)\n _log.Enqueue(l);\n }\n\n internal List GetLastLogs()\n {\n List l = new List();\n\n foreach (var s in _log)\n {\n l.Add(s);\n }\n\n return l;\n }\n\n public void SetLogLevel(int abovelevel)\n {\n _logabove = abovelevel;\n }\n }\n\n internal class logger : ILog\n {\n public logger(Type type)\n {\n typename = type.Namespace + \".\" + type.Name;\n }\n\n private string typename = \"\";\n\n private void log(string logtype, string msg, params object[] objs)\n {\n string meth = \"\";\n if (FileLogger.Instance.ShowMethodNames)\n {\n System.Diagnostics.StackTrace st = new System.Diagnostics.StackTrace(2);\n System.Diagnostics.StackFrame sf = st.GetFrame(0);\n meth = sf.GetMethod().Name;\n }\n FileLogger.Instance.Log(logtype, typename, meth, msg, objs);\n }\n\n #region ILog Members\n public void Fatal(object msg, params object[] objs)\n {\n log(\"FATAL\", \"\" + msg, objs);\n }\n\n public void Error(object msg, params object[] objs)\n {\n if (FileLogger.Instance._logabove <= 4)\n log(\"ERROR\", \"\" + msg, objs);\n }\n\n public void Warn(object msg, params object[] objs)\n {\n if (FileLogger.Instance._logabove <= 3)\n log(\"WARN\", \"\" + msg, objs);\n }\n\n public void Debug(object msg, params object[] objs)\n {\n if (FileLogger.Instance._logabove <= 2)\n log(\"DEBUG\", \"\" + msg, objs);\n }\n\n public void Info(object msg, params object[] objs)\n {\n if (FileLogger.Instance._logabove <= 1)\n log(\"INFO\", \"\" + msg, objs);\n }\n #endregion\n }\n\n public static class LogManager\n {\n public static ILog GetLogger(Type obj)\n {\n return new logger(obj);\n }\n\n public static void Configure(string filename, int sizelimitKB, bool showmethodnames)\n {\n FileLogger.Instance.Init(filename, sizelimitKB, showmethodnames);\n }\n\n public static List GetLastLogs()\n {\n return FileLogger.Instance.GetLastLogs();\n }\n\n public static void Shutdown()\n {\n FileLogger.Instance.ShutDown();\n }\n\n public static void SetLogLevel(int abovelevel)\n {\n FileLogger.Instance.SetLogLevel(abovelevel);\n }\n }\n}\n" }, { "path": "RaptorDB.Common/MiniLZO.cs", "content": "/**\n * \n * Modifications by Simon Hewitt\n * - change constructors/methods to return byte[]\n * - append original source size at the end of the destination buffer\n * - add support for MemoryStream internal buffer usage\n * \n * \n * ManagedLZO.MiniLZO\n * \n * Minimalistic reimplementation of minilzo in C#\n * \n * @author Shane Eric Bryldt, Copyright (C) 2006, All Rights Reserved\n * @note Uses unsafe/fixed pointer contexts internally\n * @liscence Bound by same liscence as minilzo as below, see file COPYING\n */\n\n/* Based on minilzo.c -- mini subset of the LZO real-time data compression library\n\n This file is part of the LZO real-time data compression library.\n\n Copyright (C) 2005 Markus Franz Xaver Johannes Oberhumer\n Copyright (C) 2004 Markus Franz Xaver Johannes Oberhumer\n Copyright (C) 2003 Markus Franz Xaver Johannes Oberhumer\n Copyright (C) 2002 Markus Franz Xaver Johannes Oberhumer\n Copyright (C) 2001 Markus Franz Xaver Johannes Oberhumer\n Copyright (C) 2000 Markus Franz Xaver Johannes Oberhumer\n Copyright (C) 1999 Markus Franz Xaver Johannes Oberhumer\n Copyright (C) 1998 Markus Franz Xaver Johannes Oberhumer\n Copyright (C) 1997 Markus Franz Xaver Johannes Oberhumer\n Copyright (C) 1996 Markus Franz Xaver Johannes Oberhumer\n All Rights Reserved.\n\n The LZO library is free software; you can redistribute it and/or\n modify it under the terms of the GNU General Public License,\n version 2, as published by the Free Software Foundation.\n\n The LZO library is distributed in the hope that it will be useful,\n but WITHOUT ANY WARRANTY; without even the implied warranty of\n MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n GNU General Public License for more details.\n\n You should have received a copy of the GNU General Public License\n along with the LZO library; see the file COPYING.\n If not, write to the Free Software Foundation, Inc.,\n 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.\n\n Markus F.X.J. Oberhumer\n \n http://www.oberhumer.com/opensource/lzo/\n */\n\n/*\n * NOTE:\n * the full LZO package can be found at\n * http://www.oberhumer.com/opensource/lzo/\n */\n\nusing System;\nusing System.IO;\n\nnamespace RaptorDB\n{\n public class MiniLZO\n {\n private const uint M2_MAX_LEN = 8;\n private const uint M4_MAX_LEN = 9;\n private const byte M3_MARKER = 32;\n private const byte M4_MARKER = 16;\n private const uint M2_MAX_OFFSET = 0x0800;\n private const uint M3_MAX_OFFSET = 0x4000;\n private const uint M4_MAX_OFFSET = 0xbfff;\n private const byte BITS = 14;\n private const uint D_MASK = (1 << BITS) - 1;\n private static uint DICT_SIZE = 65536 + 3;\n\n static MiniLZO()\n {\n if (IntPtr.Size == 8)\n DICT_SIZE = (65536 + 3) * 2;\n }\n public static byte[] Compress(byte[] src) { return Compress(src, 0, src.Length); }\n public static byte[] Compress(byte[] src, int srcCount) { return Compress(src, 0, srcCount); }\n public static byte[] Compress(byte[] src, int srcStart, int srcLength)\n {\n byte[] workMem = new byte[DICT_SIZE];\n uint dstlen = (uint)(srcLength + (srcLength / 16) + 64 + 3 + 4);\n byte[] dst = new byte[dstlen];\n\n uint compressedSize = Compress(src, (uint)srcStart, (uint)srcLength, dst, 0, dstlen, workMem, 0);\n\n if (dst.Length != compressedSize)\n {\n byte[] final = new byte[compressedSize];\n Buffer.BlockCopy(dst, 0, final, 0, (int)compressedSize);\n dst = final;\n }\n\n return dst;\n\n }\n\n public static byte[] Compress(MemoryStream source)\n {\n byte[] destinationBuffer;\n byte[] workspaceBuffer;\n uint sourceOffset;\n uint workspaceOffset;\n uint sourceLength;\n uint destinationLength;\n\n byte[] sourceBuffer = source.GetBuffer();\n uint sourceCapacity = (uint)source.Capacity;\n sourceLength = (uint)source.Length;\n destinationLength = sourceLength + (sourceLength / 16) + 64 + 3 + 4;\n\n uint unusedSpace = sourceCapacity - sourceLength;\n uint inplaceOverhead = Math.Min(sourceLength, M4_MAX_OFFSET) + sourceLength / 64 + 16 + 3 + 4;\n\n if (unusedSpace < inplaceOverhead)\n {\n sourceOffset = 0;\n destinationBuffer = new byte[destinationLength];\n }\n else\n {\n sourceOffset = inplaceOverhead;\n source.SetLength(sourceLength + inplaceOverhead);\n destinationBuffer = sourceBuffer;\n Buffer.BlockCopy(destinationBuffer, 0, destinationBuffer, (int)inplaceOverhead, (int)sourceLength);\n unusedSpace -= inplaceOverhead;\n }\n\n if (unusedSpace < DICT_SIZE)\n {\n workspaceBuffer = new byte[DICT_SIZE];\n workspaceOffset = 0;\n }\n else\n {\n workspaceBuffer = sourceBuffer;\n workspaceOffset = sourceCapacity - DICT_SIZE;\n }\n\n uint compressedSize = Compress(sourceBuffer, sourceOffset, sourceLength, destinationBuffer, 0, destinationLength, workspaceBuffer, workspaceOffset);\n\n if (destinationBuffer == sourceBuffer)\n {\n source.SetLength(compressedSize);\n source.Capacity = (int)compressedSize;\n return source.GetBuffer();\n }\n else\n {\n byte[] final = new byte[compressedSize];\n Buffer.BlockCopy(destinationBuffer, 0, final, 0, (int)compressedSize);\n return final;\n }\n\n }\n\n private static unsafe uint Compress(byte[] src, uint srcstart, uint srcLength, byte[] dst, uint dststart, uint dstlen, byte[] workmem, uint workmemstart)\n {\n uint tmp;\n if (srcLength <= M2_MAX_LEN + 5)\n {\n tmp = (uint)srcLength;\n dstlen = 0;\n }\n else\n {\n fixed (byte* work = &workmem[workmemstart], input = &src[srcstart], output = &dst[dststart])\n {\n byte** dict = (byte**)work;\n byte* in_end = input + srcLength;\n byte* ip_end = input + srcLength - M2_MAX_LEN - 5;\n byte* ii = input;\n byte* ip = input + 4;\n byte* op = output;\n bool literal = false;\n bool match = false;\n uint offset;\n uint length;\n uint index;\n byte* pos;\n\n for (; ; )\n {\n offset = 0;\n index = D_INDEX1(ip);\n pos = ip - (ip - dict[index]);\n if (pos < input || (offset = (uint)(ip - pos)) <= 0 || offset > M4_MAX_OFFSET)\n literal = true;\n else if (offset <= M2_MAX_OFFSET || pos[3] == ip[3]) { }\n else\n {\n index = D_INDEX2(index);\n pos = ip - (ip - dict[index]);\n if (pos < input || (offset = (uint)(ip - pos)) <= 0 || offset > M4_MAX_OFFSET)\n literal = true;\n else if (offset <= M2_MAX_OFFSET || pos[3] == ip[3]) { }\n else\n literal = true;\n }\n\n if (!literal)\n {\n if (*((ushort*)pos) == *((ushort*)ip) && pos[2] == ip[2])\n match = true;\n }\n\n literal = false;\n if (!match)\n {\n dict[index] = ip;\n ++ip;\n if (ip >= ip_end)\n break;\n continue;\n }\n match = false;\n dict[index] = ip;\n if (ip - ii > 0)\n {\n uint t = (uint)(ip - ii);\n if (t <= 3)\n {\n //Debug.Assert(op - 2 > output);\n op[-2] |= (byte)(t);\n }\n else if (t <= 18)\n *op++ = (byte)(t - 3);\n else\n {\n uint tt = t - 18;\n *op++ = 0;\n while (tt > 255)\n {\n tt -= 255;\n *op++ = 0;\n }\n //Debug.Assert(tt > 0);\n *op++ = (byte)(tt);\n }\n do\n {\n *op++ = *ii++;\n } while (--t > 0);\n }\n //Debug.Assert(ii == ip);\n ip += 3;\n if (pos[3] != *ip++ || pos[4] != *ip++ || pos[5] != *ip++\n || pos[6] != *ip++ || pos[7] != *ip++ || pos[8] != *ip++)\n {\n --ip;\n length = (uint)(ip - ii);\n //Debug.Assert(length >= 3);\n //Debug.Assert(length <= M2_MAX_LEN);\n if (offset <= M2_MAX_OFFSET)\n {\n --offset;\n *op++ = (byte)(((length - 1) << 5) | ((offset & 7) << 2));\n *op++ = (byte)(offset >> 3);\n }\n else if (offset <= M3_MAX_OFFSET)\n {\n --offset;\n *op++ = (byte)(M3_MARKER | (length - 2));\n *op++ = (byte)((offset & 63) << 2);\n *op++ = (byte)(offset >> 6);\n }\n else\n {\n offset -= 0x4000;\n //Debug.Assert(offset > 0);\n //Debug.Assert(offset <= 0x7FFF);\n *op++ = (byte)(M4_MARKER | ((offset & 0x4000) >> 11) | (length - 2));\n *op++ = (byte)((offset & 63) << 2);\n *op++ = (byte)(offset >> 6);\n }\n }\n else\n {\n byte* m = pos + M2_MAX_LEN + 1;\n while (ip < in_end && *m == *ip)\n {\n ++m;\n ++ip;\n }\n length = (uint)(ip - ii);\n //Debug.Assert(length > M2_MAX_LEN);\n if (offset <= M3_MAX_OFFSET)\n {\n --offset;\n if (length <= 33)\n *op++ = (byte)(M3_MARKER | (length - 2));\n else\n {\n length -= 33;\n *op++ = M3_MARKER | 0;\n while (length > 255)\n {\n length -= 255;\n *op++ = 0;\n }\n //Debug.Assert(length > 0);\n *op++ = (byte)(length);\n }\n }\n else\n {\n offset -= 0x4000;\n //Debug.Assert(offset > 0);\n //Debug.Assert(offset <= 0x7FFF);\n if (length <= M4_MAX_LEN)\n *op++ = (byte)(M4_MARKER | ((offset & 0x4000) >> 11) | (length - 2));\n else\n {\n length -= M4_MAX_LEN;\n *op++ = (byte)(M4_MARKER | ((offset & 0x4000) >> 11));\n while (length > 255)\n {\n length -= 255;\n *op++ = 0;\n }\n //Debug.Assert(length > 0);\n *op++ = (byte)(length);\n }\n }\n *op++ = (byte)((offset & 63) << 2);\n *op++ = (byte)(offset >> 6);\n }\n ii = ip;\n if (ip >= ip_end)\n break;\n }\n dstlen = (uint)(op - output);\n tmp = (uint)(in_end - ii);\n }\n }\n if (tmp > 0)\n {\n uint ii = (uint)srcLength - tmp + srcstart;\n if (dstlen == 0 && tmp <= 238)\n {\n dst[dstlen++] = (byte)(17 + tmp);\n }\n else if (tmp <= 3)\n {\n dst[dstlen - 2] |= (byte)(tmp);\n }\n else if (tmp <= 18)\n {\n dst[dstlen++] = (byte)(tmp - 3);\n }\n else\n {\n uint tt = tmp - 18;\n dst[dstlen++] = 0;\n while (tt > 255)\n {\n tt -= 255;\n dst[dstlen++] = 0;\n }\n //Debug.Assert(tt > 0);\n dst[dstlen++] = (byte)(tt);\n }\n do\n {\n dst[dstlen++] = src[ii++];\n } while (--tmp > 0);\n }\n dst[dstlen++] = M4_MARKER | 1;\n dst[dstlen++] = 0;\n dst[dstlen++] = 0;\n\n // Append the source count\n dst[dstlen++] = (byte)srcLength;\n dst[dstlen++] = (byte)(srcLength >> 8);\n dst[dstlen++] = (byte)(srcLength >> 16);\n dst[dstlen++] = (byte)(srcLength >> 24);\n\n return dstlen;\n }\n\n public static unsafe byte[] Decompress(byte[] src)\n {\n byte[] dst = new byte[(src[src.Length - 4] | (src[src.Length - 3] << 8) | (src[src.Length - 2] << 16 | src[src.Length - 1] << 24))];\n\n uint t = 0;\n fixed (byte* input = src, output = dst)\n {\n byte* pos = null;\n byte* ip_end = input + src.Length - 4;\n byte* op_end = output + dst.Length;\n byte* ip = input;\n byte* op = output;\n bool match = false;\n bool match_next = false;\n bool match_done = false;\n bool copy_match = false;\n bool first_literal_run = false;\n bool eof_found = false;\n\n if (*ip > 17)\n {\n t = (uint)(*ip++ - 17);\n if (t < 4)\n match_next = true;\n else\n {\n //Debug.Assert(t > 0);\n if ((op_end - op) < t)\n throw new OverflowException(\"Output Overrun\");\n if ((ip_end - ip) < t + 1)\n throw new OverflowException(\"Input Overrun\");\n do\n {\n *op++ = *ip++;\n } while (--t > 0);\n first_literal_run = true;\n }\n }\n while (!eof_found && ip < ip_end)\n {\n if (!match_next && !first_literal_run)\n {\n t = *ip++;\n if (t >= 16)\n match = true;\n else\n {\n if (t == 0)\n {\n if ((ip_end - ip) < 1)\n throw new OverflowException(\"Input Overrun\");\n while (*ip == 0)\n {\n t += 255;\n ++ip;\n if ((ip_end - ip) < 1)\n throw new OverflowException(\"Input Overrun\");\n }\n t += (uint)(15 + *ip++);\n }\n //Debug.Assert(t > 0);\n if ((op_end - op) < t + 3)\n throw new OverflowException(\"Output Overrun\");\n if ((ip_end - ip) < t + 4)\n throw new OverflowException(\"Input Overrun\");\n for (int x = 0; x < 4; ++x, ++op, ++ip)\n *op = *ip;\n if (--t > 0)\n {\n if (t >= 4)\n {\n do\n {\n for (int x = 0; x < 4; ++x, ++op, ++ip)\n *op = *ip;\n t -= 4;\n } while (t >= 4);\n if (t > 0)\n {\n do\n {\n *op++ = *ip++;\n } while (--t > 0);\n }\n }\n else\n {\n do\n {\n *op++ = *ip++;\n } while (--t > 0);\n }\n }\n }\n }\n if (!match && !match_next)\n {\n first_literal_run = false;\n\n t = *ip++;\n if (t >= 16)\n match = true;\n else\n {\n pos = op - (1 + M2_MAX_OFFSET);\n pos -= t >> 2;\n pos -= *ip++ << 2;\n if (pos < output || pos >= op)\n throw new OverflowException(\"Lookbehind Overrun\");\n if ((op_end - op) < 3)\n throw new OverflowException(\"Output Overrun\");\n *op++ = *pos++;\n *op++ = *pos++;\n *op++ = *pos++;\n match_done = true;\n }\n }\n match = false;\n do\n {\n if (t >= 64)\n {\n pos = op - 1;\n pos -= (t >> 2) & 7;\n pos -= *ip++ << 3;\n t = (t >> 5) - 1;\n if (pos < output || pos >= op)\n throw new OverflowException(\"Lookbehind Overrun\");\n if ((op_end - op) < t + 2)\n throw new OverflowException(\"Output Overrun\");\n copy_match = true;\n }\n else if (t >= 32)\n {\n t &= 31;\n if (t == 0)\n {\n if ((ip_end - ip) < 1)\n throw new OverflowException(\"Input Overrun\");\n while (*ip == 0)\n {\n t += 255;\n ++ip;\n if ((ip_end - ip) < 1)\n throw new OverflowException(\"Input Overrun\");\n }\n t += (uint)(31 + *ip++);\n }\n pos = op - 1;\n pos -= (*(ushort*)ip) >> 2;\n ip += 2;\n }\n else if (t >= 16)\n {\n pos = op;\n pos -= (t & 8) << 11;\n\n t &= 7;\n if (t == 0)\n {\n if ((ip_end - ip) < 1)\n throw new OverflowException(\"Input Overrun\");\n while (*ip == 0)\n {\n t += 255;\n ++ip;\n if ((ip_end - ip) < 1)\n throw new OverflowException(\"Input Overrun\");\n }\n t += (uint)(7 + *ip++);\n }\n pos -= (*(ushort*)ip) >> 2;\n ip += 2;\n if (pos == op)\n eof_found = true;\n else\n pos -= 0x4000;\n }\n else\n {\n pos = op - 1;\n pos -= t >> 2;\n pos -= *ip++ << 2;\n if (pos < output || pos >= op)\n throw new OverflowException(\"Lookbehind Overrun\");\n if ((op_end - op) < 2)\n throw new OverflowException(\"Output Overrun\");\n *op++ = *pos++;\n *op++ = *pos++;\n match_done = true;\n }\n if (!eof_found && !match_done && !copy_match)\n {\n if (pos < output || pos >= op)\n throw new OverflowException(\"Lookbehind Overrun\");\n //Debug.Assert(t > 0);\n if ((op_end - op) < t + 2)\n throw new OverflowException(\"Output Overrun\");\n }\n if (!eof_found && t >= 2 * 4 - 2 && (op - pos) >= 4 && !match_done && !copy_match)\n {\n for (int x = 0; x < 4; ++x, ++op, ++pos)\n *op = *pos;\n t -= 2;\n do\n {\n for (int x = 0; x < 4; ++x, ++op, ++pos)\n *op = *pos;\n t -= 4;\n } while (t >= 4);\n if (t > 0)\n {\n do\n {\n *op++ = *pos++;\n } while (--t > 0);\n }\n }\n else if (!eof_found && !match_done)\n {\n copy_match = false;\n\n *op++ = *pos++;\n *op++ = *pos++;\n do\n {\n *op++ = *pos++;\n } while (--t > 0);\n }\n\n if (!eof_found && !match_next)\n {\n match_done = false;\n\n t = (uint)(ip[-2] & 3);\n if (t == 0)\n break;\n }\n if (!eof_found)\n {\n match_next = false;\n //Debug.Assert(t > 0);\n //Debug.Assert(t < 4);\n if ((op_end - op) < t)\n throw new OverflowException(\"Output Overrun\");\n if ((ip_end - ip) < t + 1)\n throw new OverflowException(\"Input Overrun\");\n *op++ = *ip++;\n if (t > 1)\n {\n *op++ = *ip++;\n if (t > 2)\n *op++ = *ip++;\n }\n t = *ip++;\n }\n } while (!eof_found && ip < ip_end);\n }\n if (!eof_found)\n throw new OverflowException(\"EOF Marker Not Found\");\n else\n {\n //Debug.Assert(t == 1);\n if (ip > ip_end)\n throw new OverflowException(\"Input Overrun\");\n else if (ip < ip_end)\n throw new OverflowException(\"Input Not Consumed\");\n }\n }\n\n return dst;\n }\n\n private unsafe static uint D_INDEX1(byte* input)\n {\n return D_MS(D_MUL(0x21, D_X3(input, 5, 5, 6)) >> 5, 0);\n }\n\n private static uint D_INDEX2(uint idx)\n {\n return (idx & (D_MASK & 0x7FF)) ^ (((D_MASK >> 1) + 1) | 0x1F);\n }\n\n private static uint D_MS(uint v, byte s)\n {\n return (v & (D_MASK >> s)) << s;\n }\n\n private static uint D_MUL(uint a, uint b)\n {\n return a * b;\n }\n\n private unsafe static uint D_X2(byte* input, byte s1, byte s2)\n {\n return (uint)((((input[2] << s2) ^ input[1]) << s1) ^ input[0]);\n }\n\n private unsafe static uint D_X3(byte* input, byte s1, byte s2, byte s3)\n {\n return (D_X2(input + 1, s2, s3) << s1) ^ input[0];\n }\n }\n}" }, { "path": "RaptorDB.Common/MurMurHash2.cs", "content": "using System;\n\nnamespace RaptorDB.Common\n{\n //internal static class murmur3\n //{\n // private static uint seed = 7878;\n\n // public static uint MurmurHash3(byte[] data)\n // {\n // const uint c1 = 0xcc9e2d51;\n // const uint c2 = 0x1b873593;\n\n // int curLength = data.Length; /* Current position in byte array */\n // int length = curLength; /* the const length we need to fix tail */\n // uint h1 = seed;\n // uint k1 = 0;\n\n // /* body, eat stream a 32-bit int at a time */\n // int currentIndex = 0;\n // while (curLength >= 4)\n // {\n // /* Get four bytes from the input into an UInt32 */\n // k1 = (uint)(data[currentIndex++]\n // | data[currentIndex++] << 8\n // | data[currentIndex++] << 16\n // | data[currentIndex++] << 24);\n\n // /* bitmagic hash */\n // k1 *= c1;\n // k1 = rotl32(k1, 15);\n // k1 *= c2;\n\n // h1 ^= k1;\n // h1 = rotl32(h1, 13);\n // h1 = h1 * 5 + 0xe6546b64;\n // curLength -= 4;\n // }\n\n // /* tail, the reminder bytes that did not make it to a full int */\n // /* (this switch is slightly more ugly than the C++ implementation\n // * because we can't fall through) */\n // switch (curLength)\n // {\n // case 3:\n // k1 = (UInt32)(data[currentIndex++]\n // | data[currentIndex++] << 8\n // | data[currentIndex++] << 16);\n // k1 *= c1;\n // k1 = rotl32(k1, 15);\n // k1 *= c2;\n // h1 ^= k1;\n // break;\n // case 2:\n // k1 = (UInt32)(data[currentIndex++]\n // | data[currentIndex++] << 8);\n // k1 *= c1;\n // k1 = rotl32(k1, 15);\n // k1 *= c2;\n // h1 ^= k1;\n // break;\n // case 1:\n // k1 = (UInt32)(data[currentIndex++]);\n // k1 *= c1;\n // k1 = rotl32(k1, 15);\n // k1 *= c2;\n // h1 ^= k1;\n // break;\n // };\n\n // // finalization, magic chants to wrap it all up\n // h1 ^= (uint)length;\n // h1 = fmix(h1);\n\n // unchecked\n // {\n // return (uint)h1;\n // }\n // }\n // private static uint rotl32(uint x, byte r)\n // {\n // return (x << r) | (x >> (32 - r));\n // }\n\n // private static uint fmix(uint h)\n // {\n // h ^= h >> 16;\n // h *= 0x85ebca6b;\n // h ^= h >> 13;\n // h *= 0xc2b2ae35;\n // h ^= h >> 16;\n // return h;\n // }\n //}\n\n public class MurmurHash2Unsafe\n {\n public UInt32 Hash(Byte[] data)\n {\n return Hash(data, 0xc58f1a7b);\n }\n const UInt32 m = 0x5bd1e995;\n const Int32 r = 24;\n\n public unsafe UInt32 Hash(Byte[] data, UInt32 seed)\n {\n Int32 length = data.Length;\n if (length == 0)\n return 0;\n UInt32 h = seed ^ (UInt32)length;\n Int32 remainingBytes = length & 3; // mod 4\n Int32 numberOfLoops = length >> 2; // div 4\n fixed (byte* firstByte = &(data[0]))\n {\n UInt32* realData = (UInt32*)firstByte;\n while (numberOfLoops != 0)\n {\n UInt32 k = *realData;\n k *= m;\n k ^= k >> r;\n k *= m;\n\n h *= m;\n h ^= k;\n numberOfLoops--;\n realData++;\n }\n switch (remainingBytes)\n {\n case 3:\n h ^= (UInt16)(*realData);\n h ^= ((UInt32)(*(((Byte*)(realData)) + 2))) << 16;\n h *= m;\n break;\n case 2:\n h ^= (UInt16)(*realData);\n h *= m;\n break;\n case 1:\n h ^= *((Byte*)realData);\n h *= m;\n break;\n default:\n break;\n }\n }\n\n // Do a few final mixes of the hash to ensure the last few\n // bytes are well-incorporated.\n\n h ^= h >> 13;\n h *= m;\n h ^= h >> 15;\n\n return h;\n }\n }\n}\n" }, { "path": "RaptorDB.Common/NetworkClient.cs", "content": "using System;\nusing System.Net.Sockets;\nusing System.Threading;\nusing System.Net;\nusing System.Threading.Tasks;\n\nnamespace RaptorDB.Common\n{\n //\n // Header bits format : 0 - json = 1 , bin = 0 \n // 1 - binaryjson = 1 , text json = 0\n // 2 - compressed = 1 , uncompressed = 0 \n //\n // 0 : data format\n // 1-4 : data length\n\n public class NetworkClient\n {\n public static class Config\n {\n /// \n /// Block buffer size (default = 32kb)\n /// \n public static int BufferSize = 32 * 1024;\n /// \n /// Log data if over (default = 1,000,000)\n /// \n public static int LogDataSizesOver = 1000000;\n /// \n /// Compress data if over (default = 1,000,000)\n /// \n public static int CompressDataOver = 1000000;\n /// \n /// Kill inactive client connections (default = 30sec)\n /// \n public static int KillConnectionSeconds = 30;\n }\n\n public NetworkClient(string server, int port)\n {\n _server = server;\n _port = port;\n }\n private ILog log = LogManager.GetLogger(typeof(NetworkClient));\n private TcpClient _client;\n private string _server;\n private int _port;\n\n public Guid ClientID = Guid.NewGuid();\n public bool UseBJSON = true;\n\n public void Connect()\n {\n _client = new TcpClient(_server, _port);\n _client.SendBufferSize = Config.BufferSize;\n _client.ReceiveBufferSize = _client.SendBufferSize;\n }\n\n public object Send(object data)\n {\n try\n {\n CheckConnection();\n\n byte[] hdr = new byte[5];\n hdr[0] = (UseBJSON ? (byte)3 : (byte)0);\n byte[] dat = fastBinaryJSON.BJSON.ToBJSON(data);\n bool compressed = false;\n if (dat.Length > NetworkClient.Config.CompressDataOver)\n {\n log.Debug(\"compressing data over limit : \" + dat.Length.ToString(\"#,#\"));\n compressed = true;\n dat = MiniLZO.Compress(dat);\n log.Debug(\"new size : \" + dat.Length.ToString(\"#,#\"));\n }\n byte[] len = Helper.GetBytes(dat.Length, false);\n hdr[0] = (byte)(3 + (compressed ? 4 : 0));\n Array.Copy(len, 0, hdr, 1, 4);\n _client.Client.Send(hdr);\n _client.Client.Send(dat);\n\n byte[] rechdr = new byte[5];\n using (NetworkStream n = new NetworkStream(_client.Client))\n {\n n.Read(rechdr, 0, 5);\n int c = Helper.ToInt32(rechdr, 1);\n byte[] recd = new byte[c];\n int bytesRead = 0;\n int chunksize = 1;\n while (bytesRead < c && chunksize > 0)\n bytesRead +=\n chunksize = n.Read\n (recd, bytesRead, c - bytesRead);\n if ((rechdr[0] & (byte)4) == (byte)4)\n recd = MiniLZO.Decompress(recd);\n if ((rechdr[0] & (byte)3) == (byte)3)\n return fastBinaryJSON.BJSON.ToObject(recd);\n }\n }\n catch\n {\n\n }\n return null;\n }\n\n private void CheckConnection()\n {\n // check connected state before sending\n\n if (_client == null || !_client.Connected)\n Connect();\n }\n\n public void Close()\n {\n if (_client != null)\n {\n _client.Close();\n }\n }\n }\n\n public class NetworkServer\n {\n public delegate object ProcessPayload(object data);\n\n private ILog log = LogManager.GetLogger(typeof(NetworkServer));\n ProcessPayload _handler;\n private bool _run = true;\n private int count = 0;\n private int _port;\n\n public void Start(int port, ProcessPayload handler)\n {\n _handler = handler;\n _port = port;\n ThreadPool.SetMinThreads(50, 50);\n System.Timers.Timer t = new System.Timers.Timer(1000);\n t.AutoReset = true;\n t.Start();\n t.Elapsed += new System.Timers.ElapsedEventHandler(t_Elapsed);\n Task.Factory.StartNew(() => Run(), TaskCreationOptions.AttachedToParent);\n }\n\n private void Run()\n {\n TcpListener listener = new TcpListener(IPAddress.Any, _port);\n listener.Start();\n\n while (_run)\n {\n try\n {\n TcpClient c = listener.AcceptTcpClient();\n Task.Factory.StartNew(() => Accept(c));\n }\n catch (Exception ex) { log.Error(ex); }\n }\n }\n\n void t_Elapsed(object sender, System.Timers.ElapsedEventArgs e)\n {\n if (count > 0)\n log.Info(\"tcp connects/sec = \" + count);\n count = 0;\n }\n\n public void Stop()\n {\n _run = false;\n }\n\n void Accept(TcpClient client)\n {\n using (NetworkStream n = client.GetStream())\n {\n while (client.Connected)\n {\n this.count++;\n byte[] c = new byte[5];\n n.Read(c, 0, 5);\n int count = BitConverter.ToInt32(c, 1);\n byte[] data = new byte[count];\n int bytesRead = 0;\n int chunksize = 1;\n while (bytesRead < count && chunksize > 0)\n bytesRead +=\n chunksize = n.Read\n (data, bytesRead, count - bytesRead);\n\n object o = fastBinaryJSON.BJSON.ToObject(data);\n if ((c[0] & (byte)4) == (byte)4)\n data = MiniLZO.Decompress(data);\n\n object r = _handler(o);\n bool compressed = false;\n var dataret = fastBinaryJSON.BJSON.ToBJSON(r);\n r = null;\n if (dataret.Length > NetworkClient.Config.CompressDataOver)\n {\n log.Debug(\"compressing data over limit : \" + dataret.Length.ToString(\"#,#\"));\n compressed = true;\n dataret = MiniLZO.Compress(dataret);\n log.Debug(\"new size : \" + dataret.Length.ToString(\"#,#\"));\n }\n if (dataret.Length > NetworkClient.Config.LogDataSizesOver)\n log.Debug(\"data size (bytes) = \" + dataret.Length.ToString(\"#,#\"));\n\n byte[] b = BitConverter.GetBytes(dataret.Length);\n byte[] hdr = new byte[5];\n hdr[0] = (byte)(3 + (compressed ? 4 : 0));\n Array.Copy(b, 0, hdr, 1, 4);\n n.Write(hdr, 0, 5);\n n.Write(dataret, 0, dataret.Length);\n\n //n.Flush();\n //return;\n int wait = 0;\n bool close = false;\n var dt = FastDateTime.Now;\n while (n.DataAvailable == false && close == false)\n {\n wait++;\n if (wait < 10000) // kludge : for insert performance\n Thread.Sleep(0);\n else\n {\n Thread.Sleep(1);\n // wait done -> close connection \n if (FastDateTime.Now.Subtract(dt).TotalSeconds > NetworkClient.Config.KillConnectionSeconds)\n close = true;\n }\n }\n if (close)\n break;\n }\n n.Close();\n }\n client.Close();\n }\n }\n}\n" }, { "path": "RaptorDB.Common/Packets.cs", "content": "using System;\n\nnamespace RaptorDB.Common\n{\n public class Packet\n {\n public Packet()\n {\n OrderBy = \"\";\n }\n public string Username { get; set; }\n public string PasswordHash { get; set; }\n //public int Token { get; set; }\n //public int Session { get; set; }\n public string Command { get; set; }\n public object Data { get; set; }\n public Guid Docid { get; set; }\n public string Viewname { get; set; }\n public int Start { get; set; }\n public int Count { get; set; }\n public string OrderBy { get; set; }\n public Guid ClientID { get; set; }\n public string InstanceName { get; set; }\n }\n\n public class ReturnPacket\n {\n public ReturnPacket()\n {\n\n }\n public ReturnPacket(bool ok)\n {\n OK = ok;\n }\n public ReturnPacket(bool ok, string err)\n {\n OK = ok;\n Error = err;\n }\n public string Error { get; set; }\n public bool OK { get; set; }\n //public int Token { get; set; }\n //public int Session { get; set; }\n public object Data { get; set; }\n }\n}\n" }, { "path": "RaptorDB.Common/Properties/AssemblyInfo.cs", "content": "using System.Reflection;\nusing System.Security;\n\n[assembly: AssemblyTitle(\"RaptorDB.Common\")]\n[assembly: AssemblyDescription(\"Common classes for RaptorDB client and server\")]\n[assembly: AssemblyProduct(\"RaptorDB.Common\")]\n\n\n\n\n" }, { "path": "RaptorDB.Common/RaptorDB.Common.csproj", "content": "\n\n \n Debug\n AnyCPU\n 8.0.30703\n 2.0\n {32331D51-5BE0-41E2-AF1A-9B086C5AE809}\n Library\n Properties\n RaptorDB.Common\n RaptorDB.Common\n 512\n \n \n True\n full\n true\n bin\\Debug\\\n TRACE;DEBUG;NETSERVER; NETJSON; net4;NET4\n prompt\n 4\n True\n false\n \n \n pdbonly\n True\n bin\\Release\\\n TRACE;NETSERVER; NETJSON; net4\n prompt\n 4\n false\n true\n \n \n ..\\raptordb.snk\n \n \n true\n \n \n \n \n \n \n \n \n \n BuildVersion.cs\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n md \"$(SolutionDir)output\\net40\"\ncopy \"$(TargetPath)\" \"$(SolutionDir)output\\net40\\$(TargetFileName)\"\n \n \n \"$(SolutionDir)tools\\buildversion.exe\" \"$(SolutionDir)buildversion.cs\"\n \n \n" }, { "path": "RaptorDB.Common/RaptorDBClient.cs", "content": "using System;\nusing System.Linq;\nusing System.Text;\nusing RaptorDB.Common;\nusing System.Linq.Expressions;\nusing System.Reflection;\n\nnamespace RaptorDB\n{\n public class KVHF : IKeyStoreHF\n {\n public KVHF(NetworkClient client, string username, string password)\n {\n _client = client;\n _username = username;\n _password = password;\n }\n\n NetworkClient _client;\n private string _username;\n private string _password;\n\n\n public object GetObjectHF(string key)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.GetObjectHF;\n p.Data = key;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n if (ret.OK)\n return ret.Data;\n else\n return null;\n }\n\n public bool SetObjectHF(string key, object obj)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.SetObjectHF;\n p.Data = new object[] { key, obj };\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n\n return ret.OK;\n }\n\n public bool DeleteKeyHF(string key)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.DeleteKeyHF;\n p.Data = key;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n\n return (bool)ret.Data;\n }\n\n public int CountHF()\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.CountHF;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n\n return (int)ret.Data;\n }\n\n public bool ContainsHF(string key)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.ContainsHF;\n p.Data = key;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n\n return (bool)ret.Data;\n }\n\n public string[] GetKeysHF()\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.GetKeysHF;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n\n return ((object[])ret.Data).Cast().ToArray();\n }\n\n public void CompactStorageHF()\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.CompactStorageHF;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n\n return;\n }\n\n private Packet CreatePacket()\n {\n Packet p = new Packet();\n p.Username = _username;\n p.PasswordHash = Helper.MurMur.Hash(Encoding.UTF8.GetBytes(_username + \"|\" + _password)).ToString();\n p.ClientID = _client.ClientID;\n return p;\n }\n\n public int Increment(string key, int amount)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.IncrementHF;\n p.Data = new object[] { key, amount };\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n\n return (int)ret.Data;\n }\n\n public int Decrement(string key, int amount)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.DecrementHF;\n p.Data = new object[] { key, amount };\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n\n return (int)ret.Data;\n }\n\n public decimal Increment(string key, decimal amount)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.IncrementHF;\n p.Data = new object[] { key, amount };\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n\n return (decimal)ret.Data;\n }\n\n public decimal Decrement(string key, decimal amount)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.DecrementHF;\n p.Data = new object[] { key, amount };\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n\n return (decimal)ret.Data;\n }\n }\n\n public class RaptorDBClient : IRaptorDB\n {\n public RaptorDBClient(string server, int port, string username, string password)\n {\n _username = username;\n _password = password;\n _client = new NetworkClient(server, port);\n // speed settings\n fastJSON.JSON.Parameters.ParametricConstructorOverride = true;\n fastBinaryJSON.BJSON.Parameters.ParametricConstructorOverride = true;\n _kv = new KVHF(_client, _username, _password);\n }\n\n private KVHF _kv;\n private NetworkClient _client;\n private string _username;\n private string _password;\n private SafeDictionary _assembly = new SafeDictionary();\n\n /// \n /// Save a document to RaptorDB\n /// \n /// \n /// \n /// \n /// \n public bool Save(Guid docID, T document)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.Save;\n p.Docid = docID;\n p.Data = document;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n return ret.OK;\n }\n\n /// \n /// Save a file to RaptorDB\n /// \n /// \n /// \n /// \n public bool SaveBytes(Guid fileID, byte[] bytes)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.SaveBytes;\n p.Docid = fileID;\n p.Data = bytes;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n return ret.OK;\n }\n\n /// \n /// Query any view -> get all rows\n /// \n /// \n /// \n /// \n public Result Query(string viewname)\n {\n return Query(viewname, 0, -1);\n }\n\n /// \n /// Query a view using a string filter\n /// \n /// \n /// \n /// \n public Result Query(string viewname, string filter)\n {\n return Query(viewname, filter, 0, -1);\n }\n\n /// \n /// Fetch a document by it's ID\n /// \n /// \n /// \n public object Fetch(Guid docID)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.Fetch;\n p.Docid = docID;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n if (ret.OK)\n return ret.Data;\n else\n return null;\n }\n\n /// \n /// Fetch file data by it's ID\n /// \n /// \n /// \n public byte[] FetchBytes(Guid fileID)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.FetchBytes;\n p.Docid = fileID;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n if (ret.OK)\n return (byte[])ret.Data;\n else\n return null;\n }\n\n /// \n /// Shutdown and cleanup \n /// \n public void Shutdown()\n {\n try\n {\n // send close packet\n Packet p = CreatePacket();\n p.Command = \"_close\";\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n _client.Close();\n }\n catch { }\n\n }\n\n /// \n /// Backup the data file in incremental mode to the RaptorDB folder\n /// \n /// \n public bool Backup()\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.Backup;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n return ret.OK;\n }\n\n /// \n /// Restore backup files stored in RaptorDB folder\n /// \n public void Restore()\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.Restore;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n }\n\n /// \n /// Delete a document (the actual data is not deleted just marked so) \n /// \n /// \n /// \n public bool Delete(Guid docid)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.Delete;\n p.Docid = docid;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n return ret.OK;\n }\n\n /// \n /// Delete a file (the actual data is not deleted just marked so) \n /// \n /// \n /// \n public bool DeleteBytes(Guid fileid)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.DeleteBytes;\n p.Docid = fileid;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n return ret.OK;\n }\n\n /// \n /// Add a user for server mode login\n /// \n /// \n /// \n /// \n /// \n public bool AddUser(string username, string oldpassword, string newpassword)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.AddUser;\n p.Data = new object[] { username, oldpassword, newpassword };\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n return ret.OK;\n }\n\n /// \n /// Execute server side queries\n /// \n /// \n /// \n /// \n public object[] ServerSide(ServerSideFunc func, string filter)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.ServerSide;\n p.Data = new object[] {\n func.Method.ReflectedType.AssemblyQualifiedName,\n func.Method.Name,\n filter };\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n return (object[])ret.Data;\n }\n\n /// \n /// Execute server side queries\n /// \n /// \n /// \n /// \n /// \n public object[] ServerSide(ServerSideFunc func, Expression> filter)\n {\n LINQString ls = new LINQString();\n ls.Visit(filter);\n\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.ServerSide;\n p.Data = new object[] {\n func.Method.ReflectedType.AssemblyQualifiedName,\n func.Method.Name,\n ls.sb.ToString() };\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n return (object[])ret.Data;\n }\n\n /// \n /// Full text search the complete original document \n /// \n /// \n /// \n public int[] FullTextSearch(string filter)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.FullText;\n p.Data = new object[] { filter };\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n object[] r = (object[])ret.Data;\n return Array.ConvertAll(r, i => (int)i);\n }\n\n private Packet CreatePacket()\n {\n Packet p = new Packet();\n p.Username = _username;\n p.PasswordHash = Helper.MurMur.Hash(Encoding.UTF8.GetBytes(_username + \"|\" + _password)).ToString();\n p.ClientID = _client.ClientID;\n return p;\n }\n\n /// \n /// Query all data in a view with paging\n /// \n /// \n /// \n /// \n /// \n public Result Query(string viewname, int start, int count)\n {\n return Query(viewname, \"\", start, count);\n }\n\n /// \n /// Query a View with a string filter with paging\n /// \n /// \n /// \n /// \n /// \n /// \n public Result Query(string viewname, string filter, int start, int count, string orderby)\n {\n bool b = false;\n // check if return type exists and copy assembly if needed\n if (_assembly.TryGetValue(viewname, out b) == false)\n {\n Packet pp = CreatePacket();\n pp.Command = \"\" + COMMANDS.CheckAssembly;\n pp.Viewname = viewname;\n ReturnPacket r = (ReturnPacket)_client.Send(pp);\n string type = r.Error;\n Type t = Type.GetType(type);\n if (t == null)\n {\n if (r.Data != null)\n {\n var a = Assembly.Load((byte[])r.Data);\n _assembly.Add(viewname, true);\n }\n }\n else\n _assembly.Add(viewname, true);\n }\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.QueryStr;\n p.Viewname = viewname;\n p.Data = filter;\n p.Start = start;\n p.Count = count;\n p.OrderBy = orderby;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n return (Result)ret.Data;\n }\n\n /// \n /// Query a View with a LINQ filter with paging\n /// \n /// \n /// \n /// \n /// \n /// \n /// \n public Result Query(string viewname, Expression> filter, int start, int count, string orderby)\n {\n LINQString ls = new LINQString();\n ls.Visit(filter);\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.QueryStr;\n p.Viewname = viewname;\n p.Start = start;\n p.Count = count;\n p.Data = ls.sb.ToString();\n p.OrderBy = orderby;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n return (Result)ret.Data;\n }\n\n /// \n /// Count rows\n /// \n /// \n /// \n public int Count(string viewname)\n {\n return Count(viewname, \"\");\n }\n\n /// \n /// Count rows with a string filter\n /// \n /// \n /// \n /// \n public int Count(string viewname, string filter)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.CountStr;\n p.Viewname = viewname;\n p.Data = filter;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n return (int)ret.Data;\n }\n\n /// \n /// Query with LINQ filter\n /// \n /// \n /// \n /// \n public Result Query(Expression> filter)\n {\n return Query(filter, 0, -1, \"\");\n }\n\n /// \n /// Query with LINQ filter and paging\n /// \n /// \n /// \n /// \n /// \n /// \n public Result Query(Expression> filter, int start, int count, string orderby)\n {\n LINQString ls = new LINQString();\n ls.Visit(filter);\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.QueryType;\n p.Start = start;\n p.Count = count;\n p.OrderBy = orderby;\n p.Data = new object[] { typeof(TRowSchema).AssemblyQualifiedName, ls.sb.ToString() };\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n Result res = (Result)ret.Data;\n return GenericResult(res);\n }\n\n private static Result GenericResult(Result res)\n {\n // FEATURE : dirty hack here to cleanup\n Result result = new Result();\n if (res != null)\n {\n result.Count = res.Count;\n result.EX = res.EX;\n result.OK = res.OK;\n result.TotalCount = res.TotalCount;\n if (res.Rows != null)\n result.Rows = res.Rows.Cast().ToList();\n else\n result.Rows = null;\n }\n return result;\n }\n\n /// \n /// Query with string filter\n /// \n /// \n /// \n /// \n public Result Query(string filter)\n {\n return Query(filter, 0, -1, \"\");\n }\n\n /// \n /// Query with string filter and paging\n /// \n /// \n /// \n /// \n /// \n /// \n public Result Query(string filter, int start, int count, string orderby)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.QueryType;\n p.Start = start;\n p.Count = count;\n p.OrderBy = orderby;\n p.Data = new object[] { typeof(TRowSchema).AssemblyQualifiedName, filter };\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n Result res = (Result)ret.Data;\n return GenericResult(res);\n }\n\n /// \n /// Count with LINQ filter\n /// \n /// \n /// \n /// \n public int Count(Expression> filter)\n {\n LINQString ls = new LINQString();\n ls.Visit(filter);\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.GCount;\n p.Viewname = typeof(TRowSchema).AssemblyQualifiedName;\n p.Data = ls.sb.ToString();\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n return (int)ret.Data;\n }\n\n /// \n /// Fetch the document change history\n /// \n /// \n /// \n public int[] FetchHistory(Guid docid)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.DocHistory;\n p.Docid = docid;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n object[] r = (object[])ret.Data;\n return Array.ConvertAll(r, i => (int)i);\n }\n\n /// \n /// Fetch the file change history\n /// \n /// \n /// \n public int[] FetchBytesHistory(Guid fileid)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.FileHistory;\n p.Docid = fileid;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n object[] r = (object[])ret.Data;\n return Array.ConvertAll(r, i => (int)i);\n }\n\n /// \n /// Fetch a specific document version\n /// \n /// \n /// \n public object FetchVersion(int versionNumber)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.FetchVersion;\n p.Data = versionNumber;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n return ret.Data;\n }\n\n /// \n /// Fetch a specific file version\n /// \n /// \n /// \n public byte[] FetchBytesVersion(int versionNumber)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.FetchFileVersion;\n p.Data = versionNumber;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n return (byte[])ret.Data;\n }\n\n /// \n /// Query a View with a string filter with paging\n /// \n /// \n /// \n /// \n /// \n /// \n public Result Query(string viewname, string filter, int start, int count)\n {\n return this.Query(viewname, filter, start, count, \"\");\n }\n\n /// \n /// Query a view with paging\n /// \n /// \n /// \n /// \n /// \n /// \n /// \n public Result Query(string viewname, Expression> filter, int start, int count)\n {\n return this.Query(viewname, filter, start, count, \"\");\n }\n\n /// \n /// Query a view with paging\n /// \n /// \n /// \n /// \n /// \n /// \n public Result Query(Expression> filter, int start, int count)\n {\n return Query(filter, start, count, \"\");\n }\n\n /// \n /// Query a view with paging\n /// \n /// \n /// \n /// \n /// \n /// \n public Result Query(string filter, int start, int count)\n {\n return Query(filter, start, count, \"\");\n }\n\n /// \n /// Fetch a change history for a document with dates\n /// \n /// \n /// \n public HistoryInfo[] FetchHistoryInfo(Guid docid)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.FetchHistoryInfo;\n p.Docid = docid;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n return (HistoryInfo[])ret.Data;\n }\n\n /// \n /// Fetch a change history for a file with dates\n /// \n /// \n /// \n public HistoryInfo[] FetchBytesHistoryInfo(Guid docid)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.FetchByteHistoryInfo;\n p.Docid = docid;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n return (HistoryInfo[])ret.Data;\n }\n\n /// \n /// Delete directly from a view using a filter\n /// \n /// \n /// \n /// \n public int ViewDelete(Expression> filter)\n {\n LINQString ls = new LINQString();\n ls.Visit(filter);\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.ViewDelete_t;\n p.Data = new object[] { typeof(TRowSchema).AssemblyQualifiedName, ls.sb.ToString() };\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n return (int)ret.Data;\n }\n\n /// \n /// Delete directly from a view using a filter\n /// \n /// \n /// \n /// \n public int ViewDelete(string viewname, string filter)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.ViewDelete;\n p.Data = new object[] { viewname, filter };\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n return (int)ret.Data;\n }\n\n /// \n /// Insert directly into a view\n /// \n /// \n /// \n /// \n /// \n public bool ViewInsert(Guid id, TRowSchema row)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.ViewInsert_t;\n p.Docid = id;\n p.Data = new object[] { typeof(TRowSchema).AssemblyQualifiedName, row };\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n return (bool)ret.Data;\n }\n\n /// \n /// Insert directly into a view\n /// \n /// \n /// \n /// \n /// \n public bool ViewInsert(string viewname, Guid id, object row)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.ViewInsert;\n p.Docid = id;\n p.Data = new object[] { viewname, row };\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n return (bool)ret.Data;\n }\n\n /// \n /// Get the number of documents in the storage file regardless of versions\n /// \n /// \n public long DocumentCount()\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.DocCount;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n return (long)ret.Data;\n }\n\n public IKeyStoreHF GetKVHF()\n {\n return _kv;\n }\n\n public object[] ServerSide(ServerSideFuncWithArgs func, string filter, params object[] args)\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.ServerSideWithArgs;\n p.Data = new object[] {\n func.Method.ReflectedType.AssemblyQualifiedName,\n func.Method.Name,\n filter ,\n args };\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n return (object[])ret.Data;\n }\n\n public object[] ServerSide(ServerSideFuncWithArgs func, Expression> filter, params object[] args)\n {\n LINQString ls = new LINQString();\n ls.Visit(filter);\n\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.ServerSideWithArgs;\n p.Data = new object[] {\n func.Method.ReflectedType.AssemblyQualifiedName,\n func.Method.Name,\n ls.sb.ToString(),\n args };\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n return (object[])ret.Data;\n }\n\n public T Fetch(Guid docID) where T : class\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.Fetch;\n p.Docid = docID;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n if (ret.OK)\n return (ret.Data as T);\n else\n return null;\n }\n\n public void FreeMemory()\n {\n Packet p = CreatePacket();\n p.Command = \"\" + COMMANDS.FreeMemory;\n ReturnPacket ret = (ReturnPacket)_client.Send(p);\n }\n\n public void SaveToDocsOnViewInsert(bool yes)\n {\n // FIX: set on server\n }\n }\n}\n" }, { "path": "RaptorDB.Common/SafeDictionary.cs", "content": "using System;\nusing System.Collections;\nusing System.Collections.Generic;\nusing System.Runtime.CompilerServices;\nusing System.Runtime.InteropServices;\nusing System.Text;\n\nnamespace RaptorDB.Common\n{\n public interface IKV\n {\n bool TryGetValue(T key, out V val);\n int Count();\n IEnumerator> GetEnumerator();\n void Add(T key, V value);\n T[] Keys();\n bool Remove(T key);\n void Clear();\n V GetValue(T key);\n // safesortedlist only\n //V GetValue(int index);\n //T GetKey(int index);\n }\n\n public class ReferenceEqualityComparer : IEqualityComparer, IEqualityComparer\n {\n public static ReferenceEqualityComparer Default { get; } = new ReferenceEqualityComparer();\n\n public new bool Equals(object x, object y) => x.Equals(y);\n public int GetHashCode(object obj) => RuntimeHelpers.GetHashCode(obj);\n }\n\n public class SafeDictionary : IKV\n {\n private readonly object _Padlock = new object();\n private readonly Dictionary _Dictionary;\n\n public SafeDictionary(int capacity)\n {\n _Dictionary = new Dictionary(capacity);\n }\n\n public SafeDictionary()\n {\n _Dictionary = new Dictionary();\n }\n\n public bool TryGetValue(TKey key, out TValue value)\n {\n lock (_Padlock)\n return _Dictionary.TryGetValue(key, out value);\n }\n\n public TValue this[TKey key]\n {\n get\n {\n lock (_Padlock)\n return _Dictionary[key];\n }\n set\n {\n lock (_Padlock)\n _Dictionary[key] = value;\n }\n }\n\n public int Count()\n {\n lock (_Padlock) return _Dictionary.Count;\n }\n\n public IEnumerator> GetEnumerator()\n {\n return ((ICollection>)_Dictionary).GetEnumerator();\n }\n\n public void Add(TKey key, TValue value)\n {\n lock (_Padlock)\n {\n if (_Dictionary.ContainsKey(key) == false)\n _Dictionary.Add(key, value);\n else\n _Dictionary[key] = value;\n }\n }\n\n public TKey[] Keys()\n {\n lock (_Padlock)\n {\n TKey[] keys = new TKey[_Dictionary.Keys.Count];\n _Dictionary.Keys.CopyTo(keys, 0);\n return keys;\n }\n }\n\n public bool Remove(TKey key)\n {\n if (key == null)\n return true;\n lock (_Padlock)\n {\n return _Dictionary.Remove(key);\n }\n }\n\n public void Clear()\n {\n lock (_Padlock)\n _Dictionary.Clear();\n }\n\n public TValue GetValue(TKey key)\n {\n lock (_Padlock)\n return _Dictionary[key];\n }\n }\n\n public class SafeSortedList : IKV\n {\n private object _padlock = new object();\n SortedList _list = new SortedList();\n\n public int Count()\n {\n lock (_padlock) return _list.Count;\n }\n\n public void Add(T key, V val)\n {\n lock (_padlock)\n {\n if (_list.ContainsKey(key) == false)\n _list.Add(key, val);\n else\n _list[key] = val;\n }\n }\n\n public bool Remove(T key)\n {\n if (key == null)\n return true;\n lock (_padlock)\n return _list.Remove(key);\n }\n\n public T GetKey(int index)\n {\n lock (_padlock)\n if (index < _list.Count)\n return _list.Keys[index];\n else\n return default(T);\n }\n\n public V GetValue(int index)\n {\n lock (_padlock)\n if (index < _list.Count)\n return _list.Values[index];\n else\n return default(V);\n }\n\n public T[] Keys()\n {\n lock (_padlock)\n {\n T[] keys = new T[_list.Keys.Count];\n _list.Keys.CopyTo(keys, 0);\n return keys;\n }\n }\n\n public V this[T key]\n {\n get\n {\n lock (_padlock)\n return _list[key];\n }\n set\n {\n lock (_padlock)\n _list[key] = value;\n }\n }\n\n public IEnumerator> GetEnumerator()\n {\n return ((ICollection>)_list).GetEnumerator();\n }\n\n public bool TryGetValue(T key, out V value)\n {\n lock (_padlock)\n return _list.TryGetValue(key, out value);\n }\n\n public void Clear()\n {\n lock (_padlock)\n _list.Clear();\n }\n\n public V GetValue(T key)\n {\n lock (_padlock)\n return _list[key];\n }\n }\n\n //------------------------------------------------------------------------------------------------------------------\n public static class FastDateTime\n {\n public static TimeSpan LocalUtcOffset;\n\n public static DateTime Now\n {\n get { return DateTime.SpecifyKind(DateTime.UtcNow + LocalUtcOffset, DateTimeKind.Local); }\n\n }\n\n static FastDateTime()\n {\n LocalUtcOffset = TimeZone.CurrentTimeZone.GetUtcOffset(DateTime.Now);\n }\n }\n //------------------------------------------------------------------------------------------------------------------\n\n public static class Helper\n {\n public static MurmurHash2Unsafe MurMur = new MurmurHash2Unsafe();\n public static int CompareMemCmp(byte[] left, byte[] right)\n {\n int c = left.Length;\n if (c > right.Length)\n c = right.Length;\n return memcmp(left, right, c);\n }\n\n [DllImport(\"msvcrt.dll\", CallingConvention = CallingConvention.Cdecl)]\n private static extern int memcmp(byte[] arr1, byte[] arr2, int cnt);\n\n public static int ToInt32(byte[] value, int startIndex, bool reverse)\n {\n if (reverse)\n {\n byte[] b = new byte[4];\n Buffer.BlockCopy(value, startIndex, b, 0, 4);\n Array.Reverse(b);\n return ToInt32(b, 0);\n }\n\n return ToInt32(value, startIndex);\n }\n\n public static unsafe int ToInt32(byte[] value, int startIndex)\n {\n fixed (byte* numRef = &(value[startIndex]))\n {\n return *((int*)numRef);\n }\n }\n\n public static long ToInt64(byte[] value, int startIndex, bool reverse)\n {\n if (reverse)\n {\n byte[] b = new byte[8];\n Buffer.BlockCopy(value, startIndex, b, 0, 8);\n Array.Reverse(b);\n return ToInt64(b, 0);\n }\n return ToInt64(value, startIndex);\n }\n\n public static unsafe long ToInt64(byte[] value, int startIndex)\n {\n fixed (byte* numRef = &(value[startIndex]))\n {\n return *(((long*)numRef));\n }\n }\n\n public static short ToInt16(byte[] value, int startIndex, bool reverse)\n {\n if (reverse)\n {\n byte[] b = new byte[2];\n Buffer.BlockCopy(value, startIndex, b, 0, 2);\n Array.Reverse(b);\n return ToInt16(b, 0);\n }\n return ToInt16(value, startIndex);\n }\n\n public static unsafe short ToInt16(byte[] value, int startIndex)\n {\n fixed (byte* numRef = &(value[startIndex]))\n {\n return *(((short*)numRef));\n }\n }\n\n public static unsafe byte[] GetBytes(long num, bool reverse)\n {\n byte[] buffer = new byte[8];\n fixed (byte* numRef = buffer)\n {\n *((long*)numRef) = num;\n }\n if (reverse)\n Array.Reverse(buffer);\n return buffer;\n }\n\n public static unsafe byte[] GetBytes(int num, bool reverse)\n {\n byte[] buffer = new byte[4];\n fixed (byte* numRef = buffer)\n {\n *((int*)numRef) = num;\n }\n if (reverse)\n Array.Reverse(buffer);\n return buffer;\n }\n\n public static unsafe byte[] GetBytes(short num, bool reverse)\n {\n byte[] buffer = new byte[2];\n fixed (byte* numRef = buffer)\n {\n *((short*)numRef) = num;\n }\n if (reverse)\n Array.Reverse(buffer);\n return buffer;\n }\n\n public static byte[] GetBytes(string s)\n {\n return Encoding.UTF8.GetBytes(s); // TODO : change to unicode ??\n }\n\n public static string GetString(byte[] buffer, int index, short length)\n {\n return Encoding.UTF8.GetString(buffer, index, length); // TODO : change to unicode ??\n }\n }\n}\n" }, { "path": "RaptorDB.Common/View.cs", "content": "using System;\nusing System.Collections.Generic;\nusing System.Xml.Serialization;\n\nnamespace RaptorDB\n{\n public abstract class ViewBase\n {\n public delegate void MapFunctionDelgate(IMapAPI api, Guid docid, V doc);\n /// \n /// Increment this when you change view definitions so the engine can rebuild the contents\n /// \n public int Version { get; set; }\n \n /// \n /// Name of the view will be used for foldernames and filename and generated code\n /// \n public string Name { get; set;}\n \n /// \n /// A text for describing this views purpose for other developers \n /// \n public string Description { get; set; }\n \n /// \n /// Column definitions for the view storage \n /// \n [XmlIgnore]\n public Type Schema { get; set; }\n \n /// \n /// Is this the primary list and will be populated synchronously\n /// \n public bool isPrimaryList { get; set; }\n \n /// \n /// Is this view active and will recieve data\n /// \n public bool isActive { get; set; }\n \n /// \n /// Delete items on DocID before inserting new rows (default = true)\n /// \n public bool DeleteBeforeInsert { get; set; }\n\n /// \n /// Index in the background : better performance but reads might not have all the data\n /// \n public bool BackgroundIndexing { get; set; }\n\n /// \n /// Save documents to this view in the save process, like primary views\n /// \n public bool ConsistentSaveToThisView { get; set; }\n\n /// \n /// Apply to a Primary View and all the mappings of all views will be done in a transaction.\n /// You can use Rollback for failures.\n /// \n public bool TransactionMode { get; set; }\n\n /// \n /// When defining your own schema and you don't want dependancies to RaptorDB to propogate through your code\n /// define your full text columns here\n /// \n public List FullTextColumns;\n \n /// \n /// When defining your own schems and you don't want dependancies to RaptorDB to propogate through your code \n /// define your case insensitive columns here\n /// \n public List CaseInsensitiveColumns;\n\n public Dictionary StringIndexLength;\n\n /// \n /// Columns that you don't want to index\n /// \n public List NoIndexingColumns;\n }\n\n\n public class View : ViewBase\n {\n public View()\n {\n isActive = true;\n DeleteBeforeInsert = true;\n BackgroundIndexing = true;\n FullTextColumns = new List();\n CaseInsensitiveColumns = new List();\n StringIndexLength = new Dictionary();\n NoIndexingColumns = new List();\n isPrimaryList = true;\n ConsistentSaveToThisView = true;\n }\n\n /// \n /// Inline delegate for the mapper function used for quick applications \n /// \n [XmlIgnore]\n public MapFunctionDelgate Mapper { get; set; }\n\n public Result Verify()\n {\n if (Name == null || Name == \"\") \n throw new Exception(\"Name must be given\");\n if (Schema == null) \n throw new Exception(\"Schema must be defined\");\n if (Schema.IsSubclassOf(typeof(RDBSchema)) == false)\n {\n var pi = Schema.GetProperty(\"docid\");\n if (pi == null || pi.PropertyType != typeof(Guid))\n {\n var fi = Schema.GetField(\"docid\");\n if( fi == null || fi.FieldType != typeof(Guid))\n throw new Exception(\"The schema must be derived from RaptorDB.RDBSchema or must contain a 'docid' Guid field or property\");\n }\n }\n if (Mapper == null) \n throw new Exception(\"A map function must be defined\");\n\n if (TransactionMode == true && isPrimaryList == false)\n throw new Exception(\"Transaction mode can only be enabled on Primary Views\");\n \n // FEATURE : add more verifications\n return new Result(true);\n }\n }\n}\n" }, { "path": "RaptorDB.Common/ZipStorer.cs", "content": "// ZipStorer, by Jaime Olivares\n// Website: zipstorer.codeplex.com\n// Version: 2.35 (March 14, 2010)\n\nusing System;\nusing System.Collections.Generic;\nusing System.IO;\nusing System.IO.Compression;\nusing System.Text;\n\nnamespace RaptorDB.Common\n{\n public static class ZIP\n {\n #region [ usage sample ]\n static void Main(string[] args)\n {\n if (args.Length == 0)\n {\n printhelp();\n return;\n }\n\n var t = args[0].ToLower();\n var r = false;\n var fn = \"\";\n var dir = \"\";\n if (args[1].StartsWith(\"/\") || args[1].StartsWith(\"-\"))\n {\n r = true;\n fn = args[2];\n dir = args[3];\n }\n else\n {\n fn = args[1];\n dir = args[2];\n }\n if (t == \"z\")\n ZIP.Compress(fn, dir, r, log);\n else if (t == \"u\")\n ZIP.Decompress(fn, dir, log);\n else\n printhelp();\n }\n\n private static void log(string msg)\n {\n Console.WriteLine(msg);\n }\n\n private static void printhelp()\n {\n Console.WriteLine(\"usage :\");\n Console.WriteLine(\" zip : z /r filename.zip path\");\n Console.WriteLine(\" unzip : u filename.zip path\");\n }\n #endregion\n\n public static void Compress(string filename, string folder, bool recursive, Action log)\n {\n ZipStorer zip;\n\n if (File.Exists(filename) == false)\n // Creates a new zip file\n zip = ZipStorer.Create(filename, \"\");\n else\n // Opens existing zip file\n zip = ZipStorer.Open(filename, FileAccess.Write);\n\n zip.EncodeUTF8 = true;\n\n // Stores all the files into the zip file\n var dir = new DirectoryInfo(folder);\n var prefix = dir.FullName;\n if (prefix.EndsWith(Path.DirectorySeparatorChar.ToString()) == false)\n prefix += Path.DirectorySeparatorChar;\n\n docompressdirectory(zip, dir.FullName, prefix, recursive, log);\n // Updates and closes the zip file\n zip.Close();\n }\n\n public static void Decompress(string filename, string outputfolder, Action log)\n {\n ZipStorer zip;\n\n if (File.Exists(filename))\n // Opens existing zip file\n zip = ZipStorer.Open(filename, FileAccess.Read);\n else\n return;\n\n // Read all directory contents\n List dir = zip.ReadCentralDir();\n\n // Extract all files in target directory\n string path;\n bool result;\n foreach (ZipStorer.ZipFileEntry entry in dir)\n {\n path = Path.Combine(outputfolder, entry.FilenameInZip);\n result = zip.ExtractFile(entry, path);\n log?.Invoke(path);\n }\n zip.Close();\n }\n\n private static void docompressdirectory(ZipStorer zip, string dir, string prefix, bool recursive, Action log)\n {\n if (recursive)\n foreach (var d in Directory.GetDirectories(dir))\n docompressdirectory(zip, d, prefix, recursive, log);\n\n foreach (string path in Directory.GetFiles(dir))\n {\n var fn = path.Replace(prefix, \"\");\n zip.AddFile(ZipStorer.Compression.Deflate, path, fn, \"\");\n log?.Invoke(fn);\n }\n }\n }\n\n /// \n /// Unique class for compression/decompression file. Represents a Zip file.\n /// \n public class ZipStorer : IDisposable\n {\n /// \n /// Compression method enumeration\n /// \n public enum Compression : ushort\n {\n /// Uncompressed storage \n Store = 0,\n /// Deflate compression method\n Deflate = 8\n }\n\n /// \n /// Represents an entry in Zip file directory\n /// \n public struct ZipFileEntry\n {\n /// Compression method\n public Compression Method;\n /// Full path and filename as stored in Zip\n public string FilenameInZip;\n /// Original file size\n public uint FileSize;\n /// Compressed file size\n public uint CompressedSize;\n /// Offset of header information inside Zip storage\n public uint HeaderOffset;\n /// Offset of file inside Zip storage\n public uint FileOffset;\n /// Size of header information\n public uint HeaderSize;\n /// 32-bit checksum of entire file\n public uint Crc32;\n /// Last modification time of file\n public DateTime ModifyTime;\n /// User comment for file\n public string Comment;\n /// True if UTF8 encoding for filename and comments, false if default (CP 437)\n public bool EncodeUTF8;\n\n /// Overriden method\n /// Filename in Zip\n public override string ToString()\n {\n return this.FilenameInZip;\n }\n }\n\n #region Public fields\n /// True if UTF8 encoding for filename and comments, false if default (CP 437)\n public bool EncodeUTF8 = true;\n /// Force deflate algotithm even if it inflates the stored file. Off by default.\n public bool ForceDeflating = false;\n #endregion\n\n #region Private fields\n // List of files to store\n private List Files = new List();\n // Filename of storage file\n private string FileName;\n // Stream object of storage file\n private Stream ZipFileStream;\n // General comment\n private string Comment = \"\";\n // Central dir image\n private byte[] CentralDirImage = null;\n // Existing files in zip\n private ushort ExistingFiles = 0;\n // File access for Open method\n private FileAccess Access;\n // Static CRC32 Table\n private static UInt32[] CrcTable = null;\n // Default filename encoder\n private static Encoding DefaultEncoding = Encoding.GetEncoding(437);\n #endregion\n\n #region Public methods\n // Static constructor. Just invoked once in order to create the CRC32 lookup table.\n static ZipStorer()\n {\n // Generate CRC32 table\n CrcTable = new UInt32[256];\n for (int i = 0; i < CrcTable.Length; i++)\n {\n UInt32 c = (UInt32)i;\n for (int j = 0; j < 8; j++)\n {\n if ((c & 1) != 0)\n c = 3988292384 ^ (c >> 1);\n else\n c >>= 1;\n }\n CrcTable[i] = c;\n }\n }\n /// \n /// Method to create a new storage file\n /// \n /// Full path of Zip file to create\n /// General comment for Zip file\n /// A valid ZipStorer object\n public static ZipStorer Create(string _filename, string _comment)\n {\n Stream stream = new FileStream(_filename, FileMode.Create, FileAccess.ReadWrite);\n\n ZipStorer zip = Create(stream, _comment);\n zip.Comment = _comment;\n zip.FileName = _filename;\n\n return zip;\n }\n /// \n /// Method to create a new zip storage in a stream\n /// \n /// \n /// \n /// A valid ZipStorer object\n public static ZipStorer Create(Stream _stream, string _comment)\n {\n ZipStorer zip = new ZipStorer();\n zip.Comment = _comment;\n zip.ZipFileStream = _stream;\n zip.Access = FileAccess.Write;\n\n return zip;\n }\n /// \n /// Method to open an existing storage file\n /// \n /// Full path of Zip file to open\n /// File access mode as used in FileStream constructor\n /// A valid ZipStorer object\n public static ZipStorer Open(string _filename, FileAccess _access)\n {\n Stream stream = (Stream)new FileStream(_filename, FileMode.Open, _access == FileAccess.Read ? FileAccess.Read : FileAccess.ReadWrite);\n\n ZipStorer zip = Open(stream, _access);\n zip.FileName = _filename;\n\n return zip;\n }\n /// \n /// Method to open an existing storage from stream\n /// \n /// Already opened stream with zip contents\n /// File access mode for stream operations\n /// A valid ZipStorer object\n public static ZipStorer Open(Stream _stream, FileAccess _access)\n {\n if (!_stream.CanSeek && _access != FileAccess.Read)\n throw new InvalidOperationException(\"Stream cannot seek\");\n\n ZipStorer zip = new ZipStorer();\n //zip.FileName = _filename;\n zip.ZipFileStream = _stream;\n zip.Access = _access;\n\n if (zip.ReadFileInfo())\n return zip;\n\n throw new System.IO.InvalidDataException();\n }\n /// \n /// Add full contents of a file into the Zip storage\n /// \n /// Compression method\n /// Full path of file to add to Zip storage\n /// Filename and path as desired in Zip directory\n /// Comment for stored file \n public void AddFile(Compression _method, string _pathname, string _filenameInZip, string _comment)\n {\n if (Access == FileAccess.Read)\n throw new InvalidOperationException(\"Writing is not alowed\");\n\n FileStream stream = new FileStream(_pathname, FileMode.Open, FileAccess.Read);\n AddStream(_method, _filenameInZip, stream, File.GetLastWriteTime(_pathname), _comment);\n stream.Close();\n }\n\n public void AddFile(Compression _method, string _pathname, string _filenameInZip, string _comment, DateTime lastwrite)\n {\n if (Access == FileAccess.Read)\n throw new InvalidOperationException(\"Writing is not alowed\");\n\n FileStream stream = new FileStream(_pathname, FileMode.Open, FileAccess.Read);\n AddStream(_method, _filenameInZip, stream, lastwrite, _comment);\n stream.Close();\n }\n /// \n /// Add full contents of a stream into the Zip storage\n /// \n /// Compression method\n /// Filename and path as desired in Zip directory\n /// Stream object containing the data to store in Zip\n /// Modification time of the data to store\n /// Comment for stored file\n public void AddStream(Compression _method, string _filenameInZip, Stream _source, DateTime _modTime, string _comment)\n {\n if (Access == FileAccess.Read)\n throw new InvalidOperationException(\"Writing is not alowed\");\n\n long offset;\n if (this.Files.Count == 0)\n offset = 0;\n else\n {\n ZipFileEntry last = this.Files[this.Files.Count - 1];\n offset = last.HeaderOffset + last.HeaderSize;\n }\n\n // Prepare the fileinfo\n ZipFileEntry zfe = new ZipFileEntry();\n zfe.Method = _method;\n zfe.EncodeUTF8 = this.EncodeUTF8;\n zfe.FilenameInZip = NormalizedFilename(_filenameInZip);\n zfe.Comment = (_comment == null ? \"\" : _comment);\n\n // Even though we write the header now, it will have to be rewritten, since we don't know compressed size or crc.\n zfe.Crc32 = 0; // to be updated later\n zfe.HeaderOffset = (uint)this.ZipFileStream.Position; // offset within file of the start of this local record\n zfe.ModifyTime = _modTime;\n\n // Write local header\n WriteLocalHeader(ref zfe);\n zfe.FileOffset = (uint)this.ZipFileStream.Position;\n\n // Write file to zip (store)\n Store(ref zfe, _source);\n _source.Close();\n\n this.UpdateCrcAndSizes(ref zfe);\n\n Files.Add(zfe);\n }\n /// \n /// Updates central directory (if pertinent) and close the Zip storage\n /// \n /// This is a required step, unless automatic dispose is used\n public void Close()\n {\n if (this.Access != FileAccess.Read)\n {\n uint centralOffset = (uint)this.ZipFileStream.Position;\n uint centralSize = 0;\n\n if (this.CentralDirImage != null)\n this.ZipFileStream.Write(CentralDirImage, 0, CentralDirImage.Length);\n\n for (int i = 0; i < Files.Count; i++)\n {\n long pos = this.ZipFileStream.Position;\n this.WriteCentralDirRecord(Files[i]);\n centralSize += (uint)(this.ZipFileStream.Position - pos);\n }\n\n if (this.CentralDirImage != null)\n this.WriteEndRecord(centralSize + (uint)CentralDirImage.Length, centralOffset);\n else\n this.WriteEndRecord(centralSize, centralOffset);\n }\n\n if (this.ZipFileStream != null)\n {\n this.ZipFileStream.Flush();\n this.ZipFileStream.Dispose();\n this.ZipFileStream = null;\n }\n }\n /// \n /// Read all the file records in the central directory \n /// \n /// List of all entries in directory\n public List ReadCentralDir()\n {\n if (this.CentralDirImage == null)\n throw new InvalidOperationException(\"Central directory currently does not exist\");\n\n List result = new List();\n\n for (int pointer = 0; pointer < this.CentralDirImage.Length;)\n {\n uint signature = BitConverter.ToUInt32(CentralDirImage, pointer);\n if (signature != 0x02014b50)\n break;\n\n bool encodeUTF8 = (BitConverter.ToUInt16(CentralDirImage, pointer + 8) & 0x0800) != 0;\n ushort method = BitConverter.ToUInt16(CentralDirImage, pointer + 10);\n uint modifyTime = BitConverter.ToUInt32(CentralDirImage, pointer + 12);\n uint crc32 = BitConverter.ToUInt32(CentralDirImage, pointer + 16);\n uint comprSize = BitConverter.ToUInt32(CentralDirImage, pointer + 20);\n uint fileSize = BitConverter.ToUInt32(CentralDirImage, pointer + 24);\n ushort filenameSize = BitConverter.ToUInt16(CentralDirImage, pointer + 28);\n ushort extraSize = BitConverter.ToUInt16(CentralDirImage, pointer + 30);\n ushort commentSize = BitConverter.ToUInt16(CentralDirImage, pointer + 32);\n uint headerOffset = BitConverter.ToUInt32(CentralDirImage, pointer + 42);\n uint headerSize = (uint)(46 + filenameSize + extraSize + commentSize);\n\n Encoding encoder = encodeUTF8 ? Encoding.UTF8 : DefaultEncoding;\n\n ZipFileEntry zfe = new ZipFileEntry();\n zfe.Method = (Compression)method;\n zfe.FilenameInZip = encoder.GetString(CentralDirImage, pointer + 46, filenameSize);\n zfe.FileOffset = GetFileOffset(headerOffset);\n zfe.FileSize = fileSize;\n zfe.CompressedSize = comprSize;\n zfe.HeaderOffset = headerOffset;\n zfe.HeaderSize = headerSize;\n zfe.Crc32 = crc32;\n zfe.ModifyTime = DosTimeToDateTime(modifyTime);\n if (commentSize > 0)\n zfe.Comment = encoder.GetString(CentralDirImage, pointer + 46 + filenameSize + extraSize, commentSize);\n\n result.Add(zfe);\n pointer += (46 + filenameSize + extraSize + commentSize);\n }\n\n return result;\n }\n /// \n /// Copy the contents of a stored file into a physical file\n /// \n /// Entry information of file to extract\n /// Name of file to store uncompressed data\n /// True if success, false if not.\n /// Unique compression methods are Store and Deflate\n public bool ExtractFile(ZipFileEntry _zfe, string _filename)\n {\n // Make sure the parent directory exist\n string path = System.IO.Path.GetDirectoryName(_filename);\n\n if (!Directory.Exists(path))\n Directory.CreateDirectory(path);\n // Check it is directory. If so, do nothing\n if (Directory.Exists(_filename))\n return true;\n\n Stream output = new FileStream(_filename, FileMode.Create, FileAccess.Write);\n bool result = ExtractFile(_zfe, output);\n if (result)\n output.Close();\n\n File.SetCreationTime(_filename, _zfe.ModifyTime);\n File.SetLastWriteTime(_filename, _zfe.ModifyTime);\n\n return result;\n }\n public bool ExtractFile2(ZipFileEntry _zfe, string _filename)\n {\n // Make sure the parent directory exist\n //string path = System.IO.Path.GetDirectoryName(_filename);\n\n //if (!Directory.Exists(path))\n // Directory.CreateDirectory(path);\n //// Check it is directory. If so, do nothing\n //if (Directory.Exists(_filename))\n // return true;\n\n Stream output = new FileStream(_filename, FileMode.Create, FileAccess.Write);\n bool result = ExtractFile(_zfe, output);\n if (result)\n output.Close();\n\n File.SetCreationTime(_filename, _zfe.ModifyTime);\n File.SetLastWriteTime(_filename, _zfe.ModifyTime);\n\n return result;\n }\n /// \n /// Copy the contents of a stored file into an opened stream\n /// \n /// Entry information of file to extract\n /// Stream to store the uncompressed data\n /// True if success, false if not.\n /// Unique compression methods are Store and Deflate\n public bool ExtractFile(ZipFileEntry _zfe, Stream _stream)\n {\n if (!_stream.CanWrite)\n throw new InvalidOperationException(\"Stream cannot be written\");\n\n // check signature\n byte[] signature = new byte[4];\n this.ZipFileStream.Seek(_zfe.HeaderOffset, SeekOrigin.Begin);\n this.ZipFileStream.Read(signature, 0, 4);\n if (BitConverter.ToUInt32(signature, 0) != 0x04034b50)\n return false;\n\n // Select input stream for inflating or just reading\n Stream inStream;\n if (_zfe.Method == Compression.Store)\n inStream = this.ZipFileStream;\n else if (_zfe.Method == Compression.Deflate)\n inStream = new DeflateStream(this.ZipFileStream, CompressionMode.Decompress, true);\n else\n return false;\n\n // Buffered copy\n byte[] buffer = new byte[16384];\n this.ZipFileStream.Seek(_zfe.FileOffset, SeekOrigin.Begin);\n uint bytesPending = _zfe.FileSize;\n while (bytesPending > 0)\n {\n int bytesRead = inStream.Read(buffer, 0, (int)Math.Min(bytesPending, buffer.Length));\n _stream.Write(buffer, 0, bytesRead);\n bytesPending -= (uint)bytesRead;\n }\n _stream.Flush();\n\n if (_zfe.Method == Compression.Deflate)\n inStream.Dispose();\n return true;\n }\n /// \n /// Removes one of many files in storage. It creates a new Zip file.\n /// \n /// Reference to the current Zip object\n /// List of Entries to remove from storage\n /// True if success, false if not\n /// This method only works for storage of type FileStream\n public static bool RemoveEntries(ref ZipStorer _zip, List _zfes)\n {\n if (!(_zip.ZipFileStream is FileStream))\n throw new InvalidOperationException(\"RemoveEntries is allowed just over streams of type FileStream\");\n\n\n //Get full list of entries\n List fullList = _zip.ReadCentralDir();\n\n //In order to delete we need to create a copy of the zip file excluding the selected items\n string tempZipName = Path.GetTempFileName();\n string tempEntryName = Path.GetTempFileName();\n\n try\n {\n ZipStorer tempZip = ZipStorer.Create(tempZipName, string.Empty);\n\n foreach (ZipFileEntry zfe in fullList)\n {\n if (!_zfes.Contains(zfe))\n {\n if (_zip.ExtractFile(zfe, tempEntryName))\n {\n tempZip.AddFile(zfe.Method, tempEntryName, zfe.FilenameInZip, zfe.Comment);\n }\n }\n }\n _zip.Close();\n tempZip.Close();\n\n File.Delete(_zip.FileName);\n File.Move(tempZipName, _zip.FileName);\n\n _zip = ZipStorer.Open(_zip.FileName, _zip.Access);\n }\n catch\n {\n return false;\n }\n finally\n {\n if (File.Exists(tempZipName))\n File.Delete(tempZipName);\n if (File.Exists(tempEntryName))\n File.Delete(tempEntryName);\n }\n return true;\n }\n #endregion\n\n #region Private methods\n // Calculate the file offset by reading the corresponding local header\n private uint GetFileOffset(uint _headerOffset)\n {\n byte[] buffer = new byte[2];\n\n this.ZipFileStream.Seek(_headerOffset + 26, SeekOrigin.Begin);\n this.ZipFileStream.Read(buffer, 0, 2);\n ushort filenameSize = BitConverter.ToUInt16(buffer, 0);\n this.ZipFileStream.Read(buffer, 0, 2);\n ushort extraSize = BitConverter.ToUInt16(buffer, 0);\n\n return (uint)(30 + filenameSize + extraSize + _headerOffset);\n }\n /* Local file header:\n local file header signature 4 bytes (0x04034b50)\n version needed to extract 2 bytes\n general purpose bit flag 2 bytes\n compression method 2 bytes\n last mod file time 2 bytes\n last mod file date 2 bytes\n crc-32 4 bytes\n compressed size 4 bytes\n uncompressed size 4 bytes\n filename length 2 bytes\n extra field length 2 bytes\n\n filename (variable size)\n extra field (variable size)\n */\n private void WriteLocalHeader(ref ZipFileEntry _zfe)\n {\n long pos = this.ZipFileStream.Position;\n Encoding encoder = _zfe.EncodeUTF8 ? Encoding.UTF8 : DefaultEncoding;\n byte[] encodedFilename = encoder.GetBytes(_zfe.FilenameInZip);\n\n this.ZipFileStream.Write(new byte[] { 80, 75, 3, 4, 20, 0 }, 0, 6); // No extra header\n this.ZipFileStream.Write(BitConverter.GetBytes((ushort)(_zfe.EncodeUTF8 ? 0x0800 : 0)), 0, 2); // filename and comment encoding \n this.ZipFileStream.Write(BitConverter.GetBytes((ushort)_zfe.Method), 0, 2); // zipping method\n this.ZipFileStream.Write(BitConverter.GetBytes(DateTimeToDosTime(_zfe.ModifyTime)), 0, 4); // zipping date and time\n this.ZipFileStream.Write(new byte[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, 0, 12); // unused CRC, un/compressed size, updated later\n this.ZipFileStream.Write(BitConverter.GetBytes((ushort)encodedFilename.Length), 0, 2); // filename length\n this.ZipFileStream.Write(BitConverter.GetBytes((ushort)0), 0, 2); // extra length\n\n this.ZipFileStream.Write(encodedFilename, 0, encodedFilename.Length);\n _zfe.HeaderSize = (uint)(this.ZipFileStream.Position - pos);\n }\n /* Central directory's File header:\n central file header signature 4 bytes (0x02014b50)\n version made by 2 bytes\n version needed to extract 2 bytes\n general purpose bit flag 2 bytes\n compression method 2 bytes\n last mod file time 2 bytes\n last mod file date 2 bytes\n crc-32 4 bytes\n compressed size 4 bytes\n uncompressed size 4 bytes\n filename length 2 bytes\n extra field length 2 bytes\n file comment length 2 bytes\n disk number start 2 bytes\n internal file attributes 2 bytes\n external file attributes 4 bytes\n relative offset of local header 4 bytes\n\n filename (variable size)\n extra field (variable size)\n file comment (variable size)\n */\n private void WriteCentralDirRecord(ZipFileEntry _zfe)\n {\n Encoding encoder = _zfe.EncodeUTF8 ? Encoding.UTF8 : DefaultEncoding;\n byte[] encodedFilename = encoder.GetBytes(_zfe.FilenameInZip);\n byte[] encodedComment = encoder.GetBytes(_zfe.Comment);\n\n this.ZipFileStream.Write(new byte[] { 80, 75, 1, 2, 23, 0xB, 20, 0 }, 0, 8);\n this.ZipFileStream.Write(BitConverter.GetBytes((ushort)(_zfe.EncodeUTF8 ? 0x0800 : 0)), 0, 2); // filename and comment encoding \n this.ZipFileStream.Write(BitConverter.GetBytes((ushort)_zfe.Method), 0, 2); // zipping method\n this.ZipFileStream.Write(BitConverter.GetBytes(DateTimeToDosTime(_zfe.ModifyTime)), 0, 4); // zipping date and time\n this.ZipFileStream.Write(BitConverter.GetBytes(_zfe.Crc32), 0, 4); // file CRC\n this.ZipFileStream.Write(BitConverter.GetBytes(_zfe.CompressedSize), 0, 4); // compressed file size\n this.ZipFileStream.Write(BitConverter.GetBytes(_zfe.FileSize), 0, 4); // uncompressed file size\n this.ZipFileStream.Write(BitConverter.GetBytes((ushort)encodedFilename.Length), 0, 2); // Filename in zip\n this.ZipFileStream.Write(BitConverter.GetBytes((ushort)0), 0, 2); // extra length\n this.ZipFileStream.Write(BitConverter.GetBytes((ushort)encodedComment.Length), 0, 2);\n\n this.ZipFileStream.Write(BitConverter.GetBytes((ushort)0), 0, 2); // disk=0\n this.ZipFileStream.Write(BitConverter.GetBytes((ushort)0), 0, 2); // file type: binary\n this.ZipFileStream.Write(BitConverter.GetBytes((ushort)0), 0, 2); // Internal file attributes\n this.ZipFileStream.Write(BitConverter.GetBytes((ushort)0x8100), 0, 2); // External file attributes (normal/readable)\n this.ZipFileStream.Write(BitConverter.GetBytes(_zfe.HeaderOffset), 0, 4); // Offset of header\n\n this.ZipFileStream.Write(encodedFilename, 0, encodedFilename.Length);\n this.ZipFileStream.Write(encodedComment, 0, encodedComment.Length);\n }\n /* End of central dir record:\n end of central dir signature 4 bytes (0x06054b50)\n number of this disk 2 bytes\n number of the disk with the\n start of the central directory 2 bytes\n total number of entries in\n the central dir on this disk 2 bytes\n total number of entries in\n the central dir 2 bytes\n size of the central directory 4 bytes\n offset of start of central\n directory with respect to\n the starting disk number 4 bytes\n zipfile comment length 2 bytes\n zipfile comment (variable size)\n */\n private void WriteEndRecord(uint _size, uint _offset)\n {\n Encoding encoder = this.EncodeUTF8 ? Encoding.UTF8 : DefaultEncoding;\n byte[] encodedComment = encoder.GetBytes(this.Comment);\n\n this.ZipFileStream.Write(new byte[] { 80, 75, 5, 6, 0, 0, 0, 0 }, 0, 8);\n this.ZipFileStream.Write(BitConverter.GetBytes((ushort)Files.Count + ExistingFiles), 0, 2);\n this.ZipFileStream.Write(BitConverter.GetBytes((ushort)Files.Count + ExistingFiles), 0, 2);\n this.ZipFileStream.Write(BitConverter.GetBytes(_size), 0, 4);\n this.ZipFileStream.Write(BitConverter.GetBytes(_offset), 0, 4);\n this.ZipFileStream.Write(BitConverter.GetBytes((ushort)encodedComment.Length), 0, 2);\n this.ZipFileStream.Write(encodedComment, 0, encodedComment.Length);\n }\n // Copies all source file into storage file\n private void Store(ref ZipFileEntry _zfe, Stream _source)\n {\n byte[] buffer = new byte[16384];\n int bytesRead;\n uint totalRead = 0;\n Stream outStream;\n\n long posStart = this.ZipFileStream.Position;\n long sourceStart = _source.Position;\n\n if (_zfe.Method == Compression.Store)\n outStream = this.ZipFileStream;\n else\n outStream = new DeflateStream(this.ZipFileStream, CompressionMode.Compress, true);\n\n _zfe.Crc32 = 0 ^ 0xffffffff;\n\n do\n {\n bytesRead = _source.Read(buffer, 0, buffer.Length);\n totalRead += (uint)bytesRead;\n if (bytesRead > 0)\n {\n outStream.Write(buffer, 0, bytesRead);\n\n for (uint i = 0; i < bytesRead; i++)\n {\n _zfe.Crc32 = ZipStorer.CrcTable[(_zfe.Crc32 ^ buffer[i]) & 0xFF] ^ (_zfe.Crc32 >> 8);\n }\n }\n } while (bytesRead == buffer.Length);\n outStream.Flush();\n\n if (_zfe.Method == Compression.Deflate)\n outStream.Dispose();\n\n _zfe.Crc32 ^= 0xffffffff;\n _zfe.FileSize = totalRead;\n _zfe.CompressedSize = (uint)(this.ZipFileStream.Position - posStart);\n\n // Verify for real compression\n if (_zfe.Method == Compression.Deflate && !this.ForceDeflating && _source.CanSeek && _zfe.CompressedSize > _zfe.FileSize)\n {\n // Start operation again with Store algorithm\n _zfe.Method = Compression.Store;\n this.ZipFileStream.Position = posStart;\n this.ZipFileStream.SetLength(posStart);\n _source.Position = sourceStart;\n this.Store(ref _zfe, _source);\n }\n }\n /* DOS Date and time:\n MS-DOS date. The date is a packed value with the following format. Bits Description \n 0-4 Day of the month (1�31) \n 5-8 Month (1 = January, 2 = February, and so on) \n 9-15 Year offset from 1980 (add 1980 to get actual year) \n MS-DOS time. The time is a packed value with the following format. Bits Description \n 0-4 Second divided by 2 \n 5-10 Minute (0�59) \n 11-15 Hour (0�23 on a 24-hour clock) \n */\n private uint DateTimeToDosTime(DateTime _dt)\n {\n return (uint)(\n (_dt.Second / 2) | (_dt.Minute << 5) | (_dt.Hour << 11) |\n (_dt.Day << 16) | (_dt.Month << 21) | ((_dt.Year - 1980) << 25));\n }\n private DateTime DosTimeToDateTime(uint _dt)\n {\n return new DateTime(\n (int)(_dt >> 25) + 1980,\n (int)(_dt >> 21) & 15,\n (int)(_dt >> 16) & 31,\n (int)(_dt >> 11) & 31,\n (int)(_dt >> 5) & 63,\n (int)(_dt & 31) * 2);\n }\n\n /* CRC32 algorithm\n The 'magic number' for the CRC is 0xdebb20e3. \n The proper CRC pre and post conditioning\n is used, meaning that the CRC register is\n pre-conditioned with all ones (a starting value\n of 0xffffffff) and the value is post-conditioned by\n taking the one's complement of the CRC residual.\n If bit 3 of the general purpose flag is set, this\n field is set to zero in the local header and the correct\n value is put in the data descriptor and in the central\n directory.\n */\n private void UpdateCrcAndSizes(ref ZipFileEntry _zfe)\n {\n long lastPos = this.ZipFileStream.Position; // remember position\n\n this.ZipFileStream.Position = _zfe.HeaderOffset + 8;\n this.ZipFileStream.Write(BitConverter.GetBytes((ushort)_zfe.Method), 0, 2); // zipping method\n\n this.ZipFileStream.Position = _zfe.HeaderOffset + 14;\n this.ZipFileStream.Write(BitConverter.GetBytes(_zfe.Crc32), 0, 4); // Update CRC\n this.ZipFileStream.Write(BitConverter.GetBytes(_zfe.CompressedSize), 0, 4); // Compressed size\n this.ZipFileStream.Write(BitConverter.GetBytes(_zfe.FileSize), 0, 4); // Uncompressed size\n\n this.ZipFileStream.Position = lastPos; // restore position\n }\n // Replaces backslashes with slashes to store in zip header\n private string NormalizedFilename(string _filename)\n {\n string filename = _filename.Replace('\\\\', '/');\n\n int pos = filename.IndexOf(':');\n if (pos >= 0)\n filename = filename.Remove(0, pos + 1);\n\n return filename.Trim('/');\n }\n // Reads the end-of-central-directory record\n private bool ReadFileInfo()\n {\n if (this.ZipFileStream.Length < 22)\n return false;\n\n try\n {\n this.ZipFileStream.Seek(-17, SeekOrigin.End);\n BinaryReader br = new BinaryReader(this.ZipFileStream);\n do\n {\n this.ZipFileStream.Seek(-5, SeekOrigin.Current);\n UInt32 sig = br.ReadUInt32();\n if (sig == 0x06054b50)\n {\n this.ZipFileStream.Seek(6, SeekOrigin.Current);\n\n UInt16 entries = br.ReadUInt16();\n Int32 centralSize = br.ReadInt32();\n UInt32 centralDirOffset = br.ReadUInt32();\n UInt16 commentSize = br.ReadUInt16();\n\n // check if comment field is the very last data in file\n if (this.ZipFileStream.Position + commentSize != this.ZipFileStream.Length)\n return false;\n\n // Copy entire central directory to a memory buffer\n this.ExistingFiles = entries;\n this.CentralDirImage = new byte[centralSize];\n this.ZipFileStream.Seek(centralDirOffset, SeekOrigin.Begin);\n this.ZipFileStream.Read(this.CentralDirImage, 0, centralSize);\n\n // Leave the pointer at the begining of central dir, to append new files\n this.ZipFileStream.Seek(centralDirOffset, SeekOrigin.Begin);\n return true;\n }\n } while (this.ZipFileStream.Position > 0);\n }\n catch { }\n\n return false;\n }\n #endregion\n\n #region IDisposable Members\n /// \n /// Closes the Zip file stream\n /// \n public void Dispose()\n {\n this.Close();\n }\n #endregion\n }\n}\n" }, { "path": "RaptorDB.Common/fastBinaryJSON/BJSON.cs", "content": "using System;\nusing System.Collections;\nusing System.Collections.Generic;\n#if !SILVERLIGHT\nusing System.Data;\n#endif\nusing System.IO;\nusing System.Collections.Specialized;\nusing fastJSON;\n\nnamespace fastBinaryJSON\n{\n public sealed class TOKENS\n {\n public const byte DOC_START = 1;\n public const byte DOC_END = 2;\n public const byte ARRAY_START = 3;\n public const byte ARRAY_END = 4;\n public const byte COLON = 5;\n public const byte COMMA = 6;\n public const byte NAME = 7;\n public const byte STRING = 8;\n public const byte BYTE = 9;\n public const byte INT = 10;\n public const byte UINT = 11;\n public const byte LONG = 12;\n public const byte ULONG = 13;\n public const byte SHORT = 14;\n public const byte USHORT = 15;\n public const byte DATETIME = 16;\n public const byte GUID = 17;\n public const byte DOUBLE = 18;\n public const byte FLOAT = 19;\n public const byte DECIMAL = 20;\n public const byte CHAR = 21;\n public const byte BYTEARRAY = 22;\n public const byte NULL = 23;\n public const byte TRUE = 24;\n public const byte FALSE = 25;\n public const byte UNICODE_STRING = 26;\n public const byte DATETIMEOFFSET = 27;\n public const byte ARRAY_TYPED = 28;\n public const byte TYPES_POINTER = 29;\n public const byte TIMESPAN = 30;\n public const byte ARRAY_TYPED_LONG = 31;\n public const byte NAME_UNI = 32;\n }\n\n public class typedarray\n {\n public string typename;\n public int count;\n public List data = new List();\n }\n\n\n\n public sealed class BJSONParameters\n {\n /// \n /// Optimize the schema for Datasets (default = True)\n /// \n public bool UseOptimizedDatasetSchema = true;\n /// \n /// Serialize readonly properties (default = False)\n /// \n public bool ShowReadOnlyProperties = false;\n /// \n /// Use global types $types for more compact size when using a lot of classes (default = True)\n /// \n public bool UsingGlobalTypes = true;\n /// \n /// Use Unicode strings = T (faster), Use UTF8 strings = F (smaller) (default = True)\n /// \n public bool UseUnicodeStrings = true;\n /// \n /// Serialize Null values to the output (default = False)\n /// \n public bool SerializeNulls = false;\n /// \n /// Enable fastBinaryJSON extensions $types, $type, $map (default = True)\n /// \n public bool UseExtensions = true;\n /// \n /// Anonymous types have read only properties \n /// \n public bool EnableAnonymousTypes = false;\n /// \n /// Use the UTC date format (default = False)\n /// \n public bool UseUTCDateTime = false;\n /// \n /// Ignore attributes to check for (default : XmlIgnoreAttribute, NonSerialized)\n /// \n public List IgnoreAttributes = new List { typeof(System.Xml.Serialization.XmlIgnoreAttribute), typeof(NonSerializedAttribute) };\n /// \n /// If you have parametric and no default constructor for you classes (default = False)\n /// \n /// IMPORTANT NOTE : If True then all initial values within the class will be ignored and will be not set\n /// \n public bool ParametricConstructorOverride = false;\n /// \n /// Maximum depth the serializer will go to to avoid loops (default = 20 levels)\n /// \n public short SerializerMaxDepth = 20;\n /// \n /// Use typed arrays t[] into object = t[] not object[] (default = true)\n /// \n public bool UseTypedArrays = true;\n /// \n /// Backward compatible Typed array type name as UTF8 (default = false -> fast v1.5 unicode)\n /// \n public bool v1_4TypedArray = false;\n\n public void FixValues()\n {\n if (UseExtensions == false) // disable conflicting params\n UsingGlobalTypes = false;\n\n if (EnableAnonymousTypes)\n ShowReadOnlyProperties = true;\n }\n\n internal BJSONParameters MakeCopy()\n {\n return new BJSONParameters\n { \n UseOptimizedDatasetSchema = UseOptimizedDatasetSchema,\n ShowReadOnlyProperties = ShowReadOnlyProperties,\n EnableAnonymousTypes = EnableAnonymousTypes,\n UsingGlobalTypes = UsingGlobalTypes,\n IgnoreAttributes = new List(IgnoreAttributes),\n UseUnicodeStrings = UseUnicodeStrings,\n SerializeNulls = SerializeNulls,\n ParametricConstructorOverride = ParametricConstructorOverride,\n SerializerMaxDepth = SerializerMaxDepth,\n UseTypedArrays = UseTypedArrays,\n UseExtensions = UseExtensions,\n UseUTCDateTime = UseUTCDateTime,\n v1_4TypedArray = v1_4TypedArray//,\n //OptimizeSize = OptimizeSize\n\n };\n }\n }\n\n public static class BJSON\n {\n /// \n /// Globally set-able parameters for controlling the serializer\n /// \n public static BJSONParameters Parameters = new BJSONParameters();\n /// \n /// Parse a json and generate a Dictionary<string,object> or List<object> structure\n /// \n /// \n /// \n public static object Parse(byte[] json)\n {\n return new BJsonParser(json, Parameters.UseUTCDateTime, Parameters.v1_4TypedArray).Decode();\n }\n#if NET4\n /// \n /// Create a .net4 dynamic object from the binary json byte array\n /// \n /// \n /// \n public static dynamic ToDynamic(byte[] json)\n {\n return new DynamicJson(json);\n }\n#endif\n /// \n /// Register custom type handlers for your own types not natively handled by fastBinaryJSON\n /// \n /// \n /// \n /// \n public static void RegisterCustomType(Type type, Reflection.Serialize serializer, Reflection.Deserialize deserializer)\n {\n Reflection.Instance.RegisterCustomType(type, serializer, deserializer);\n }\n /// \n /// Create a binary json representation for an object\n /// \n /// \n /// \n public static byte[] ToBJSON(object obj)\n {\n return ToBJSON(obj, Parameters);\n }\n /// \n /// Create a binary json representation for an object with parameter override on this call\n /// \n /// \n /// \n /// \n public static byte[] ToBJSON(object obj, BJSONParameters param)\n {\n param.FixValues();\n param = param.MakeCopy();\n Type t = null;\n if (obj == null)\n return new byte[] { TOKENS.NULL };\n if (obj.GetType().IsGenericType)\n t = Reflection.Instance.GetGenericTypeDefinition(obj.GetType());// obj.GetType().GetGenericTypeDefinition();\n if (t == typeof(Dictionary<,>) || t == typeof(List<>))\n param.UsingGlobalTypes = false;\n // FEATURE : enable extensions when you can deserialize anon types\n if (param.EnableAnonymousTypes) { param.UseExtensions = false; param.UsingGlobalTypes = false; }\n\n return new BJSONSerializer(param).ConvertToBJSON(obj);\n }\n /// \n /// Fill a given object with the binary json represenation\n /// \n /// \n /// \n /// \n public static object FillObject(object input, byte[] json)\n {\n return new deserializer(Parameters).FillObject(input, json);\n }\n /// \n /// Create a generic object from the json\n /// \n /// \n /// \n /// \n public static T ToObject(byte[] json)\n {\n return new deserializer(Parameters).ToObject(json);\n }\n /// \n /// Create a generic object from the json with parameter override on this call\n /// \n /// \n /// \n /// \n /// \n public static T ToObject(byte[] json, BJSONParameters param)\n {\n return new deserializer(param).ToObject(json);\n }\n /// \n /// Create an object from the json \n /// \n /// \n /// \n public static object ToObject(byte[] json)\n {\n return new deserializer(Parameters).ToObject(json, null);\n }\n /// \n /// Create an object from the json with parameter override on this call\n /// \n /// \n /// \n /// \n public static object ToObject(byte[] json, BJSONParameters param)\n {\n param.FixValues();\n param = param.MakeCopy();\n return new deserializer(param).ToObject(json, null);\n }\n /// \n /// Create a typed object from the json\n /// \n /// \n /// \n /// \n public static object ToObject(byte[] json, Type type)\n {\n return new deserializer(Parameters).ToObject(json, type);\n }\n /// \n /// Clear the internal reflection cache so you can start from new (you will loose performance)\n /// \n public static void ClearReflectionCache()\n {\n Reflection.Instance.ClearReflectionCache();\n }\n /// \n /// Deep copy an object i.e. clone to a new object\n /// \n /// \n /// \n public static object DeepCopy(object obj)\n {\n return new deserializer(Parameters).ToObject(ToBJSON(obj));\n }\n }\n\n internal class deserializer\n {\n public deserializer(BJSONParameters param)\n {\n _params = param;\n _params = param.MakeCopy();\n }\n\n private BJSONParameters _params;\n private Dictionary _circobj = new Dictionary();\n private Dictionary _cirrev = new Dictionary();\n\n public T ToObject(byte[] json)\n {\n return (T)ToObject(json, typeof(T));\n }\n\n public object ToObject(byte[] json)\n {\n return ToObject(json, null);\n }\n\n public object ToObject(byte[] json, Type type)\n {\n //_params.FixValues();\n Type t = null;\n if (type != null && type.IsGenericType)\n t = Reflection.Instance.GetGenericTypeDefinition(type);// type.GetGenericTypeDefinition();\n _globalTypes = _params.UsingGlobalTypes;\n if (t == typeof(Dictionary<,>) || t == typeof(List<>))\n _globalTypes = false;\n\n var o = new BJsonParser(json, _params.UseUTCDateTime, _params.v1_4TypedArray).Decode();\n if (type?.IsEnum == true) return CreateEnum(type, o);\n#if !SILVERLIGHT\n if (type != null && type == typeof(DataSet))\n return CreateDataset(o as Dictionary, null);\n\n if (type != null && type == typeof(DataTable))\n return CreateDataTable(o as Dictionary, null);\n#endif\n if (o is typedarray)\n {\n return ParseTypedArray(new Dictionary(), o);\n }\n if (o is IDictionary)\n {\n if (type != null && t == typeof(Dictionary<,>)) // deserialize a dictionary\n return RootDictionary(o, type);\n else // deserialize an object\n return ParseDictionary(o as Dictionary, null, type, null);\n }\n\n if (o is List)\n {\n if (type != null && t == typeof(Dictionary<,>)) // kv format\n return RootDictionary(o, type);\n\n if (type != null && t == typeof(List<>)) // deserialize to generic list\n return RootList(o, type);\n\n if (type == typeof(Hashtable))\n return RootHashTable((List)o);\n else if (type == null)\n {\n List l = (List)o;\n if (l.Count > 0 && l[0].GetType() == typeof(Dictionary))\n {\n Dictionary globals = new Dictionary();\n List op = new List();\n // try to get $types \n foreach (var i in l)\n op.Add(ParseDictionary((Dictionary)i, globals, null, null));\n return op;\n }\n return l.ToArray();\n }\n }\n else if (type != null && o.GetType() != type)\n return ChangeType(o, type);\n\n return o;\n }\n\n private object ChangeType(object o, Type type)\n {\n if (Reflection.Instance.IsTypeRegistered(type))\n return Reflection.Instance.CreateCustom((string)o, type);\n else\n return o;\n }\n\n public object FillObject(object input, byte[] json)\n {\n _params.FixValues();\n Dictionary ht = new BJsonParser(json, _params.UseUTCDateTime, _params.v1_4TypedArray).Decode() as Dictionary;\n if (ht == null) return null;\n return ParseDictionary(ht, null, input.GetType(), input);\n }\n\n private object RootHashTable(List o)\n {\n Hashtable h = new Hashtable();\n\n foreach (Dictionary values in o)\n {\n object key = values[\"k\"];\n object val = values[\"v\"];\n if (key is Dictionary)\n key = ParseDictionary((Dictionary)key, null, typeof(object), null);\n\n if (val is Dictionary)\n val = ParseDictionary((Dictionary)val, null, typeof(object), null);\n\n h.Add(key, val);\n }\n\n return h;\n }\n\n private object RootList(object parse, Type type)\n {\n Type[] gtypes = Reflection.Instance.GetGenericArguments(type);// type.GetGenericArguments();\n IList o = (IList)Reflection.Instance.FastCreateList(type, ((IList)parse).Count);\n Dictionary globals = new Dictionary();\n\n foreach (var k in (IList)parse)\n {\n _globalTypes = false;\n object v = k;\n if (k is Dictionary)\n v = ParseDictionary(k as Dictionary, globals, gtypes[0], null);\n else\n v = k;\n\n o.Add(v);\n }\n return o;\n }\n\n private object RootDictionary(object parse, Type type)\n {\n Type[] gtypes = Reflection.Instance.GetGenericArguments(type);\n Type t1 = null;\n Type t2 = null;\n if (gtypes != null)\n {\n t1 = gtypes[0];\n t2 = gtypes[1];\n }\n var arraytype = t2.GetElementType();\n\n if (parse is Dictionary)\n {\n IDictionary o = (IDictionary)Reflection.Instance.FastCreateInstance(type);\n\n foreach (var kv in (Dictionary)parse)\n {\n _globalTypes = false;\n object v;\n object k = kv.Key;\n if (t2.Name.StartsWith(\"Dictionary\")) // deserialize a dictionary\n v = RootDictionary(kv.Value, t2);\n\n else if (kv.Value is Dictionary)\n v = ParseDictionary(kv.Value as Dictionary, null, t2, null);\n\n else if (t2 == typeof(byte[]))\n v = kv.Value;\n\n else if (gtypes != null && t2.IsArray)\n v = CreateArray((List)kv.Value, t2, arraytype, null);\n\n else if (kv.Value is IList)\n v = CreateGenericList((List)kv.Value, t2, t1, null);\n\n else\n v = kv.Value;\n\n o.Add(k, v);\n }\n\n return o;\n }\n if (parse is List)\n return CreateDictionary(parse as List, type, gtypes, null);\n\n return null;\n }\n\n private bool _globalTypes = false;\n private object ParseDictionary(Dictionary d, Dictionary globaltypes, Type type, object input)\n {\n object tn = \"\";\n if (type == typeof(NameValueCollection))\n return CreateNV(d);\n if (type == typeof(StringDictionary))\n return CreateSD(d);\n\n if (d.TryGetValue(\"$i\", out tn))\n {\n object v = null;\n _cirrev.TryGetValue((int)tn, out v);\n return v;\n }\n\n if (d.TryGetValue(\"$types\", out tn))\n {\n _globalTypes = true;\n if (globaltypes == null)\n globaltypes = new Dictionary();\n foreach (var kv in (Dictionary)tn)\n {\n globaltypes.Add((string)kv.Key, kv.Value);\n }\n }\n\n if (globaltypes != null)\n _globalTypes = true;\n\n bool found = d.TryGetValue(\"$type\", out tn);\n#if !SILVERLIGHT\n if (found == false && type == typeof(System.Object))\n {\n return d; // CreateDataset(d, globaltypes);\n }\n#endif\n if (found)\n {\n if (_globalTypes && globaltypes != null)\n {\n object tname = \"\";\n if (globaltypes != null && globaltypes.TryGetValue((string)tn, out tname))\n tn = tname;\n }\n type = Reflection.Instance.GetTypeFromCache((string)tn, true);\n }\n\n if (type == null)\n throw new Exception(\"Cannot determine type\");\n\n string typename = type.FullName;\n object o = input;\n if (o == null)\n {\n if (_params.ParametricConstructorOverride)\n o = System.Runtime.Serialization.FormatterServices.GetUninitializedObject(type);\n else\n o = Reflection.Instance.FastCreateInstance(type);\n }\n\n int circount = 0;\n if (_circobj.TryGetValue(o, out circount) == false)\n {\n circount = _circobj.Count + 1;\n _circobj.Add(o, circount);\n _cirrev.Add(circount, o);\n }\n\n Dictionary props = Reflection.Instance.Getproperties(type, typename, _params.ShowReadOnlyProperties);//, Reflection.Instance.IsTypeRegistered(type));\n foreach (var kv in d)\n {\n var n = kv.Key;\n var v = kv.Value;\n string name = n.ToLowerInvariant();\n myPropInfo pi;\n if (props.TryGetValue(name, out pi) == false)\n continue;\n if (pi.CanWrite)\n {\n //object v = d[n];\n\n if (v != null)\n {\n object oset = v;\n if (v is typedarray)\n {\n oset = ParseTypedArray(globaltypes, v);\n }\n else\n {\n switch (pi.Type)\n {\n#if !SILVERLIGHT\n case myPropInfoType.DataSet:\n oset = CreateDataset((Dictionary)v, globaltypes);\n break;\n case myPropInfoType.DataTable:\n oset = CreateDataTable((Dictionary)v, globaltypes);\n break;\n#endif\n case myPropInfoType.Custom:\n oset = Reflection.Instance.CreateCustom((string)v, pi.pt);\n break;\n case myPropInfoType.Enum:\n oset = CreateEnum(pi.pt, v);\n break;\n case myPropInfoType.StringKeyDictionary:\n oset = CreateStringKeyDictionary((Dictionary)v, pi.pt, pi.GenericTypes, globaltypes);\n break;\n case myPropInfoType.Hashtable:\n case myPropInfoType.Dictionary:\n oset = CreateDictionary((List)v, pi.pt, pi.GenericTypes, globaltypes);\n break;\n case myPropInfoType.NameValue: oset = CreateNV((Dictionary)v); break;\n case myPropInfoType.StringDictionary: oset = CreateSD((Dictionary)v); break;\n case myPropInfoType.Array:\n oset = CreateArray((List)v, pi.pt, pi.bt, globaltypes);\n break;\n default:\n {\n if (pi.IsGenericType && pi.IsValueType == false)\n oset = CreateGenericList((List)v, pi.pt, pi.bt, globaltypes);\n else if ((pi.IsClass || pi.IsStruct || pi.IsInterface) && v is Dictionary)\n {\n var oo = (Dictionary)v;\n if (oo.ContainsKey(\"$schema\"))\n oset = CreateDataset(oo, globaltypes);\n else\n oset = ParseDictionary(oo, globaltypes, pi.pt, input);\n }\n else if (v is List)\n oset = CreateArray((List)v, pi.pt, typeof(object), globaltypes);\n break;\n }\n }\n }\n o = pi.setter(o, oset);\n }\n }\n }\n return o;\n }\n\n private object ParseTypedArray(Dictionary globaltypes, object v)\n {\n object oset;\n var ta = (typedarray)v;\n var t = Reflection.Instance.GetTypeFromCache(ta.typename, true);\n IList a = Array.CreateInstance(t, ta.count);\n int i = 0;\n foreach (var dd in ta.data)\n {\n object oo = null;\n if (dd == null)\n oo = null;\n else if (dd is typedarray)\n oo = ParseTypedArray(globaltypes, dd);\n else if (dd is Dictionary)\n oo = ParseDictionary((Dictionary)dd, globaltypes, t, null);\n else if (dd is List)\n oo = CreateArray((List)dd, t, t.GetElementType(), globaltypes);\n else\n oo = dd;\n a[i++] = oo;\n }\n oset = a;\n return oset;\n }\n\n private StringDictionary CreateSD(Dictionary d)\n {\n StringDictionary nv = new StringDictionary();\n\n foreach (var o in d)\n nv.Add(o.Key, (string)o.Value);\n\n return nv;\n }\n\n private NameValueCollection CreateNV(Dictionary d)\n {\n NameValueCollection nv = new NameValueCollection();\n\n foreach (var o in d)\n nv.Add(o.Key, (string)o.Value);\n\n return nv;\n }\n\n private object CreateEnum(Type pt, object v)\n {\n // FEATURE : optimize create enum\n#if !SILVERLIGHT\n return Enum.Parse(pt, v.ToString());\n#else\n return Enum.Parse(pt, v, true);\n#endif\n }\n\n private object CreateArray(List data, Type pt, Type bt, Dictionary globalTypes)\n {\n if (bt == null)\n bt = typeof(object);\n\n Array col = Array.CreateInstance(bt, data.Count);\n var arraytype = bt.GetElementType();\n // create an array of objects\n for (int i = 0; i < data.Count; i++)// each (object ob in data)\n {\n object ob = data[i];\n if (ob == null)\n {\n continue;\n }\n if (ob is IDictionary)\n col.SetValue(ParseDictionary((Dictionary)ob, globalTypes, bt, null), i);\n else if (ob is ICollection)\n col.SetValue(CreateArray((List)ob, bt, arraytype, globalTypes), i);\n else\n col.SetValue(ob, i);\n }\n\n return col;\n }\n\n private object CreateGenericList(List data, Type pt, Type bt, Dictionary globalTypes)\n {\n if (pt != typeof(object))\n {\n IList col = (IList)Reflection.Instance.FastCreateList(pt, data.Count);\n // create an array of objects\n foreach (object ob in data)\n {\n if (ob is IDictionary)\n col.Add(ParseDictionary((Dictionary)ob, globalTypes, bt, null));\n\n else if (ob is List)\n {\n if (bt.IsGenericType)\n col.Add((List)ob);\n else\n col.Add(((List)ob).ToArray());\n }\n else if(ob is typedarray)\n col.Add(((typedarray)ob).data.ToArray());\n else\n col.Add(ob);\n }\n return col;\n }\n return data;\n }\n\n private object CreateStringKeyDictionary(Dictionary reader, Type pt, Type[] types, Dictionary globalTypes)\n {\n var col = (IDictionary)Reflection.Instance.FastCreateInstance(pt);\n Type arraytype = null;\n Type t2 = null;\n if (types != null)\n t2 = types[1];\n\n Type generictype = null;\n var ga = Reflection.Instance.GetGenericArguments(t2);\n if (ga.Length > 0)\n generictype = ga[0];\n arraytype = t2.GetElementType();\n\n foreach (KeyValuePair values in reader)\n {\n var key = values.Key;\n object val = null;\n\n if (values.Value is Dictionary)\n val = ParseDictionary((Dictionary)values.Value, globalTypes, t2, null);\n\n else if (types != null && t2.IsArray)\n {\n if (values.Value is Array)\n val = values.Value;\n else\n val = CreateArray((List)values.Value, t2, arraytype, globalTypes);\n }\n else if (values.Value is IList)\n val = CreateGenericList((List)values.Value, t2, generictype, globalTypes);\n\n else\n val = values.Value;\n\n col.Add(key, val);\n }\n\n return col;\n }\n\n private object CreateDictionary(List reader, Type pt, Type[] types, Dictionary globalTypes)\n {\n IDictionary col = (IDictionary)Reflection.Instance.FastCreateInstance(pt);\n Type t1 = null;\n Type t2 = null;\n if (types != null)\n {\n t1 = types[0];\n t2 = types[1];\n }\n\n foreach (Dictionary values in reader)\n {\n object key = values[\"k\"];\n object val = values[\"v\"];\n\n if (key is Dictionary)\n key = ParseDictionary((Dictionary)key, globalTypes, t1, null);\n\n if (typeof(IDictionary).IsAssignableFrom(t2))\n val = RootDictionary(val, t2);\n\n else if (val is Dictionary)\n val = ParseDictionary((Dictionary)val, globalTypes, t2, null);\n\n col.Add(key, val);\n }\n\n return col;\n }\n\n#if !SILVERLIGHT\n private DataSet CreateDataset(Dictionary reader, Dictionary globalTypes)\n {\n DataSet ds = new DataSet();\n ds.EnforceConstraints = false;\n ds.BeginInit();\n\n // read dataset schema here\n var schema = reader[\"$schema\"];\n\n if (schema is string)\n {\n TextReader tr = new StringReader((string)schema);\n ds.ReadXmlSchema(tr);\n }\n else\n {\n DatasetSchema ms = (DatasetSchema)ParseDictionary((Dictionary)schema, globalTypes, typeof(DatasetSchema), null);\n ds.DataSetName = ms.Name;\n for (int i = 0; i < ms.Info.Count; i += 3)\n {\n if (ds.Tables.Contains(ms.Info[i]) == false)\n ds.Tables.Add(ms.Info[i]);\n ds.Tables[ms.Info[i]].Columns.Add(ms.Info[i + 1], Type.GetType(ms.Info[i + 2]));\n }\n }\n\n foreach (KeyValuePair pair in reader)\n {\n if (pair.Key == \"$type\" || pair.Key == \"$schema\") continue;\n\n List rows = (List)pair.Value;\n if (rows == null) continue;\n\n DataTable dt = ds.Tables[pair.Key];\n ReadDataTable(rows, dt);\n }\n\n ds.EndInit();\n\n return ds;\n }\n\n private void ReadDataTable(List rows, DataTable dt)\n {\n dt.BeginInit();\n dt.BeginLoadData();\n\n foreach (List row in rows)\n {\n object[] v = new object[row.Count];\n row.CopyTo(v, 0);\n dt.Rows.Add(v);\n }\n\n dt.EndLoadData();\n dt.EndInit();\n }\n\n DataTable CreateDataTable(Dictionary reader, Dictionary globalTypes)\n {\n var dt = new DataTable();\n\n // read dataset schema here\n var schema = reader[\"$schema\"];\n\n if (schema is string)\n {\n TextReader tr = new StringReader((string)schema);\n dt.ReadXmlSchema(tr);\n }\n else\n {\n var ms = (DatasetSchema)this.ParseDictionary((Dictionary)schema, globalTypes, typeof(DatasetSchema), null);\n dt.TableName = ms.Info[0];\n for (int i = 0; i < ms.Info.Count; i += 3)\n {\n dt.Columns.Add(ms.Info[i + 1], Type.GetType(ms.Info[i + 2]));\n }\n }\n\n foreach (var pair in reader)\n {\n if (pair.Key == \"$type\" || pair.Key == \"$schema\")\n continue;\n\n var rows = (List)pair.Value;\n if (rows == null)\n continue;\n\n if (!dt.TableName.Equals(pair.Key, StringComparison.InvariantCultureIgnoreCase))\n continue;\n\n ReadDataTable(rows, dt);\n }\n\n return dt;\n }\n#endif\n }\n\n}\n" }, { "path": "RaptorDB.Common/fastBinaryJSON/BJsonParser.cs", "content": "using fastJSON;\nusing System;\nusing System.Collections.Generic;\n\nnamespace fastBinaryJSON\n{\n internal sealed class BJsonParser\n {\n readonly byte[] _json;\n int _index;\n bool _useUTC = true;\n bool _v1_4TA = false;\n\n internal BJsonParser(byte[] json, bool useUTC, bool v1_4TA)\n {\n this._json = json;\n _v1_4TA = v1_4TA;\n _useUTC = useUTC;\n }\n\n public object Decode()\n {\n bool b = false;\n return ParseValue(out b);\n }\n\n private Dictionary ParseObject()\n {\n Dictionary dic = new Dictionary(10);\n bool breakparse = false;\n while (!breakparse)\n {\n byte t = GetToken();\n if (t == TOKENS.COMMA)\n continue;\n if (t == TOKENS.DOC_END)\n break;\n if (t == TOKENS.TYPES_POINTER)\n {\n // save curr index position\n int savedindex = _index;\n // set index = pointer \n _index = ParseInt();\n t = GetToken();\n // read $types\n breakparse = readkeyvalue(dic, ref t);\n // set index = saved + 4\n _index = savedindex + 4;\n }\n else\n breakparse = readkeyvalue(dic, ref t);\n }\n return dic;\n }\n\n private bool readkeyvalue(Dictionary dic, ref byte t)\n {\n bool breakparse;\n string key = \"\";\n //if (t != TOKENS.NAME)\n if (t == TOKENS.NAME)\n key = ParseName();\n else if (t == TOKENS.NAME_UNI)\n key = ParseName2();\n else\n throw new Exception(\"excpecting a name field\");\n\n t = GetToken();\n if (t != TOKENS.COLON)\n throw new Exception(\"expecting a colon\");\n object val = ParseValue(out breakparse);\n\n if (breakparse == false)\n dic.Add(key, val);\n\n return breakparse;\n }\n\n private string ParseName2() // unicode byte len string -> <128 len chars\n {\n byte c = _json[_index++];\n string s = Reflection.UnicodeGetString(_json, _index, c);\n _index += c;\n return s;\n }\n\n private string ParseName()\n {\n byte c = _json[_index++];\n string s = Reflection.UTF8GetString(_json, _index, c);\n _index += c;\n return s;\n }\n\n private List ParseArray()\n {\n List array = new List();\n\n bool breakparse = false;\n while (!breakparse)\n {\n object o = ParseValue(out breakparse);\n byte t = 0;\n if (breakparse == false)\n {\n array.Add(o);\n t = GetToken();\n }\n else t = (byte)o;\n\n if (t == TOKENS.COMMA)\n continue;\n if (t == TOKENS.ARRAY_END)\n break;\n }\n return array;\n }\n\n private object ParseValue(out bool breakparse)\n {\n byte t = GetToken();\n breakparse = false;\n switch (t)\n {\n case TOKENS.BYTE:\n return ParseByte();\n case TOKENS.BYTEARRAY:\n return ParseByteArray();\n case TOKENS.CHAR:\n return ParseChar();\n case TOKENS.DATETIME:\n return ParseDateTime();\n case TOKENS.DECIMAL:\n return ParseDecimal();\n case TOKENS.DOUBLE:\n return ParseDouble();\n case TOKENS.FLOAT:\n return ParseFloat();\n case TOKENS.GUID:\n return ParseGuid();\n case TOKENS.INT:\n return ParseInt();\n case TOKENS.LONG:\n return ParseLong();\n case TOKENS.SHORT:\n return ParseShort();\n case TOKENS.UINT:\n return ParseUint();\n case TOKENS.ULONG:\n return ParseULong();\n case TOKENS.USHORT:\n return ParseUShort();\n case TOKENS.UNICODE_STRING:\n return ParseUnicodeString();\n case TOKENS.STRING:\n return ParseString();\n case TOKENS.DOC_START:\n return ParseObject();\n case TOKENS.ARRAY_START:\n return ParseArray();\n case TOKENS.TRUE:\n return true;\n case TOKENS.FALSE:\n return false;\n case TOKENS.NULL:\n return null;\n case TOKENS.ARRAY_END:\n breakparse = true;\n return TOKENS.ARRAY_END;\n case TOKENS.DOC_END:\n breakparse = true;\n return TOKENS.DOC_END;\n case TOKENS.COMMA:\n breakparse = true;\n return TOKENS.COMMA;\n case TOKENS.ARRAY_TYPED:\n case TOKENS.ARRAY_TYPED_LONG:\n return ParseTypedArray(t);\n case TOKENS.TIMESPAN:\n return ParsTimeSpan();\n }\n\n throw new Exception(\"Unrecognized token at index = \" + _index);\n }\n\n private TimeSpan ParsTimeSpan()\n {\n long l = Helper.ToInt64(_json, _index);\n _index += 8;\n\n TimeSpan dt = new TimeSpan(l);\n\n return dt;\n }\n\n private object ParseTypedArray(byte token)\n {\n typedarray ar = new typedarray();\n if (token == TOKENS.ARRAY_TYPED)\n {\n if (_v1_4TA)\n ar.typename = ParseName(); \n else\n ar.typename = ParseName2();\n }\n else\n ar.typename = ParseNameLong();\n\n ar.count = ParseInt();\n\n bool breakparse = false;\n while (!breakparse)\n {\n object o = ParseValue(out breakparse);\n byte b = 0;\n if (breakparse == false)\n {\n ar.data.Add(o);\n b = GetToken();\n }\n else b = (byte)o;\n\n if (b == TOKENS.COMMA)\n continue;\n if (b == TOKENS.ARRAY_END)\n break;\n }\n return ar;\n }\n\n private string ParseNameLong() // unicode short len string -> <32k chars\n {\n short c = Helper.ToInt16(_json, _index);\n _index += 2;\n string s = Reflection.UnicodeGetString(_json, _index, c);\n _index += c;\n return s;\n }\n\n private object ParseChar()\n {\n short u = Helper.ToInt16(_json, _index);\n _index += 2;\n return u;\n }\n\n private Guid ParseGuid()\n {\n byte[] b = new byte[16];\n Buffer.BlockCopy(_json, _index, b, 0, 16);\n _index += 16;\n return new Guid(b);\n }\n\n private float ParseFloat()\n {\n float f = BitConverter.ToSingle(_json, _index);\n _index += 4;\n return f;\n }\n\n private ushort ParseUShort()\n {\n ushort u = (ushort)Helper.ToInt16(_json, _index);\n _index += 2;\n return u;\n }\n\n private ulong ParseULong()\n {\n ulong u = (ulong)Helper.ToInt64(_json, _index);\n _index += 8;\n return u;\n }\n\n private uint ParseUint()\n {\n uint u = (uint)Helper.ToInt32(_json, _index);\n _index += 4;\n return u;\n }\n\n private short ParseShort()\n {\n short u = (short)Helper.ToInt16(_json, _index);\n _index += 2;\n return u;\n }\n\n private long ParseLong()\n {\n long u = (long)Helper.ToInt64(_json, _index);\n _index += 8;\n return u;\n }\n\n private int ParseInt()\n {\n int u = (int)Helper.ToInt32(_json, _index);\n _index += 4;\n return u;\n }\n\n private double ParseDouble()\n {\n double d = BitConverter.ToDouble(_json, _index);\n _index += 8;\n return d;\n }\n\n private object ParseUnicodeString()\n {\n int c = Helper.ToInt32(_json, _index);\n _index += 4;\n\n string s = Reflection.UnicodeGetString(_json, _index, c);\n _index += c;\n return s;\n }\n\n private string ParseString()\n {\n int c = Helper.ToInt32(_json, _index);\n _index += 4;\n\n string s = Reflection.UTF8GetString(_json, _index, c);\n _index += c;\n return s;\n }\n\n private decimal ParseDecimal()\n {\n int[] i = new int[4];\n i[0] = Helper.ToInt32(_json, _index);\n _index += 4;\n i[1] = Helper.ToInt32(_json, _index);\n _index += 4;\n i[2] = Helper.ToInt32(_json, _index);\n _index += 4;\n i[3] = Helper.ToInt32(_json, _index);\n _index += 4;\n\n return new decimal(i);\n }\n\n private DateTime ParseDateTime()\n {\n long l = Helper.ToInt64(_json, _index);\n _index += 8;\n\n DateTime dt = new DateTime(l);\n if (_useUTC)\n dt = dt.ToLocalTime(); // to local time\n\n return dt;\n }\n\n private byte[] ParseByteArray()\n {\n int c = Helper.ToInt32(_json, _index);\n _index += 4;\n byte[] b = new byte[c];\n Buffer.BlockCopy(_json, _index, b, 0, c);\n _index += c;\n return b;\n }\n\n private byte ParseByte()\n {\n return _json[_index++];\n }\n\n private byte GetToken()\n {\n byte b = _json[_index++];\n return b;\n }\n }\n}\n" }, { "path": "RaptorDB.Common/fastBinaryJSON/BJsonSerializer.cs", "content": "using System;\nusing System.Collections;\nusing System.Collections.Generic;\n#if !SILVERLIGHT\nusing System.Data;\n#endif\nusing System.IO;\nusing System.Collections.Specialized;\nusing fastJSON;\n\nnamespace fastBinaryJSON\n{\n internal sealed class BJSONSerializer : IDisposable\n {\n private MemoryStream _output = new MemoryStream();\n //private MemoryStream _before = new MemoryStream();\n private int _typespointer = 0;\n private int _MAX_DEPTH = 20;\n int _current_depth = 0;\n private Dictionary _globalTypes = new Dictionary();\n private Dictionary _cirobj = new Dictionary();\n private BJSONParameters _params;\n\n private void Dispose(bool disposing)\n {\n if (disposing)\n {\n // dispose managed resources\n _output.Close();\n //_before.Close();\n }\n // free native resources\n }\n\n public void Dispose()\n {\n Dispose(true);\n GC.SuppressFinalize(this);\n }\n\n internal BJSONSerializer(BJSONParameters param)\n {\n _params = param;\n _MAX_DEPTH = param.SerializerMaxDepth;\n }\n\n internal byte[] ConvertToBJSON(object obj)\n {\n WriteValue(obj);\n\n // add $types\n if (_params.UsingGlobalTypes && _globalTypes != null && _globalTypes.Count > 0)\n {\n var pointer = (int)_output.Length;\n WriteName(\"$types\");\n WriteColon();\n WriteTypes(_globalTypes);\n //var i = _output.Length;\n _output.Seek(_typespointer, SeekOrigin.Begin);\n _output.Write(Helper.GetBytes(pointer, false), 0, 4);\n\n return _output.ToArray();\n }\n\n return _output.ToArray();\n }\n\n private void WriteTypes(Dictionary dic)\n {\n _output.WriteByte(TOKENS.DOC_START);\n\n bool pendingSeparator = false;\n\n foreach (var entry in dic)\n {\n if (pendingSeparator) WriteComma();\n\n WritePair(entry.Value.ToString(), entry.Key);\n\n pendingSeparator = true;\n }\n _output.WriteByte(TOKENS.DOC_END);\n }\n\n private void WriteValue(object obj)\n {\n if (obj == null || obj is DBNull)\n WriteNull();\n\n else if (obj is string)\n WriteString((string)obj);\n\n else if (obj is char)\n WriteChar((char)obj);\n\n else if (obj is Guid)\n WriteGuid((Guid)obj);\n\n else if (obj is bool)\n WriteBool((bool)obj);\n\n else if (obj is int)\n WriteInt((int)obj);\n\n else if (obj is uint)\n WriteUInt((uint)obj);\n\n else if (obj is long)\n WriteLong((long)obj);\n\n else if (obj is ulong)\n WriteULong((ulong)obj);\n\n else if (obj is decimal)\n WriteDecimal((decimal)obj);\n\n else if (obj is sbyte)\n WriteSByte((sbyte)obj);\n\n else if (obj is byte)\n WriteByte((byte)obj);\n\n else if (obj is double)\n WriteDouble((double)obj);\n\n else if (obj is float)\n WriteFloat((float)obj);\n\n else if (obj is short)\n WriteShort((short)obj);\n\n else if (obj is ushort)\n WriteUShort((ushort)obj);\n\n else if (obj is DateTime)\n WriteDateTime((DateTime)obj);\n\n else if (obj is TimeSpan)\n WriteTimeSpan((TimeSpan)obj);\n#if NET4\n else if (obj is System.Dynamic.ExpandoObject)\n WriteStringDictionary((IDictionary)obj);\n#endif\n\n else if (obj is IDictionary && obj.GetType().IsGenericType && obj.GetType().GetGenericArguments()[0] == typeof(string))\n WriteStringDictionary((IDictionary)obj);\n\n else if (obj is IDictionary)\n WriteDictionary((IDictionary)obj);\n#if !SILVERLIGHT\n else if (obj is DataSet)\n WriteDataset((DataSet)obj);\n\n else if (obj is DataTable)\n WriteDataTable((DataTable)obj);\n#endif\n else if (obj is byte[])\n WriteBytes((byte[])obj);\n\n else if (obj is StringDictionary)\n WriteSD((StringDictionary)obj);\n\n else if (obj is NameValueCollection)\n WriteNV((NameValueCollection)obj);\n\n else if (_params.UseTypedArrays && obj is Array)\n WriteTypedArray((ICollection)obj);\n\n else if (obj is IEnumerable)\n WriteArray((IEnumerable)obj);\n\n else if (obj is Enum)\n WriteEnum((Enum)obj);\n\n else if (Reflection.Instance.IsTypeRegistered(obj.GetType()))\n WriteCustom(obj);\n\n else\n WriteObject(obj);\n }\n\n private void WriteSByte(sbyte p)\n {\n _output.WriteByte(TOKENS.BYTE);\n byte i = (byte)p;\n _output.WriteByte(i);\n }\n\n private void WriteTimeSpan(TimeSpan obj)\n {\n _output.WriteByte(TOKENS.TIMESPAN);\n byte[] b = Helper.GetBytes(obj.Ticks, false);\n _output.Write(b, 0, b.Length);\n }\n\n private void WriteTypedArray(ICollection array)\n {\n bool pendingSeperator = false;\n bool token = true;\n var t = array.GetType();\n if (t.IsGenericType == false)// != null) // non generic array\n {\n //if (t.GetElementType().IsClass)\n {\n token = false;\n byte[] b;\n // array type name\n if (_params.v1_4TypedArray)\n b = Reflection.UTF8GetBytes(Reflection.Instance.GetTypeAssemblyName(t.GetElementType()));\n else\n b = Reflection.UnicodeGetBytes(Reflection.Instance.GetTypeAssemblyName(t.GetElementType()));\n if (b.Length < 256)\n {\n _output.WriteByte(TOKENS.ARRAY_TYPED);\n _output.WriteByte((byte)b.Length);\n _output.Write(b, 0, b.Length);\n }\n else\n {\n _output.WriteByte(TOKENS.ARRAY_TYPED_LONG);\n _output.Write(Helper.GetBytes(b.Length, false), 0, 2);\n _output.Write(b, 0, b.Length);\n }\n // array count\n _output.Write(Helper.GetBytes(array.Count, false), 0, 4); //count\n }\n }\n if (token)\n _output.WriteByte(TOKENS.ARRAY_START);\n\n foreach (object obj in array)\n {\n if (pendingSeperator) WriteComma();\n\n WriteValue(obj);\n\n pendingSeperator = true;\n }\n _output.WriteByte(TOKENS.ARRAY_END);\n }\n\n private void WriteNV(NameValueCollection nameValueCollection)\n {\n _output.WriteByte(TOKENS.DOC_START);\n\n bool pendingSeparator = false;\n\n foreach (string key in nameValueCollection)\n {\n if (pendingSeparator) _output.WriteByte(TOKENS.COMMA);\n\n WritePair(key, nameValueCollection[key]);\n\n pendingSeparator = true;\n }\n _output.WriteByte(TOKENS.DOC_END);\n }\n\n private void WriteSD(StringDictionary stringDictionary)\n {\n _output.WriteByte(TOKENS.DOC_START);\n\n bool pendingSeparator = false;\n\n foreach (DictionaryEntry entry in stringDictionary)\n {\n if (pendingSeparator) _output.WriteByte(TOKENS.COMMA);\n\n WritePair((string)entry.Key, entry.Value);\n\n pendingSeparator = true;\n }\n _output.WriteByte(TOKENS.DOC_END);\n }\n\n private void WriteUShort(ushort p)\n {\n _output.WriteByte(TOKENS.USHORT);\n _output.Write(Helper.GetBytes(p, false), 0, 2);\n }\n\n private void WriteShort(short p)\n {\n _output.WriteByte(TOKENS.SHORT);\n _output.Write(Helper.GetBytes(p, false), 0, 2);\n }\n\n private void WriteFloat(float p)\n {\n _output.WriteByte(TOKENS.FLOAT);\n byte[] b = BitConverter.GetBytes(p);\n _output.Write(b, 0, b.Length);\n }\n\n private void WriteDouble(double p)\n {\n _output.WriteByte(TOKENS.DOUBLE);\n var b = BitConverter.GetBytes(p);\n _output.Write(b, 0, b.Length);\n }\n\n private void WriteByte(byte p)\n {\n _output.WriteByte(TOKENS.BYTE);\n _output.WriteByte(p);\n }\n\n private void WriteDecimal(decimal p)\n {\n _output.WriteByte(TOKENS.DECIMAL);\n var b = decimal.GetBits(p);\n foreach (var c in b)\n _output.Write(Helper.GetBytes(c, false), 0, 4);\n }\n\n private void WriteULong(ulong p)\n {\n _output.WriteByte(TOKENS.ULONG);\n _output.Write(Helper.GetBytes((long)p, false), 0, 8);\n }\n\n private void WriteUInt(uint p)\n {\n _output.WriteByte(TOKENS.UINT);\n _output.Write(Helper.GetBytes(p, false), 0, 4);\n }\n\n private void WriteLong(long p)\n {\n _output.WriteByte(TOKENS.LONG);\n _output.Write(Helper.GetBytes(p, false), 0, 8);\n }\n\n private void WriteChar(char p)\n {\n _output.WriteByte(TOKENS.CHAR);\n _output.Write(Helper.GetBytes((short)p, false), 0, 2);\n }\n\n private void WriteBytes(byte[] p)\n {\n _output.WriteByte(TOKENS.BYTEARRAY);\n _output.Write(Helper.GetBytes(p.Length, false), 0, 4);\n _output.Write(p, 0, p.Length);\n }\n\n private void WriteBool(bool p)\n {\n if (p)\n _output.WriteByte(TOKENS.TRUE);\n else\n _output.WriteByte(TOKENS.FALSE);\n }\n\n private void WriteNull()\n {\n _output.WriteByte(TOKENS.NULL);\n }\n\n\n private void WriteCustom(object obj)\n {\n Reflection.Serialize s;\n Reflection.Instance._customSerializer.TryGetValue(obj.GetType(), out s);\n WriteString(s(obj));\n }\n\n private void WriteColon()\n {\n _output.WriteByte(TOKENS.COLON);\n }\n\n private void WriteComma()\n {\n _output.WriteByte(TOKENS.COMMA);\n }\n\n private void WriteEnum(Enum e)\n {\n WriteString(e.ToString());\n }\n\n private void WriteInt(int i)\n {\n _output.WriteByte(TOKENS.INT);\n _output.Write(Helper.GetBytes(i, false), 0, 4);\n }\n\n private void WriteGuid(Guid g)\n {\n _output.WriteByte(TOKENS.GUID);\n _output.Write(g.ToByteArray(), 0, 16);\n }\n\n private void WriteDateTime(DateTime dateTime)\n {\n DateTime dt = dateTime;\n if (_params.UseUTCDateTime)\n dt = dateTime.ToUniversalTime();\n\n _output.WriteByte(TOKENS.DATETIME);\n byte[] b = Helper.GetBytes(dt.Ticks, false);\n _output.Write(b, 0, b.Length);\n }\n\n#if !SILVERLIGHT\n private DatasetSchema GetSchema(DataTable ds)\n {\n if (ds == null) return null;\n\n DatasetSchema m = new DatasetSchema();\n m.Info = new List();\n m.Name = ds.TableName;\n\n foreach (DataColumn c in ds.Columns)\n {\n m.Info.Add(ds.TableName);\n m.Info.Add(c.ColumnName);\n m.Info.Add(c.DataType.ToString());\n }\n // FEATURE : serialize relations and constraints here\n\n return m;\n }\n\n private DatasetSchema GetSchema(DataSet ds)\n {\n if (ds == null) return null;\n\n DatasetSchema m = new DatasetSchema();\n m.Info = new List();\n m.Name = ds.DataSetName;\n\n foreach (DataTable t in ds.Tables)\n {\n foreach (DataColumn c in t.Columns)\n {\n m.Info.Add(t.TableName);\n m.Info.Add(c.ColumnName);\n m.Info.Add(c.DataType.ToString());\n }\n }\n // FEATURE : serialize relations and constraints here\n\n return m;\n }\n\n private string GetXmlSchema(DataTable dt)\n {\n using (var writer = new StringWriter())\n {\n dt.WriteXmlSchema(writer);\n return dt.ToString();\n }\n }\n\n private void WriteDataset(DataSet ds)\n {\n _output.WriteByte(TOKENS.DOC_START);\n {\n WritePair(\"$schema\", _params.UseOptimizedDatasetSchema ? (object)GetSchema(ds) : ds.GetXmlSchema());\n WriteComma();\n }\n bool tablesep = false;\n foreach (DataTable table in ds.Tables)\n {\n if (tablesep) WriteComma();\n tablesep = true;\n WriteDataTableData(table);\n }\n // end dataset\n _output.WriteByte(TOKENS.DOC_END);\n }\n\n private void WriteDataTableData(DataTable table)\n {\n WriteName(table.TableName);\n WriteColon();\n _output.WriteByte(TOKENS.ARRAY_START);\n DataColumnCollection cols = table.Columns;\n bool rowseparator = false;\n foreach (DataRow row in table.Rows)\n {\n if (rowseparator) WriteComma();\n rowseparator = true;\n _output.WriteByte(TOKENS.ARRAY_START);\n\n bool pendingSeperator = false;\n foreach (DataColumn column in cols)\n {\n if (pendingSeperator) WriteComma();\n WriteValue(row[column]);\n pendingSeperator = true;\n }\n _output.WriteByte(TOKENS.ARRAY_END);\n }\n\n _output.WriteByte(TOKENS.ARRAY_END);\n }\n\n void WriteDataTable(DataTable dt)\n {\n _output.WriteByte(TOKENS.DOC_START);\n //if (this.useExtension)\n {\n this.WritePair(\"$schema\", _params.UseOptimizedDatasetSchema ? (object)this.GetSchema(dt) : this.GetXmlSchema(dt));\n WriteComma();\n }\n\n WriteDataTableData(dt);\n\n // end datatable\n _output.WriteByte(TOKENS.DOC_END);\n }\n#endif\n bool _TypesWritten = false;\n\n private void WriteObject(object obj)\n {\n int i = 0;\n if (_cirobj.TryGetValue(obj, out i) == false)\n _cirobj.Add(obj, _cirobj.Count + 1);\n else\n {\n if (_current_depth > 0)\n {\n //_circular = true;\n _output.WriteByte(TOKENS.DOC_START);\n WriteName(\"$i\");\n WriteColon();\n WriteValue(i);\n _output.WriteByte(TOKENS.DOC_END);\n return;\n }\n }\n if (_params.UsingGlobalTypes == false)\n _output.WriteByte(TOKENS.DOC_START);\n else\n {\n if (_TypesWritten == false)\n {\n _output.WriteByte(TOKENS.DOC_START);\n // write pointer to $types position\n _output.WriteByte(TOKENS.TYPES_POINTER);\n _typespointer = (int)_output.Length; // place holder\n _output.Write(new byte[4], 0, 4); // zero pointer for now\n //_output = new MemoryStream();\n _TypesWritten = true;\n }\n else\n _output.WriteByte(TOKENS.DOC_START);\n\n }\n _current_depth++;\n if (_current_depth > _MAX_DEPTH)\n throw new Exception(\"Serializer encountered maximum depth of \" + _MAX_DEPTH);\n\n Type t = obj.GetType();\n bool append = false;\n if (_params.UseExtensions)\n {\n if (_params.UsingGlobalTypes == false)\n WritePairFast(\"$type\", Reflection.Instance.GetTypeAssemblyName(t));\n else\n {\n int dt = 0;\n string ct = Reflection.Instance.GetTypeAssemblyName(t);\n if (_globalTypes.TryGetValue(ct, out dt) == false)\n {\n dt = _globalTypes.Count + 1;\n _globalTypes.Add(ct, dt);\n }\n WritePairFast(\"$type\", dt.ToString());\n }\n append = true;\n }\n\n Getters[] g = Reflection.Instance.GetGetters(t, /*_params.ShowReadOnlyProperties,*/ _params.IgnoreAttributes);\n int c = g.Length;\n for (int ii = 0; ii < c; ii++)\n {\n var p = g[ii];\n var o = p.Getter(obj);\n if (_params.SerializeNulls == false && (o == null || o is DBNull))\n {\n\n }\n else\n {\n if (append)\n WriteComma();\n WritePair(p.Name, o);\n append = true;\n }\n }\n _output.WriteByte(TOKENS.DOC_END);\n _current_depth--;\n }\n\n private void WritePairFast(string name, string value)\n {\n if (_params.SerializeNulls == false && (value == null))\n return;\n WriteName(name);\n\n WriteColon();\n\n WriteString(value);\n }\n\n private void WritePair(string name, object value)\n {\n if (_params.SerializeNulls == false && (value == null || value is DBNull))\n return;\n WriteName(name);\n\n WriteColon();\n\n WriteValue(value);\n }\n\n private void WriteArray(IEnumerable array)\n {\n _output.WriteByte(TOKENS.ARRAY_START);\n\n bool pendingSeperator = false;\n\n foreach (object obj in array)\n {\n if (pendingSeperator) WriteComma();\n\n WriteValue(obj);\n\n pendingSeperator = true;\n }\n _output.WriteByte(TOKENS.ARRAY_END);\n }\n\n private void WriteStringDictionary(IDictionary dic)\n {\n _output.WriteByte(TOKENS.DOC_START);\n\n bool pendingSeparator = false;\n\n foreach (DictionaryEntry entry in dic)\n {\n if (pendingSeparator) WriteComma();\n\n WritePair((string)entry.Key, entry.Value);\n\n pendingSeparator = true;\n }\n _output.WriteByte(TOKENS.DOC_END);\n }\n\n private void WriteStringDictionary(IDictionary dic)\n {\n _output.WriteByte(TOKENS.DOC_START);\n\n bool pendingSeparator = false;\n\n foreach (KeyValuePair entry in dic)\n {\n if (pendingSeparator) WriteComma();\n\n WritePair((string)entry.Key, entry.Value);\n\n pendingSeparator = true;\n }\n _output.WriteByte(TOKENS.DOC_END);\n }\n\n private void WriteDictionary(IDictionary dic)\n {\n _output.WriteByte(TOKENS.ARRAY_START);\n\n bool pendingSeparator = false;\n\n foreach (DictionaryEntry entry in dic)\n {\n if (pendingSeparator) WriteComma();\n _output.WriteByte(TOKENS.DOC_START);\n WritePair(\"k\", entry.Key);\n WriteComma();\n WritePair(\"v\", entry.Value);\n _output.WriteByte(TOKENS.DOC_END);\n\n pendingSeparator = true;\n }\n _output.WriteByte(TOKENS.ARRAY_END);\n }\n\n private void WriteName(string s)\n {\n byte[] b;\n if (_params.UseUnicodeStrings == false)\n {\n _output.WriteByte(TOKENS.NAME);\n b = Reflection.UTF8GetBytes(s);\n }\n else\n {\n _output.WriteByte(TOKENS.NAME_UNI);\n b = Reflection.UnicodeGetBytes(s);\n }\n _output.WriteByte((byte)b.Length);\n _output.Write(b, 0, b.Length % 256);\n }\n\n private void WriteString(string s)\n {\n byte[] b = null;\n if (_params.UseUnicodeStrings)\n {\n _output.WriteByte(TOKENS.UNICODE_STRING);\n b = Reflection.UnicodeGetBytes(s);\n }\n else\n {\n _output.WriteByte(TOKENS.STRING);\n b = Reflection.UTF8GetBytes(s);\n }\n _output.Write(Helper.GetBytes(b.Length, false), 0, 4);\n _output.Write(b, 0, b.Length);\n }\n }\n}\n" }, { "path": "RaptorDB.Common/fastBinaryJSON/Helper.cs", "content": "using System;\nusing System.Collections.Generic;\nusing System.Linq;\nusing System.Text;\n\nnamespace fastBinaryJSON\n{\n internal static class Helper\n {\n internal static unsafe int ToInt32(byte[] value, int startIndex, bool reverse)\n {\n if (reverse)\n {\n byte[] b = new byte[4];\n Buffer.BlockCopy(value, startIndex, b, 0, 4);\n Array.Reverse(b);\n return ToInt32(b, 0);\n }\n\n return ToInt32(value, startIndex);\n }\n\n internal static unsafe int ToInt32(byte[] value, int startIndex)\n {\n fixed (byte* numRef = &(value[startIndex]))\n {\n return *((int*)numRef);\n }\n }\n\n internal static unsafe long ToInt64(byte[] value, int startIndex, bool reverse)\n {\n if (reverse)\n {\n byte[] b = new byte[8];\n Buffer.BlockCopy(value, startIndex, b, 0, 8);\n Array.Reverse(b);\n return ToInt64(b, 0);\n }\n return ToInt64(value, startIndex);\n }\n\n internal static unsafe long ToInt64(byte[] value, int startIndex)\n {\n fixed (byte* numRef = &(value[startIndex]))\n {\n return *(((long*)numRef));\n }\n }\n\n internal static unsafe short ToInt16(byte[] value, int startIndex, bool reverse)\n {\n if (reverse)\n {\n byte[] b = new byte[2];\n Buffer.BlockCopy(value, startIndex, b, 0, 2);\n Array.Reverse(b);\n return ToInt16(b, 0);\n }\n return ToInt16(value, startIndex);\n }\n\n internal static unsafe short ToInt16(byte[] value, int startIndex)\n {\n fixed (byte* numRef = &(value[startIndex]))\n {\n return *(((short*)numRef));\n }\n }\n\n internal static unsafe byte[] GetBytes(long num, bool reverse)\n {\n byte[] buffer = new byte[8];\n fixed (byte* numRef = buffer)\n {\n *((long*)numRef) = num;\n }\n if (reverse)\n Array.Reverse(buffer);\n return buffer;\n }\n\n public static unsafe byte[] GetBytes(int num, bool reverse)\n {\n byte[] buffer = new byte[4];\n fixed (byte* numRef = buffer)\n {\n *((int*)numRef) = num;\n }\n if (reverse)\n Array.Reverse(buffer);\n return buffer;\n }\n }\n}\n" }, { "path": "RaptorDB.Common/fastBinaryJSON/dynamic.cs", "content": "#if NET4\nusing System;\nusing System.Collections;\nusing System.Collections.Generic;\nusing System.Dynamic;\n\nnamespace fastBinaryJSON\n{\n internal class DynamicJson : DynamicObject, IEnumerable\n {\n private IDictionary _dictionary { get; set; }\n private List _list { get; set; }\n\n public DynamicJson(byte[] json)\n {\n var parse = fastBinaryJSON.BJSON.Parse(json);\n\n if (parse is IDictionary)\n _dictionary = (IDictionary)parse;\n else if (parse is typedarray)\n _list = ((typedarray)parse).data;\n else\n _list = (List)parse;\n }\n\n private DynamicJson(object dictionary)\n {\n if (dictionary is IDictionary)\n _dictionary = (IDictionary)dictionary;\n }\n\n public override bool TryGetIndex(GetIndexBinder binder, Object[] indexes, out Object result)\n {\n var index = indexes[0];\n if (index is int)\n {\n result = _list[(int)index];\n }\n else\n {\n result = _dictionary[(string)index];\n }\n if (result is IDictionary)\n result = new DynamicJson(result as IDictionary);\n return true;\n }\n\n public override bool TryGetMember(GetMemberBinder binder, out object result)\n {\n if (_dictionary.TryGetValue(binder.Name, out result) == false)\n if (_dictionary.TryGetValue(binder.Name.ToLowerInvariant(), out result) == false)\n return false;// throw new Exception(\"property not found \" + binder.Name);\n\n if (result is IDictionary)\n {\n result = new DynamicJson(result as IDictionary);\n }\n else if (result is List)\n {\n List list = new List();\n foreach (object item in (List)result)\n {\n if (item is IDictionary)\n list.Add(new DynamicJson(item as IDictionary));\n else\n list.Add(item);\n }\n result = list;\n }\n\n return _dictionary.ContainsKey(binder.Name);\n }\n\n public IEnumerator GetEnumerator()\n {\n foreach (var o in _list)\n {\n yield return new DynamicJson(o as IDictionary);\n }\n }\n }\n}\n#endif" }, { "path": "RaptorDB.Common/fastJSON/Formatter.cs", "content": "using System.Text;\n\nnamespace fastJSON\n{\n internal static class Formatter\n {\n //private static string _indent = \" \";\n\n private static void AppendIndent(StringBuilder sb, int count, string indent)\n {\n for (; count > 0; --count) sb.Append(indent);\n }\n\n public static string PrettyPrint(string input)\n {\n return PrettyPrint(input, new string(' ', JSON.Parameters.FormatterIndentSpaces));// \" \");\n }\n\n public static string PrettyPrint(string input, string spaces)\n {\n //_indent = spaces;\n var output = new StringBuilder();\n int depth = 0;\n int len = input.Length;\n char[] chars = input.ToCharArray();\n for (int i = 0; i < len; ++i)\n {\n char ch = chars[i];\n\n if (ch == '\\\"') // found string span\n {\n bool str = true;\n while (str)\n {\n output.Append(ch);\n ch = chars[++i];\n if (ch == '\\\\')\n {\n output.Append(ch);\n ch = chars[++i];\n }\n else if (ch == '\\\"')\n str = false;\n }\n }\n\n switch (ch)\n {\n case '{':\n case '[':\n output.Append(ch);\n output.AppendLine();\n AppendIndent(output, ++depth, spaces);\n break;\n case '}':\n case ']':\n output.AppendLine();\n AppendIndent(output, --depth, spaces);\n output.Append(ch);\n break;\n case ',':\n output.Append(ch);\n output.AppendLine();\n AppendIndent(output, depth, spaces);\n break;\n case ':':\n output.Append(\" : \");\n break;\n default:\n if (!char.IsWhiteSpace(ch))\n output.Append(ch);\n break;\n }\n }\n\n return output.ToString();\n }\n }\n}" }, { "path": "RaptorDB.Common/fastJSON/Getters.cs", "content": "using System.Collections.Generic;\n\nnamespace fastJSON\n{\n public sealed class DatasetSchema\n {\n public List Info ;//{ get; set; }\n public string Name ;//{ get; set; }\n }\n}\n" }, { "path": "RaptorDB.Common/fastJSON/Helper.cs", "content": "using System;\nusing System.Collections.Generic;\nusing System.Collections.Specialized;\n\nnamespace fastJSON\n{\n //public class FJObject : Dictionary\n //{\n\n //}\n\n class Helper\n {\n public static bool IsNullable(Type t)\n {\n if (!t.IsGenericType) return false;\n Type g = t.GetGenericTypeDefinition();\n return (g.Equals(typeof(Nullable<>)));\n }\n\n public static Type UnderlyingTypeOf(Type t)\n {\n return Reflection.Instance.GetGenericArguments(t)[0];\n }\n\n public static DateTimeOffset CreateDateTimeOffset(int year, int month, int day, int hour, int min, int sec, int milli, int extraTicks, TimeSpan offset)\n {\n var dt = new DateTimeOffset(year, month, day, hour, min, sec, milli, offset);\n\n if (extraTicks > 0)\n dt += TimeSpan.FromTicks(extraTicks);\n\n return dt;\n }\n\n public static bool BoolConv(object v)\n {\n bool oset = false;\n if (v is bool)\n oset = (bool)v;\n else if (v is long)\n oset = (long)v > 0 ? true : false;\n else if (v is string)\n {\n var s = (string)v;\n s = s.ToLowerInvariant();\n if (s == \"1\" || s == \"true\" || s == \"yes\" || s == \"on\")\n oset = true;\n }\n\n return oset;\n }\n\n public static long AutoConv(object value, JSONParameters param)\n {\n if (value is string)\n {\n if (param.AutoConvertStringToNumbers == true)\n {\n string s = (string)value;\n return CreateLong(s, 0, s.Length);\n }\n else\n throw new Exception(\"AutoConvertStringToNumbers is disabled for converting string : \" + value);\n }\n else if (value is long)\n return (long)value;\n else\n return Convert.ToInt64(value);\n }\n\n public static long CreateLong(string s, int index, int count)\n {\n //long num = 0;\n //bool neg = false;\n //for (int x = 0; x < count; x++, index++)\n //{\n // char cc = s[index];\n\n // if (cc == '-')\n // neg = true;\n // else if (cc == '+')\n // neg = false;\n // else\n // {\n // num *= 10;\n // num += (int)(cc - '0');\n // }\n //}\n //if (neg) num = -num;\n\n //return num;\n bool neg = false;\n\n char cc = s[index++];\n if (cc == '-')\n {\n neg = true;\n count--;\n cc = s[index++];\n }\n else if (cc == '+')\n {\n count--;\n cc = s[index++];\n }\n\n long num = (long)(cc - '0');\n\n while (--count > 0)\n num = num * 10 + (long)(s[index++] - '0');\n\n if (neg) num = -num;\n\n return num;\n }\n\n public static long CreateLong(char[] s, int index, int count)\n {\n //long num = 0;\n //bool neg = false;\n //for (int x = 0; x < count; x++, index++)\n //{\n // char cc = s[index];\n\n // if (cc == '-')\n // neg = true;\n // else if (cc == '+')\n // neg = false;\n // else\n // {\n // num *= 10;\n // num += (int)(cc - '0');\n // }\n //}\n //if (neg) num = -num;\n\n //return num;\n bool neg = false;\n\n char cc = s[index++];\n if (cc == '-')\n {\n neg = true;\n count--;\n cc = s[index++];\n }\n else if (cc == '+')\n {\n count--;\n cc = s[index++];\n }\n\n long num = (long)(cc - '0');\n\n while (--count > 0)\n num = num * 10 + (long)(s[index++] - '0');\n\n if (neg) num = -num;\n\n return num;\n }\n\n public static int CreateInteger(string s, int index, int count)\n {\n //int num = 0;\n //bool neg = false;\n //for (int x = 0; x < count; x++, index++)\n //{\n // char cc = s[index];\n\n // if (cc == '-')\n // neg = true;\n // else if (cc == '+')\n // neg = false;\n // else\n // {\n // num *= 10;\n // num += (int)(cc - '0');\n // }\n //}\n //if (neg) num = -num;\n\n //return num;\n bool neg = false;\n\n char cc = s[index++];\n if (cc == '-')\n {\n neg = true;\n count--;\n cc = s[index++];\n }\n else if (cc == '+')\n {\n count--;\n cc = s[index++];\n }\n\n int num = (int)(cc - '0');\n\n while (--count > 0)\n num = num * 10 + (int)(s[index++] - '0');\n\n if (neg) num = -num;\n\n return num;\n }\n\n public static object CreateEnum(Type pt, object v)\n {\n // FEATURE : optimize create enum\n#if !SILVERLIGHT\n return Enum.Parse(pt, v.ToString(), true);\n#else\n return Enum.Parse(pt, v, true);\n#endif\n }\n\n public static Guid CreateGuid(string s)\n {\n if (s.Length > 30)\n return new Guid(s);\n else\n return new Guid(Convert.FromBase64String(s));\n }\n\n public static StringDictionary CreateSD(Dictionary d)\n {\n StringDictionary nv = new StringDictionary();\n\n foreach (var o in d)\n nv.Add(o.Key, (string)o.Value);\n\n return nv;\n }\n\n public static NameValueCollection CreateNV(Dictionary d)\n {\n NameValueCollection nv = new NameValueCollection();\n\n foreach (var o in d)\n nv.Add(o.Key, (string)o.Value);\n\n return nv;\n }\n\n public static object CreateDateTimeOffset(string value)\n {\n // 0123456789012345678 9012 9/3 0/4 1/5\n // datetime format = yyyy-MM-ddTHH:mm:ss .nnn _ + 00:00\n\n // ISO8601 roundtrip formats have 7 digits for ticks, and no space before the '+'\n // datetime format = yyyy-MM-ddTHH:mm:ss .nnnnnnn + 00:00 \n // datetime format = yyyy-MM-ddTHH:mm:ss .nnnnnnn Z \n\n int year;\n int month;\n int day;\n int hour;\n int min;\n int sec;\n int ms = 0;\n int usTicks = 0; // ticks for xxx.x microseconds\n int th = 0;\n int tm = 0;\n\n year = CreateInteger(value, 0, 4);\n month = CreateInteger(value, 5, 2);\n day = CreateInteger(value, 8, 2);\n hour = CreateInteger(value, 11, 2);\n min = CreateInteger(value, 14, 2);\n sec = CreateInteger(value, 17, 2);\n\n int p = 20;\n\n if (value.Length > 21 && value[19] == '.')\n {\n ms = CreateInteger(value, p, 3);\n p = 23;\n\n // handle 7 digit case\n if (value.Length > 25 && char.IsDigit(value[p]))\n {\n usTicks = CreateInteger(value, p, 4);\n p = 27;\n }\n }\n\n if (value[p] == 'Z')\n // UTC\n return CreateDateTimeOffset(year, month, day, hour, min, sec, ms, usTicks, TimeSpan.Zero);\n\n if (value[p] == ' ')\n ++p;\n\n // +00:00\n th = CreateInteger(value, p + 1, 2);\n tm = CreateInteger(value, p + 1 + 2 + 1, 2);\n\n if (value[p] == '-')\n th = -th;\n\n return CreateDateTimeOffset(year, month, day, hour, min, sec, ms, usTicks, new TimeSpan(th, tm, 0));\n }\n\n public static DateTime CreateDateTime(string value, bool UseUTCDateTime)\n {\n if (value.Length < 19)\n return DateTime.MinValue;\n\n bool utc = false;\n // 0123456789012345678 9012 9/3\n // datetime format = yyyy-MM-ddTHH:mm:ss .nnn Z\n int year;\n int month;\n int day;\n int hour;\n int min;\n int sec;\n int ms = 0;\n\n year = CreateInteger(value, 0, 4);\n month = CreateInteger(value, 5, 2);\n day = CreateInteger(value, 8, 2);\n hour = CreateInteger(value, 11, 2);\n min = CreateInteger(value, 14, 2);\n sec = CreateInteger(value, 17, 2);\n if (value.Length > 21 && value[19] == '.')\n ms = CreateInteger(value, 20, 3);\n\n if (value[value.Length - 1] == 'Z')\n utc = true;\n\n if (UseUTCDateTime == false && utc == false)\n return new DateTime(year, month, day, hour, min, sec, ms);\n else\n return new DateTime(year, month, day, hour, min, sec, ms, DateTimeKind.Utc).ToLocalTime();\n }\n }\n}\n" }, { "path": "RaptorDB.Common/fastJSON/JSON.cs", "content": "using System;\nusing System.Collections;\nusing System.Collections.Generic;\n#if !SILVERLIGHT\nusing System.Data;\n#endif\nusing System.Globalization;\nusing System.IO;\nusing System.Collections.Specialized;\nusing RaptorDB.Common;\n\nnamespace fastJSON\n{\n //public delegate string Serialize(object data);\n //public delegate object Deserialize(string data);\n\n public sealed class JSONParameters\n {\n /// \n /// Use the optimized fast Dataset Schema format (default = True)\n /// \n public bool UseOptimizedDatasetSchema = true;\n /// \n /// Use the fast GUID format (default = True)\n /// \n public bool UseFastGuid = true;\n /// \n /// Serialize null values to the output (default = True)\n /// \n public bool SerializeNullValues = true;\n /// \n /// Use the UTC date format (default = True)\n /// \n public bool UseUTCDateTime = true;\n /// \n /// Show the readonly properties of types in the output (default = False)\n /// \n public bool ShowReadOnlyProperties = false;\n /// \n /// Use the $types extension to optimise the output json (default = True)\n /// \n public bool UsingGlobalTypes = true;\n /// \n /// Ignore case when processing json and deserializing \n /// \n [Obsolete(\"Not needed anymore and will always match\")]\n public bool IgnoreCaseOnDeserialize = false;\n /// \n /// Anonymous types have read only properties \n /// \n public bool EnableAnonymousTypes = false;\n /// \n /// Enable fastJSON extensions $types, $type, $map (default = True)\n /// \n public bool UseExtensions = true;\n /// \n /// Use escaped unicode i.e. \\uXXXX format for non ASCII characters (default = True)\n /// \n public bool UseEscapedUnicode = true;\n /// \n /// Output string key dictionaries as \"k\"/\"v\" format (default = False) \n /// \n public bool KVStyleStringDictionary = false;\n /// \n /// Output Enum values instead of names (default = False)\n /// \n public bool UseValuesOfEnums = false;\n /// \n /// Ignore attributes to check for (default : XmlIgnoreAttribute, NonSerialized)\n /// \n public List IgnoreAttributes = new List { typeof(System.Xml.Serialization.XmlIgnoreAttribute), typeof(NonSerializedAttribute) };\n /// \n /// If you have parametric and no default constructor for you classes (default = False)\n /// \n /// IMPORTANT NOTE : If True then all initial values within the class will be ignored and will be not set\n /// \n public bool ParametricConstructorOverride = false;\n /// \n /// Serialize DateTime milliseconds i.e. yyyy-MM-dd HH:mm:ss.nnn (default = false)\n /// \n public bool DateTimeMilliseconds = false;\n /// \n /// Maximum depth for circular references in inline mode (default = 20)\n /// \n public byte SerializerMaxDepth = 20;\n /// \n /// Inline circular or already seen objects instead of replacement with $i (default = false) \n /// \n public bool InlineCircularReferences = false;\n /// \n /// Save property/field names as lowercase (default = false)\n /// \n public bool SerializeToLowerCaseNames = false;\n /// \n /// Formatter indent spaces (default = 3)\n /// \n public byte FormatterIndentSpaces = 3;\n /// \n /// TESTING - allow non quoted keys in the json like javascript (default = false)\n /// \n public bool AllowNonQuotedKeys = false;\n /// \n /// Auto convert string values to numbers when needed (default = true)\n /// \n /// When disabled you will get an exception if the types don't match\n /// \n public bool AutoConvertStringToNumbers = true;\n /// \n /// Override object equality hash code checking (default = false)\n /// \n public bool OverrideObjectHashCodeChecking = false;\n /// \n /// Checking black list types to prevent friday 13th json attacks (default = true)\n /// \n /// Will throw an exception if encountered and set\n /// \n public bool BlackListTypeChecking = true;\n\n public void FixValues()\n {\n if (UseExtensions == false) // disable conflicting params\n {\n UsingGlobalTypes = false;\n InlineCircularReferences = true;\n }\n if (EnableAnonymousTypes)\n ShowReadOnlyProperties = true;\n }\n\n public JSONParameters MakeCopy()\n {\n return new JSONParameters\n {\n AllowNonQuotedKeys = AllowNonQuotedKeys,\n DateTimeMilliseconds = DateTimeMilliseconds,\n EnableAnonymousTypes = EnableAnonymousTypes,\n FormatterIndentSpaces = FormatterIndentSpaces,\n IgnoreAttributes = new List(IgnoreAttributes),\n //IgnoreCaseOnDeserialize = IgnoreCaseOnDeserialize,\n InlineCircularReferences = InlineCircularReferences,\n KVStyleStringDictionary = KVStyleStringDictionary,\n ParametricConstructorOverride = ParametricConstructorOverride,\n SerializeNullValues = SerializeNullValues,\n SerializerMaxDepth = SerializerMaxDepth,\n SerializeToLowerCaseNames = SerializeToLowerCaseNames,\n ShowReadOnlyProperties = ShowReadOnlyProperties,\n UseEscapedUnicode = UseEscapedUnicode,\n UseExtensions = UseExtensions,\n UseFastGuid = UseFastGuid,\n UseOptimizedDatasetSchema = UseOptimizedDatasetSchema,\n UseUTCDateTime = UseUTCDateTime,\n UseValuesOfEnums = UseValuesOfEnums,\n UsingGlobalTypes = UsingGlobalTypes,\n AutoConvertStringToNumbers = AutoConvertStringToNumbers,\n OverrideObjectHashCodeChecking = OverrideObjectHashCodeChecking,\n BlackListTypeChecking = BlackListTypeChecking\n };\n }\n }\n\n public static class JSON\n {\n /// \n /// Globally set-able parameters for controlling the serializer\n /// \n public static JSONParameters Parameters = new JSONParameters();\n /// \n /// Create a formatted json string (beautified) from an object\n /// \n /// \n /// \n public static string ToNiceJSON(object obj)\n {\n string s = ToJSON(obj, Parameters); // use default params\n\n return Beautify(s);\n }\n /// \n /// Create a formatted json string (beautified) from an object\n /// \n /// \n /// \n /// \n public static string ToNiceJSON(object obj, JSONParameters param)\n {\n string s = ToJSON(obj, param);\n\n return Beautify(s, param.FormatterIndentSpaces);\n }\n /// \n /// Create a json representation for an object\n /// \n /// \n /// \n public static string ToJSON(object obj)\n {\n return ToJSON(obj, Parameters);\n }\n /// \n /// Create a json representation for an object with parameter override on this call\n /// \n /// \n /// \n /// \n public static string ToJSON(object obj, JSONParameters param)\n {\n param.FixValues();\n param = param.MakeCopy();\n Type t = null;\n\n if (obj == null)\n return \"null\";\n\n if (obj.GetType().IsGenericType)\n t = Reflection.Instance.GetGenericTypeDefinition(obj.GetType());\n if (t == typeof(Dictionary<,>) || t == typeof(List<>))\n param.UsingGlobalTypes = false;\n\n // FEATURE : enable extensions when you can deserialize anon types\n if (param.EnableAnonymousTypes) { param.UseExtensions = false; param.UsingGlobalTypes = false; }\n return new JSONSerializer(param).ConvertToJSON(obj);\n }\n /// \n /// Parse a json string and generate a Dictionary<string,object> or List<object> structure\n /// \n /// \n /// \n public static object Parse(string json)\n {\n return new JsonParser(json, Parameters.AllowNonQuotedKeys).Decode();\n }\n#if NET4\n /// \n /// Create a .net4 dynamic object from the json string\n /// \n /// \n /// \n public static dynamic ToDynamic(string json)\n {\n return new DynamicJson(json);\n }\n#endif\n /// \n /// Create a typed generic object from the json\n /// \n /// \n /// \n /// \n public static T ToObject(string json)\n {\n return new deserializer(Parameters).ToObject(json);\n }\n /// \n /// Create a typed generic object from the json with parameter override on this call\n /// \n /// \n /// \n /// \n /// \n public static T ToObject(string json, JSONParameters param)\n {\n return new deserializer(param).ToObject(json);\n }\n /// \n /// Create an object from the json\n /// \n /// \n /// \n public static object ToObject(string json)\n {\n return new deserializer(Parameters).ToObject(json, null);\n }\n /// \n /// Create an object from the json with parameter override on this call\n /// \n /// \n /// \n /// \n public static object ToObject(string json, JSONParameters param)\n {\n return new deserializer(param).ToObject(json, null);\n }\n /// \n /// Create an object of type from the json\n /// \n /// \n /// \n /// \n public static object ToObject(string json, Type type)\n {\n return new deserializer(Parameters).ToObject(json, type);\n }\n /// \n /// Create an object of type from the json with parameter override on this call\n /// \n /// \n /// \n /// \n /// \n public static object ToObject(string json, Type type, JSONParameters par)\n {\n return new deserializer(par).ToObject(json, type);\n }\n /// \n /// Fill a given object with the json represenation\n /// \n /// \n /// \n /// \n public static object FillObject(object input, string json)\n {\n Dictionary ht = new JsonParser(json, Parameters.AllowNonQuotedKeys).Decode() as Dictionary;\n if (ht == null) return null;\n return new deserializer(Parameters).ParseDictionary(ht, null, input.GetType(), input);\n }\n /// \n /// Deep copy an object i.e. clone to a new object\n /// \n /// \n /// \n public static object DeepCopy(object obj)\n {\n return new deserializer(Parameters).ToObject(ToJSON(obj));\n }\n /// \n /// \n /// \n /// \n /// \n /// \n public static T DeepCopy(T obj)\n {\n return new deserializer(Parameters).ToObject(ToJSON(obj));\n }\n\n /// \n /// Create a human readable string from the json \n /// \n /// \n /// \n public static string Beautify(string input)\n {\n var i = new string(' ', JSON.Parameters.FormatterIndentSpaces);\n return Formatter.PrettyPrint(input, i);\n }\n /// \n /// Create a human readable string from the json with specified indent spaces\n /// \n /// \n /// \n /// \n public static string Beautify(string input, byte spaces)\n {\n var i = new string(' ', spaces);\n return Formatter.PrettyPrint(input, i);\n }\n /// \n /// Register custom type handlers for your own types not natively handled by fastJSON\n /// \n /// \n /// \n /// \n public static void RegisterCustomType(Type type, Reflection.Serialize serializer, Reflection.Deserialize deserializer)\n {\n Reflection.Instance.RegisterCustomType(type, serializer, deserializer);\n }\n /// \n /// Clear the internal reflection cache so you can start from new (you will loose performance)\n /// \n public static void ClearReflectionCache()\n {\n Reflection.Instance.ClearReflectionCache();\n }\n }\n\n internal class deserializer\n {\n public deserializer(JSONParameters param)\n {\n if (param.OverrideObjectHashCodeChecking)\n _circobj = new Dictionary(10, ReferenceEqualityComparer.Default);\n else\n _circobj = new Dictionary();\n param.FixValues();\n _params = param.MakeCopy();\n }\n\n private JSONParameters _params;\n private bool _usingglobals = false;\n private Dictionary _circobj;// = new Dictionary();\n private Dictionary _cirrev = new Dictionary();\n\n public T ToObject(string json)\n {\n Type t = typeof(T);\n var o = ToObject(json, t);\n\n if (t.IsArray)\n {\n if ((o as ICollection).Count == 0) // edge case for \"[]\" -> T[]\n {\n Type tt = t.GetElementType();\n object oo = Array.CreateInstance(tt, 0);\n return (T)oo;\n }\n else\n return (T)o;\n }\n else\n return (T)o;\n }\n\n public object ToObject(string json)\n {\n return ToObject(json, null);\n }\n\n public object ToObject(string json, Type type)\n {\n //_params.FixValues();\n Type t = null;\n if (type != null && type.IsGenericType)\n t = Reflection.Instance.GetGenericTypeDefinition(type);\n _usingglobals = _params.UsingGlobalTypes;\n if (t == typeof(Dictionary<,>) || t == typeof(List<>))\n _usingglobals = false;\n\n object o = new JsonParser(json, _params.AllowNonQuotedKeys).Decode();\n if (o == null)\n return null;\n#if !SILVERLIGHT\n if (type != null)\n {\n if (type == typeof(DataSet))\n return CreateDataset(o as Dictionary, null);\n else if (type == typeof(DataTable))\n return CreateDataTable(o as Dictionary, null);\n }\n#endif\n if (o is IDictionary)\n {\n if (type != null && t == typeof(Dictionary<,>)) // deserialize a dictionary\n return RootDictionary(o, type);\n else // deserialize an object\n return ParseDictionary(o as Dictionary, null, type, null);\n }\n else if (o is List)\n {\n if (type != null)\n {\n if (t == typeof(Dictionary<,>)) // kv format\n return RootDictionary(o, type);\n else if (t == typeof(List<>)) // deserialize to generic list\n return RootList(o, type);\n else if (type.IsArray)\n return RootArray(o, type);\n else if (type == typeof(Hashtable))\n return RootHashTable((List)o);\n }\n else //if (type == null)\n {\n List l = (List)o;\n if (l.Count > 0 && l[0].GetType() == typeof(Dictionary))\n {\n Dictionary globals = new Dictionary();\n List op = new List();\n // try to get $types \n foreach (var i in l)\n op.Add(ParseDictionary((Dictionary)i, globals, null, null));\n return op;\n }\n return l.ToArray();\n }\n }\n else if (type != null && o.GetType() != type)\n return ChangeType(o, type);\n\n return o;\n }\n\n #region [ p r i v a t e m e t h o d s ]\n private object RootHashTable(List o)\n {\n Hashtable h = new Hashtable();\n\n foreach (Dictionary values in o)\n {\n object key = values[\"k\"];\n object val = values[\"v\"];\n if (key is Dictionary)\n key = ParseDictionary((Dictionary)key, null, typeof(object), null);\n\n if (val is Dictionary)\n val = ParseDictionary((Dictionary)val, null, typeof(object), null);\n\n h.Add(key, val);\n }\n\n return h;\n }\n\n private object ChangeType(object value, Type conversionType)\n {\n if (conversionType == typeof(int))\n {\n string s = value as string;\n if (s == null)\n return (int)((long)value);\n else if (_params.AutoConvertStringToNumbers)\n return Helper.CreateInteger(s, 0, s.Length);\n else\n throw new Exception(\"AutoConvertStringToNumbers is disabled for converting string : \" + value);\n }\n else if (conversionType == typeof(long))\n {\n string s = value as string;\n if (s == null)\n return (long)value;\n else if (_params.AutoConvertStringToNumbers)\n return Helper.CreateLong(s, 0, s.Length);\n else\n throw new Exception(\"AutoConvertStringToNumbers is disabled for converting string : \" + value);\n }\n else if (conversionType == typeof(string))\n return (string)value;\n\n else if (conversionType.IsEnum)\n return Helper.CreateEnum(conversionType, value);\n\n else if (conversionType == typeof(DateTime))\n return Helper.CreateDateTime((string)value, _params.UseUTCDateTime);\n\n else if (conversionType == typeof(DateTimeOffset))\n return Helper.CreateDateTimeOffset((string)value);\n\n else if (Reflection.Instance.IsTypeRegistered(conversionType))\n return Reflection.Instance.CreateCustom((string)value, conversionType);\n\n // 8-30-2014 - James Brooks - Added code for nullable types.\n if (Helper.IsNullable(conversionType))\n {\n if (value == null)\n return value;\n conversionType = Helper.UnderlyingTypeOf(conversionType);\n }\n\n // 8-30-2014 - James Brooks - Nullable Guid is a special case so it was moved after the \"IsNullable\" check.\n if (conversionType == typeof(Guid))\n return Helper.CreateGuid((string)value);\n\n // 2016-04-02 - Enrico Padovani - proper conversion of byte[] back from string\n if (conversionType == typeof(byte[]))\n return Convert.FromBase64String((string)value);\n\n if (conversionType == typeof(TimeSpan))\n return new TimeSpan((long)value);\n\n return Convert.ChangeType(value, conversionType, CultureInfo.InvariantCulture);\n }\n\n private object RootList(object parse, Type type)\n {\n Type[] gtypes = Reflection.Instance.GetGenericArguments(type);\n var o = (IList)Reflection.Instance.FastCreateList(type, ((IList)parse).Count);\n DoParseList((IList)parse, gtypes[0], o);\n return o;\n }\n\n private void DoParseList(IList parse, Type it, IList o)\n {\n Dictionary globals = new Dictionary();\n\n foreach (var k in parse)\n {\n _usingglobals = false;\n object v = k;\n var a = k as Dictionary;\n if (a != null)\n v = ParseDictionary(a, globals, it, null);\n else\n v = ChangeType(k, it);\n\n o.Add(v);\n }\n }\n\n private object RootArray(object parse, Type type)\n {\n Type it = type.GetElementType();\n var o = (IList)Reflection.Instance.FastCreateInstance(typeof(List<>).MakeGenericType(it));\n DoParseList((IList)parse, it, o);\n var array = Array.CreateInstance(it, o.Count);\n o.CopyTo(array, 0);\n\n return array;\n }\n\n private object RootDictionary(object parse, Type type)\n {\n Type[] gtypes = Reflection.Instance.GetGenericArguments(type);\n Type t1 = null;\n Type t2 = null;\n bool dictionary = false;\n if (gtypes != null)\n {\n t1 = gtypes[0];\n t2 = gtypes[1];\n if (t2 != null)\n dictionary = t2.Name.StartsWith(\"Dictionary\");\n }\n\n var arraytype = t2.GetElementType();\n if (parse is Dictionary)\n {\n IDictionary o = (IDictionary)Reflection.Instance.FastCreateInstance(type);\n\n foreach (var kv in (Dictionary)parse)\n {\n object v;\n object k = ChangeType(kv.Key, t1);\n\n if (dictionary) // deserialize a dictionary\n v = RootDictionary(kv.Value, t2);\n\n else if (kv.Value is Dictionary)\n v = ParseDictionary(kv.Value as Dictionary, null, t2, null);\n\n else if (t2.IsArray && t2 != typeof(byte[]))\n v = CreateArray((List)kv.Value, t2, arraytype, null);\n\n else if (kv.Value is IList)\n v = CreateGenericList((List)kv.Value, t2, t1, null);\n\n else\n v = ChangeType(kv.Value, t2);\n\n o.Add(k, v);\n }\n\n return o;\n }\n if (parse is List)\n return CreateDictionary(parse as List, type, gtypes, null);\n\n return null;\n }\n\n internal object ParseDictionary(Dictionary d, Dictionary globaltypes, Type type, object input)\n {\n object tn = \"\";\n if (type == typeof(NameValueCollection))\n return Helper.CreateNV(d);\n if (type == typeof(StringDictionary))\n return Helper.CreateSD(d);\n\n if (d.TryGetValue(\"$i\", out tn))\n {\n object v = null;\n _cirrev.TryGetValue((int)(long)tn, out v);\n return v;\n }\n\n if (d.TryGetValue(\"$types\", out tn))\n {\n _usingglobals = true;\n if (globaltypes == null)\n globaltypes = new Dictionary();\n foreach (var kv in (Dictionary)tn)\n {\n globaltypes.Add((string)kv.Value, kv.Key);\n }\n }\n if (globaltypes != null)\n _usingglobals = true;\n\n bool found = d.TryGetValue(\"$type\", out tn);\n#if !SILVERLIGHT\n if (found == false && type == typeof(System.Object))\n {\n return d; // CreateDataset(d, globaltypes);\n }\n#endif\n if (found)\n {\n if (_usingglobals)\n {\n object tname = \"\";\n if (globaltypes != null && globaltypes.TryGetValue((string)tn, out tname))\n tn = tname;\n }\n type = Reflection.Instance.GetTypeFromCache((string)tn, _params.BlackListTypeChecking);\n }\n\n if (type == null)\n throw new Exception(\"Cannot determine type : \" + tn);\n\n string typename = type.FullName;\n object o = input;\n if (o == null)\n {\n if (_params.ParametricConstructorOverride)\n o = System.Runtime.Serialization.FormatterServices.GetUninitializedObject(type);\n else\n o = Reflection.Instance.FastCreateInstance(type);\n }\n int circount = 0;\n if (_circobj.TryGetValue(o, out circount) == false)\n {\n circount = _circobj.Count + 1;\n _circobj.Add(o, circount);\n _cirrev.Add(circount, o);\n }\n\n var props = Reflection.Instance.Getproperties(type, typename, _params.ShowReadOnlyProperties);\n foreach (var kv in d)\n {\n var n = kv.Key;\n var v = kv.Value;\n\n string name = n;//.ToLower();\n if (name == \"$map\")\n {\n ProcessMap(o, props, (Dictionary)d[name]);\n continue;\n }\n myPropInfo pi;\n if (props.TryGetValue(name, out pi) == false)\n if (props.TryGetValue(name.ToLowerInvariant(), out pi) == false)\n continue;\n\n if (pi.CanWrite)\n {\n if (v != null)\n {\n object oset = null;\n\n switch (pi.Type)\n {\n case myPropInfoType.Int: oset = (int)Helper.AutoConv(v, _params); break;\n case myPropInfoType.Long: oset = Helper.AutoConv(v, _params); break;\n case myPropInfoType.String: oset = v.ToString(); break;\n case myPropInfoType.Bool: oset = Helper.BoolConv(v); break;\n case myPropInfoType.DateTime: oset = Helper.CreateDateTime((string)v, _params.UseUTCDateTime); break;\n case myPropInfoType.Enum: oset = Helper.CreateEnum(pi.pt, v); break;\n case myPropInfoType.Guid: oset = Helper.CreateGuid((string)v); break;\n\n case myPropInfoType.Array:\n if (!pi.IsValueType)\n oset = CreateArray((List)v, pi.pt, pi.bt, globaltypes);\n // what about 'else'?\n break;\n case myPropInfoType.ByteArray: oset = Convert.FromBase64String((string)v); break;\n#if !SILVERLIGHT\n case myPropInfoType.DataSet: oset = CreateDataset((Dictionary)v, globaltypes); break;\n case myPropInfoType.DataTable: oset = CreateDataTable((Dictionary)v, globaltypes); break;\n case myPropInfoType.Hashtable: // same case as Dictionary\n#endif\n case myPropInfoType.Dictionary: oset = CreateDictionary((List)v, pi.pt, pi.GenericTypes, globaltypes); break;\n case myPropInfoType.StringKeyDictionary: oset = CreateStringKeyDictionary((Dictionary)v, pi.pt, pi.GenericTypes, globaltypes); break;\n case myPropInfoType.NameValue: oset = Helper.CreateNV((Dictionary)v); break;\n case myPropInfoType.StringDictionary: oset = Helper.CreateSD((Dictionary)v); break;\n case myPropInfoType.Custom: oset = Reflection.Instance.CreateCustom((string)v, pi.pt); break;\n default:\n {\n if (pi.IsGenericType && pi.IsValueType == false && v is List)\n oset = CreateGenericList((List)v, pi.pt, pi.bt, globaltypes);\n\n else if ((pi.IsClass || pi.IsStruct || pi.IsInterface) && v is Dictionary)\n oset = ParseDictionary((Dictionary)v, globaltypes, pi.pt, null);// pi.getter(o));\n\n else if (v is List)\n oset = CreateArray((List)v, pi.pt, typeof(object), globaltypes);\n\n else if (pi.IsValueType)\n oset = ChangeType(v, pi.changeType);\n\n else\n oset = v;\n }\n break;\n }\n\n o = pi.setter(o, oset);\n }\n }\n }\n return o;\n }\n\n private static void ProcessMap(object obj, Dictionary props, Dictionary dic)\n {\n foreach (KeyValuePair kv in dic)\n {\n myPropInfo p = props[kv.Key];\n object o = p.getter(obj);\n Type t = Type.GetType((string)kv.Value);\n if (t == typeof(Guid))\n p.setter(obj, Helper.CreateGuid((string)o));\n }\n }\n\n private object CreateArray(List data, Type pt, Type bt, Dictionary globalTypes)\n {\n if (bt == null)\n bt = typeof(object);\n\n Array col = Array.CreateInstance(bt, data.Count);\n var arraytype = bt.GetElementType();\n // create an array of objects\n for (int i = 0; i < data.Count; i++)\n {\n object ob = data[i];\n if (ob == null)\n {\n continue;\n }\n if (ob is IDictionary)\n col.SetValue(ParseDictionary((Dictionary)ob, globalTypes, bt, null), i);\n else if (ob is ICollection)\n col.SetValue(CreateArray((List)ob, bt, arraytype, globalTypes), i);\n else\n col.SetValue(ChangeType(ob, bt), i);\n }\n\n return col;\n }\n\n private object CreateGenericList(List data, Type pt, Type bt, Dictionary globalTypes)\n {\n if (pt != typeof(object))\n {\n IList col = (IList)Reflection.Instance.FastCreateList(pt, data.Count);\n var it = Reflection.Instance.GetGenericArguments(pt)[0];// pt.GetGenericArguments()[0];\n // create an array of objects\n foreach (object ob in data)\n {\n if (ob is IDictionary)\n col.Add(ParseDictionary((Dictionary)ob, globalTypes, it, null));\n\n else if (ob is List)\n {\n if (bt.IsGenericType)\n col.Add((List)ob);//).ToArray());\n else\n col.Add(((List)ob).ToArray());\n }\n else\n col.Add(ChangeType(ob, it));\n }\n return col;\n }\n return data;\n }\n\n private object CreateStringKeyDictionary(Dictionary reader, Type pt, Type[] types, Dictionary globalTypes)\n {\n var col = (IDictionary)Reflection.Instance.FastCreateInstance(pt);\n Type arraytype = null;\n Type t2 = null;\n if (types != null)\n t2 = types[1];\n\n Type generictype = null;\n var ga = Reflection.Instance.GetGenericArguments(t2);// t2.GetGenericArguments();\n if (ga.Length > 0)\n generictype = ga[0];\n arraytype = t2.GetElementType();\n\n foreach (KeyValuePair values in reader)\n {\n var key = values.Key;\n object val = null;\n\n if (values.Value is Dictionary)\n val = ParseDictionary((Dictionary)values.Value, globalTypes, t2, null);\n\n else if (types != null && t2.IsArray)\n {\n if (values.Value is Array)\n val = values.Value;\n else\n val = CreateArray((List)values.Value, t2, arraytype, globalTypes);\n }\n else if (values.Value is IList)\n val = CreateGenericList((List)values.Value, t2, generictype, globalTypes);\n\n else\n val = ChangeType(values.Value, t2);\n\n col.Add(key, val);\n }\n\n return col;\n }\n\n private object CreateDictionary(List reader, Type pt, Type[] types, Dictionary globalTypes)\n {\n IDictionary col = (IDictionary)Reflection.Instance.FastCreateInstance(pt);\n Type t1 = null;\n Type t2 = null;\n Type generictype = null;\n if (types != null)\n {\n t1 = types[0];\n t2 = types[1];\n }\n Type arraytype = t2;\n if (t2 != null)\n {\n var ga = Reflection.Instance.GetGenericArguments(t2);// t2.GetGenericArguments();\n if (ga.Length > 0)\n generictype = ga[0];\n arraytype = t2.GetElementType();\n }\n bool root = typeof(IDictionary).IsAssignableFrom(t2);\n\n foreach (Dictionary values in reader)\n {\n object key = values[\"k\"];\n object val = values[\"v\"];\n\n if (key is Dictionary)\n key = ParseDictionary((Dictionary)key, globalTypes, t1, null);\n else\n key = ChangeType(key, t1);\n\n if (root)\n val = RootDictionary(val, t2);\n\n else if (val is Dictionary)\n val = ParseDictionary((Dictionary)val, globalTypes, t2, null);\n\n else if (types != null && t2.IsArray)\n val = CreateArray((List)val, t2, arraytype, globalTypes);\n\n else if (val is IList)\n val = CreateGenericList((List)val, t2, generictype, globalTypes);\n\n else\n val = ChangeType(val, t2);\n\n col.Add(key, val);\n }\n\n return col;\n }\n\n#if !SILVERLIGHT\n private DataSet CreateDataset(Dictionary reader, Dictionary globalTypes)\n {\n DataSet ds = new DataSet();\n ds.EnforceConstraints = false;\n ds.BeginInit();\n\n // read dataset schema here\n var schema = reader[\"$schema\"];\n\n if (schema is string)\n {\n TextReader tr = new StringReader((string)schema);\n ds.ReadXmlSchema(tr);\n }\n else\n {\n DatasetSchema ms = (DatasetSchema)ParseDictionary((Dictionary)schema, globalTypes, typeof(DatasetSchema), null);\n ds.DataSetName = ms.Name;\n for (int i = 0; i < ms.Info.Count; i += 3)\n {\n if (ds.Tables.Contains(ms.Info[i]) == false)\n ds.Tables.Add(ms.Info[i]);\n ds.Tables[ms.Info[i]].Columns.Add(ms.Info[i + 1], Type.GetType(ms.Info[i + 2]));\n }\n }\n\n foreach (KeyValuePair pair in reader)\n {\n if (pair.Key == \"$type\" || pair.Key == \"$schema\") continue;\n\n List rows = (List)pair.Value;\n if (rows == null) continue;\n\n DataTable dt = ds.Tables[pair.Key];\n ReadDataTable(rows, dt);\n }\n\n ds.EndInit();\n\n return ds;\n }\n\n private void ReadDataTable(List rows, DataTable dt)\n {\n dt.BeginInit();\n dt.BeginLoadData();\n List guidcols = new List();\n List datecol = new List();\n List bytearraycol = new List();\n\n foreach (DataColumn c in dt.Columns)\n {\n if (c.DataType == typeof(Guid) || c.DataType == typeof(Guid?))\n guidcols.Add(c.Ordinal);\n if (_params.UseUTCDateTime && (c.DataType == typeof(DateTime) || c.DataType == typeof(DateTime?)))\n datecol.Add(c.Ordinal);\n if (c.DataType == typeof(byte[]))\n bytearraycol.Add(c.Ordinal);\n }\n\n foreach (List row in rows)\n {\n object[] v = new object[row.Count];\n row.CopyTo(v, 0);\n foreach (int i in guidcols)\n {\n string s = (string)v[i];\n if (s != null && s.Length < 36)\n v[i] = new Guid(Convert.FromBase64String(s));\n }\n foreach (int i in bytearraycol)\n {\n string s = (string)v[i];\n if (s != null)\n v[i] = Convert.FromBase64String(s);\n }\n if (_params.UseUTCDateTime)\n {\n foreach (int i in datecol)\n {\n string s = (string)v[i];\n if (s != null)\n v[i] = Helper.CreateDateTime(s, _params.UseUTCDateTime);\n }\n }\n dt.Rows.Add(v);\n }\n\n dt.EndLoadData();\n dt.EndInit();\n }\n\n DataTable CreateDataTable(Dictionary reader, Dictionary globalTypes)\n {\n var dt = new DataTable();\n\n // read dataset schema here\n var schema = reader[\"$schema\"];\n\n if (schema is string)\n {\n TextReader tr = new StringReader((string)schema);\n dt.ReadXmlSchema(tr);\n }\n else\n {\n var ms = (DatasetSchema)ParseDictionary((Dictionary)schema, globalTypes, typeof(DatasetSchema), null);\n dt.TableName = ms.Info[0];\n for (int i = 0; i < ms.Info.Count; i += 3)\n {\n dt.Columns.Add(ms.Info[i + 1], Type.GetType(ms.Info[i + 2]));\n }\n }\n\n foreach (var pair in reader)\n {\n if (pair.Key == \"$type\" || pair.Key == \"$schema\")\n continue;\n\n var rows = (List)pair.Value;\n if (rows == null)\n continue;\n\n if (!dt.TableName.Equals(pair.Key, StringComparison.InvariantCultureIgnoreCase))\n continue;\n\n ReadDataTable(rows, dt);\n }\n\n return dt;\n }\n#endif\n #endregion\n }\n\n}\n" }, { "path": "RaptorDB.Common/fastJSON/JsonParser.cs", "content": "using System;\nusing System.Collections.Generic;\nusing System.Globalization;\nusing System.Text;\n\nnamespace fastJSON\n{\n\n /// \n /// This class encodes and decodes JSON strings.\n /// Spec. details, see http://www.json.org/\n /// \n internal sealed class JsonParser\n {\n enum Token\n {\n None = -1, // Used to denote no Lookahead available\n Curly_Open,\n Curly_Close,\n Squared_Open,\n Squared_Close,\n Colon,\n Comma,\n String,\n Number,\n True,\n False,\n Null//, \n //Key\n }\n\n readonly char[] json;\n readonly StringBuilder s = new StringBuilder(); // used for inner string parsing \" \\\"\\r\\n\\u1234\\'\\t \" \n Token lookAheadToken = Token.None;\n int index;\n bool allownonquotedkey = false;\n int _len = 0;\n\n internal JsonParser(string json, bool AllowNonQuotedKeys)\n {\n this.allownonquotedkey = AllowNonQuotedKeys;\n this.json = json.ToCharArray();\n _len = json.Length;\n }\n\n public unsafe object Decode()\n {\n fixed(char* p = json)\n return ParseValue(p, false);\n }\n\n private unsafe Dictionary ParseObject(char* p)\n {\n Dictionary table = new Dictionary(10);\n\n ConsumeToken(); // {\n\n while (true)\n {\n switch (LookAhead(p))\n {\n\n case Token.Comma:\n ConsumeToken();\n break;\n\n case Token.Curly_Close:\n ConsumeToken();\n return table;\n\n default:\n {\n // name\n string name = ParseString(p, false);\n\n var n = NextToken(p);\n // :\n if (n != Token.Colon)\n {\n throw new Exception(\"Expected colon at index \" + index);\n }\n\n // value\n object value = ParseValue(p, true);\n\n //table.Add(name, value);\n table[name] = value;\n }\n break;\n }\n }\n }\n\n private unsafe List ParseArray(char* p)\n {\n List array = new List(10);\n ConsumeToken(); // [\n\n while (true)\n {\n switch (LookAhead(p))\n {\n case Token.Comma:\n ConsumeToken();\n break;\n\n case Token.Squared_Close:\n ConsumeToken();\n return array;\n\n default:\n array.Add(ParseValue(p, false));\n break;\n }\n }\n }\n\n private unsafe object ParseValue(char* p, bool val)\n {\n switch (LookAhead(p))\n {\n case Token.Number:\n return ParseNumber(p);\n\n case Token.String:\n return ParseString(p, val);\n\n case Token.Curly_Open:\n return ParseObject(p);\n\n case Token.Squared_Open:\n return ParseArray(p);\n\n case Token.True:\n ConsumeToken();\n return true;\n\n case Token.False:\n ConsumeToken();\n return false;\n\n case Token.Null:\n ConsumeToken();\n return null;\n }\n\n throw new Exception(\"Unrecognized token at index\" + index);\n }\n\n private unsafe string ParseString(char* p, bool val)\n {\n ConsumeToken(); // \"\n\n if (s.Length > 0)\n s.Length = 0;\n //s.Clear();\n bool instr = val;\n int runIndex = -1;\n int l = _len;\n //fixed (char* p = json)\n //char[] p = json;\n {\n while (index < l)\n {\n var c = p[index++];\n if (c == '\"')\n instr = true;\n\n if (c == '\"' || (allownonquotedkey && instr == false && (c == ':' || c == ' ' || c == '\\t')))\n {\n int len = 1;\n if (allownonquotedkey && c != '\"' && instr == false)\n {\n index--;\n len = 0;\n }\n\n if (runIndex != -1)\n {\n if (s.Length == 0)\n return UnsafeSubstring(p, runIndex, index - runIndex - len);\n\n s.Append(json, runIndex, index - runIndex - 1);\n }\n return s.ToString();\n }\n\n if (c != '\\\\')\n {\n if (runIndex == -1)\n runIndex = index - 1;\n\n continue;\n }\n\n if (index == l) break;\n\n if (runIndex != -1)\n {\n s.Append(json, runIndex, index - runIndex - 1);\n runIndex = -1;\n }\n\n switch (p[index++])\n {\n case '\"':\n s.Append('\"');\n break;\n\n case '\\\\':\n s.Append('\\\\');\n break;\n\n case '/':\n s.Append('/');\n break;\n\n case 'b':\n s.Append('\\b');\n break;\n\n case 'f':\n s.Append('\\f');\n break;\n\n case 'n':\n s.Append('\\n');\n break;\n\n case 'r':\n s.Append('\\r');\n break;\n\n case 't':\n s.Append('\\t');\n break;\n\n case 'u':\n {\n int remainingLength = l - index;\n if (remainingLength < 4) break;\n\n // parse the 32 bit hex into an integer codepoint\n uint codePoint = ParseUnicode(p[index], p[index + 1], p[index + 2], p[index + 3]);\n s.Append((char)codePoint);\n\n // skip 4 chars\n index += 4;\n }\n break;\n }\n }\n }\n\n throw new Exception(\"Unexpectedly reached end of string\");\n }\n\n private uint ParseSingleChar(char c1, uint multipliyer)\n {\n uint p1 = 0;\n if (c1 >= '0' && c1 <= '9')\n p1 = (uint)(c1 - '0') * multipliyer;\n else if (c1 >= 'A' && c1 <= 'F')\n p1 = (uint)((c1 - 'A') + 10) * multipliyer;\n else if (c1 >= 'a' && c1 <= 'f')\n p1 = (uint)((c1 - 'a') + 10) * multipliyer;\n return p1;\n }\n\n private uint ParseUnicode(char c1, char c2, char c3, char c4)\n {\n uint p1 = ParseSingleChar(c1, 0x1000);\n uint p2 = ParseSingleChar(c2, 0x100);\n uint p3 = ParseSingleChar(c3, 0x10);\n uint p4 = ParseSingleChar(c4, 1);\n\n return p1 + p2 + p3 + p4;\n }\n\n private unsafe object ParseNumber(char* p)\n {\n ConsumeToken();\n\n // Need to start back one place because the first digit is also a token and would have been consumed\n var startIndex = index - 1;\n bool dec = false;\n bool dob = false;\n do\n {\n if (index == _len)\n break;\n var c = json[index];\n\n if ((c >= '0' && c <= '9') || c == '.' || c == '-' || c == '+' || c == 'e' || c == 'E')\n {\n if (/*c == '.' ||*/ c == 'e' || c == 'E')\n dob = true;\n if (c == '.')\n dec = true;\n if (++index == _len)\n break;//throw new Exception(\"Unexpected end of string whilst parsing number\");\n continue;\n }\n break;\n } while (true);\n\n if (dob)\n {\n string s = UnsafeSubstring(p, startIndex, index - startIndex);// json.Substring(startIndex, index - startIndex);\n return double.Parse(s, NumberFormatInfo.InvariantInfo);\n }\n if (dec == false && index - startIndex < 20 )// && json[startIndex] != '-')\n return Helper.CreateLong(json, startIndex, index - startIndex);\n else\n {\n string s = UnsafeSubstring(p, startIndex, index - startIndex);//json.Substring(startIndex, index - startIndex);\n return decimal.Parse(s, NumberFormatInfo.InvariantInfo);\n }\n }\n\n private unsafe Token LookAhead(char* p)\n {\n if (lookAheadToken != Token.None) return lookAheadToken;\n\n return lookAheadToken = NextTokenCore(p);\n }\n\n private void ConsumeToken()\n {\n lookAheadToken = Token.None;\n }\n\n private unsafe Token NextToken(char* p)\n {\n var result = lookAheadToken != Token.None ? lookAheadToken : NextTokenCore(p);\n\n lookAheadToken = Token.None;\n\n return result;\n }\n\n private unsafe Token NextTokenCore(char* p)\n {\n char c;\n int len = _len;\n\n // Skip past whitespace\n do\n {\n //fixed (char* p = json)\n {\n c = p[index];\n\n if (c == '/' && p[index + 1] == '/') // c++ style single line comments\n {\n index++;\n index++;\n do\n {\n c = p[index];\n if (c == '\\r' || c == '\\n') break; // read till end of line\n }\n while (++index < len);\n }\n if (c > ' ') break;\n if (c != ' ' && c != '\\t' && c != '\\n' && c != '\\r') break;\n }\n } while (++index < len);\n\n if (index == len)\n {\n throw new Exception(\"Reached end of string unexpectedly\");\n }\n\n //fixed (char* p = json)\n {\n c = p[index];\n\n index++;\n\n switch (c)\n {\n case '{':\n return Token.Curly_Open;\n\n case '}':\n return Token.Curly_Close;\n\n case '[':\n return Token.Squared_Open;\n\n case ']':\n return Token.Squared_Close;\n\n case ',':\n return Token.Comma;\n\n case '\"':\n return Token.String;\n\n case '0':\n case '1':\n case '2':\n case '3':\n case '4':\n case '5':\n case '6':\n case '7':\n case '8':\n case '9':\n case '-':\n case '+':\n case '.':\n return Token.Number;\n\n case ':':\n return Token.Colon;\n\n case 'f':\n if (len - index >= 4 &&\n p[index + 0] == 'a' &&\n p[index + 1] == 'l' &&\n p[index + 2] == 's' &&\n p[index + 3] == 'e')\n {\n index += 4;\n return Token.False;\n }\n break;\n\n case 't':\n if (len - index >= 3 &&\n p[index + 0] == 'r' &&\n p[index + 1] == 'u' &&\n p[index + 2] == 'e')\n {\n index += 3;\n return Token.True;\n }\n break;\n\n case 'n':\n if (len - index >= 3 &&\n p[index + 0] == 'u' &&\n p[index + 1] == 'l' &&\n p[index + 2] == 'l')\n {\n index += 3;\n return Token.Null;\n }\n break;\n }\n }\n if (allownonquotedkey)\n {\n index--;\n return Token.String;\n }\n else\n throw new Exception(\"Could not find token at index \" + --index);\n }\n\n private static unsafe string UnsafeSubstring(//char[] source, \n char* p, int startIndex, int length)\n {\n //fixed (char* c = source)\n return new string(p, startIndex, length);\n }\n }\n}\n" }, { "path": "RaptorDB.Common/fastJSON/JsonSerializer.cs", "content": "using System;\nusing System.Collections;\nusing System.Collections.Generic;\n#if !SILVERLIGHT\nusing System.Data;\n#endif\nusing System.Globalization;\nusing System.IO;\nusing System.Text;\nusing System.Collections.Specialized;\nusing RaptorDB.Common;\n\nnamespace fastJSON\n{\n internal sealed class JSONSerializer\n {\n private StringBuilder _output = new StringBuilder();\n //private StringBuilder _before = new StringBuilder();\n private int _before;\n private int _MAX_DEPTH = 20;\n int _current_depth = 0;\n private Dictionary _globalTypes = new Dictionary();\n private Dictionary _cirobj;\n private JSONParameters _params;\n private bool _useEscapedUnicode = false;\n\n internal JSONSerializer(JSONParameters param)\n {\n if (param.OverrideObjectHashCodeChecking)\n _cirobj = new Dictionary(10, ReferenceEqualityComparer.Default);\n else\n _cirobj = new Dictionary();\n _params = param;\n _useEscapedUnicode = _params.UseEscapedUnicode;\n _MAX_DEPTH = _params.SerializerMaxDepth;\n }\n\n internal string ConvertToJSON(object obj)\n {\n WriteValue(obj);\n\n if (_params.UsingGlobalTypes && _globalTypes != null && _globalTypes.Count > 0)\n {\n var sb = new StringBuilder();\n sb.Append(\"\\\"$types\\\":{\");\n var pendingSeparator = false;\n foreach (var kv in _globalTypes)\n {\n if (pendingSeparator) sb.Append(',');\n pendingSeparator = true;\n sb.Append('\\\"');\n sb.Append(kv.Key);\n sb.Append(\"\\\":\\\"\");\n sb.Append(kv.Value);\n sb.Append('\\\"');\n }\n sb.Append(\"},\");\n _output.Insert(_before, sb.ToString());\n }\n return _output.ToString();\n }\n\n private void WriteValue(object obj)\n {\n if (obj == null || obj is DBNull)\n _output.Append(\"null\");\n\n else if (obj is string || obj is char)\n WriteString(obj.ToString());\n\n else if (obj is Guid)\n WriteGuid((Guid)obj);\n\n else if (obj is bool)\n _output.Append(((bool)obj) ? \"true\" : \"false\"); // conform to standard\n\n else if (\n obj is int || obj is long ||\n obj is decimal ||\n obj is byte || obj is short ||\n obj is sbyte || obj is ushort ||\n obj is uint || obj is ulong\n )\n _output.Append(((IConvertible)obj).ToString(NumberFormatInfo.InvariantInfo));\n\n else if (obj is double || obj is Double)\n {\n double d = (double)obj;\n if (double.IsNaN(d))\n _output.Append(\"\\\"NaN\\\"\");\n else if (double.IsInfinity(d))\n {\n _output.Append(\"\\\"\");\n _output.Append(((IConvertible)obj).ToString(NumberFormatInfo.InvariantInfo));\n _output.Append(\"\\\"\");\n }\n else\n _output.Append(((IConvertible)obj).ToString(NumberFormatInfo.InvariantInfo));\n }\n else if (obj is float || obj is Single)\n {\n float d = (float)obj;\n if (float.IsNaN(d))\n _output.Append(\"\\\"NaN\\\"\");\n else if (float.IsInfinity(d))\n {\n _output.Append(\"\\\"\");\n _output.Append(((IConvertible)obj).ToString(NumberFormatInfo.InvariantInfo));\n _output.Append(\"\\\"\");\n }\n else\n _output.Append(((IConvertible)obj).ToString(NumberFormatInfo.InvariantInfo));\n }\n\n else if (obj is DateTime)\n WriteDateTime((DateTime)obj);\n\n else if (obj is DateTimeOffset)\n WriteDateTimeOffset((DateTimeOffset)obj);\n\n else if (obj is TimeSpan)\n _output.Append(((TimeSpan)obj).Ticks);\n\n#if NET4\n else if (_params.KVStyleStringDictionary == false &&\n obj is IEnumerable>)\n\n WriteStringDictionary((IEnumerable>)obj);\n#endif\n\n else if (_params.KVStyleStringDictionary == false && obj is IDictionary &&\n obj.GetType().IsGenericType && Reflection.Instance.GetGenericArguments(obj.GetType())[0] == typeof(string))\n\n WriteStringDictionary((IDictionary)obj);\n else if (obj is IDictionary)\n WriteDictionary((IDictionary)obj);\n#if !SILVERLIGHT\n else if (obj is DataSet)\n WriteDataset((DataSet)obj);\n\n else if (obj is DataTable)\n this.WriteDataTable((DataTable)obj);\n#endif\n else if (obj is byte[])\n WriteBytes((byte[])obj);\n\n else if (obj is StringDictionary)\n WriteSD((StringDictionary)obj);\n\n else if (obj is NameValueCollection)\n WriteNV((NameValueCollection)obj);\n\n else if (obj is IEnumerable)\n WriteArray((IEnumerable)obj);\n\n else if (obj is Enum)\n WriteEnum((Enum)obj);\n\n else if (Reflection.Instance.IsTypeRegistered(obj.GetType()))\n WriteCustom(obj);\n\n else\n WriteObject(obj);\n }\n\n private void WriteDateTimeOffset(DateTimeOffset d)\n {\n DateTime dt = _params.UseUTCDateTime ? d.UtcDateTime : d.DateTime;\n \n write_date_value(dt);\n\n var ticks = dt.Ticks % TimeSpan.TicksPerSecond;\n _output.Append('.');\n _output.Append(ticks.ToString(\"0000000\", NumberFormatInfo.InvariantInfo));\n\n if (_params.UseUTCDateTime)\n _output.Append('Z');\n else\n {\n if (d.Offset.Hours > 0)\n _output.Append(\"+\");\n else\n _output.Append(\"-\");\n _output.Append(d.Offset.Hours.ToString(\"00\", NumberFormatInfo.InvariantInfo));\n _output.Append(\":\");\n _output.Append(d.Offset.Minutes.ToString(\"00\", NumberFormatInfo.InvariantInfo));\n }\n\n _output.Append('\\\"');\n }\n\n private void WriteNV(NameValueCollection nameValueCollection)\n {\n _output.Append('{');\n\n bool pendingSeparator = false;\n\n foreach (string key in nameValueCollection)\n {\n if (_params.SerializeNullValues == false && (nameValueCollection[key] == null))\n {\n }\n else\n {\n if (pendingSeparator) _output.Append(',');\n if (_params.SerializeToLowerCaseNames)\n WritePair(key.ToLowerInvariant(), nameValueCollection[key]);\n else\n WritePair(key, nameValueCollection[key]);\n pendingSeparator = true;\n }\n }\n _output.Append('}');\n }\n\n private void WriteSD(StringDictionary stringDictionary)\n {\n _output.Append('{');\n\n bool pendingSeparator = false;\n\n foreach (DictionaryEntry entry in stringDictionary)\n {\n if (_params.SerializeNullValues == false && (entry.Value == null))\n {\n }\n else\n {\n if (pendingSeparator) _output.Append(',');\n\n string k = (string)entry.Key;\n if (_params.SerializeToLowerCaseNames)\n WritePair(k.ToLowerInvariant(), entry.Value);\n else\n WritePair(k, entry.Value);\n pendingSeparator = true;\n }\n }\n _output.Append('}');\n }\n\n private void WriteCustom(object obj)\n {\n Reflection.Serialize s;\n Reflection.Instance._customSerializer.TryGetValue(obj.GetType(), out s);\n WriteStringFast(s(obj));\n }\n\n private void WriteEnum(Enum e)\n {\n // FEATURE : optimize enum write\n if (_params.UseValuesOfEnums)\n WriteValue(Convert.ToInt32(e));\n else\n WriteStringFast(e.ToString());\n }\n\n private void WriteGuid(Guid g)\n {\n if (_params.UseFastGuid == false)\n WriteStringFast(g.ToString());\n else\n WriteBytes(g.ToByteArray());\n }\n\n private void WriteBytes(byte[] bytes)\n {\n#if !SILVERLIGHT\n WriteStringFast(Convert.ToBase64String(bytes, 0, bytes.Length, Base64FormattingOptions.None));\n#else\n WriteStringFast(Convert.ToBase64String(bytes, 0, bytes.Length));\n#endif\n }\n\n private void WriteDateTime(DateTime dateTime)\n {\n // datetime format standard : yyyy-MM-dd HH:mm:ss\n DateTime dt = dateTime;\n if (_params.UseUTCDateTime)\n dt = dateTime.ToUniversalTime();\n\n write_date_value(dt);\n\n if (_params.DateTimeMilliseconds)\n {\n _output.Append('.');\n _output.Append(dt.Millisecond.ToString(\"000\", NumberFormatInfo.InvariantInfo));\n }\n\n if (_params.UseUTCDateTime)\n _output.Append('Z');\n\n _output.Append('\\\"');\n }\n\n private void write_date_value(DateTime dt)\n {\n _output.Append('\\\"');\n _output.Append(dt.Year.ToString(\"0000\", NumberFormatInfo.InvariantInfo));\n _output.Append('-');\n _output.Append(dt.Month.ToString(\"00\", NumberFormatInfo.InvariantInfo));\n _output.Append('-');\n _output.Append(dt.Day.ToString(\"00\", NumberFormatInfo.InvariantInfo));\n _output.Append('T'); // strict ISO date compliance \n _output.Append(dt.Hour.ToString(\"00\", NumberFormatInfo.InvariantInfo));\n _output.Append(':');\n _output.Append(dt.Minute.ToString(\"00\", NumberFormatInfo.InvariantInfo));\n _output.Append(':');\n _output.Append(dt.Second.ToString(\"00\", NumberFormatInfo.InvariantInfo));\n }\n\n#if !SILVERLIGHT\n private DatasetSchema GetSchema(DataTable ds)\n {\n if (ds == null) return null;\n\n DatasetSchema m = new DatasetSchema();\n m.Info = new List();\n m.Name = ds.TableName;\n\n foreach (DataColumn c in ds.Columns)\n {\n m.Info.Add(ds.TableName);\n m.Info.Add(c.ColumnName);\n m.Info.Add(c.DataType.ToString());\n }\n // FEATURE : serialize relations and constraints here\n\n return m;\n }\n\n private DatasetSchema GetSchema(DataSet ds)\n {\n if (ds == null) return null;\n\n DatasetSchema m = new DatasetSchema();\n m.Info = new List();\n m.Name = ds.DataSetName;\n\n foreach (DataTable t in ds.Tables)\n {\n foreach (DataColumn c in t.Columns)\n {\n m.Info.Add(t.TableName);\n m.Info.Add(c.ColumnName);\n m.Info.Add(c.DataType.ToString());\n }\n }\n // FEATURE : serialize relations and constraints here\n\n return m;\n }\n\n private string GetXmlSchema(DataTable dt)\n {\n using (var writer = new StringWriter())\n {\n dt.WriteXmlSchema(writer);\n return dt.ToString();\n }\n }\n\n private void WriteDataset(DataSet ds)\n {\n _output.Append('{');\n if (_params.UseExtensions)\n {\n WritePair(\"$schema\", _params.UseOptimizedDatasetSchema ? (object)GetSchema(ds) : ds.GetXmlSchema());\n _output.Append(',');\n }\n bool tablesep = false;\n foreach (DataTable table in ds.Tables)\n {\n if (tablesep) _output.Append(',');\n tablesep = true;\n WriteDataTableData(table);\n }\n // end dataset\n _output.Append('}');\n }\n\n private void WriteDataTableData(DataTable table)\n {\n _output.Append('\\\"');\n _output.Append(table.TableName);\n _output.Append(\"\\\":[\");\n DataColumnCollection cols = table.Columns;\n bool rowseparator = false;\n foreach (DataRow row in table.Rows)\n {\n if (rowseparator) _output.Append(',');\n rowseparator = true;\n _output.Append('[');\n\n bool pendingSeperator = false;\n foreach (DataColumn column in cols)\n {\n if (pendingSeperator) _output.Append(',');\n WriteValue(row[column]);\n pendingSeperator = true;\n }\n _output.Append(']');\n }\n\n _output.Append(']');\n }\n\n void WriteDataTable(DataTable dt)\n {\n this._output.Append('{');\n if (_params.UseExtensions)\n {\n this.WritePair(\"$schema\", _params.UseOptimizedDatasetSchema ? (object)this.GetSchema(dt) : this.GetXmlSchema(dt));\n this._output.Append(',');\n }\n\n WriteDataTableData(dt);\n\n // end datatable\n this._output.Append('}');\n }\n#endif\n\n bool _TypesWritten = false;\n private void WriteObject(object obj)\n {\n int i = 0;\n if (_cirobj.TryGetValue(obj, out i) == false)\n _cirobj.Add(obj, _cirobj.Count + 1);\n else\n {\n if (_current_depth > 0 && _params.InlineCircularReferences == false)\n {\n //_circular = true;\n _output.Append(\"{\\\"$i\\\":\");\n _output.Append(i.ToString());\n _output.Append(\"}\");\n return;\n }\n }\n if (_params.UsingGlobalTypes == false)\n _output.Append('{');\n else\n {\n if (_TypesWritten == false)\n {\n _output.Append('{');\n _before = _output.Length;\n //_output = new StringBuilder();\n }\n else\n _output.Append('{');\n }\n _TypesWritten = true;\n _current_depth++;\n if (_current_depth > _MAX_DEPTH)\n throw new Exception(\"Serializer encountered maximum depth of \" + _MAX_DEPTH);\n\n\n Dictionary map = new Dictionary();\n Type t = obj.GetType();\n bool append = false;\n if (_params.UseExtensions)\n {\n if (_params.UsingGlobalTypes == false)\n WritePairFast(\"$type\", Reflection.Instance.GetTypeAssemblyName(t));\n else\n {\n int dt = 0;\n string ct = Reflection.Instance.GetTypeAssemblyName(t);\n if (_globalTypes.TryGetValue(ct, out dt) == false)\n {\n dt = _globalTypes.Count + 1;\n _globalTypes.Add(ct, dt);\n }\n WritePairFast(\"$type\", dt.ToString());\n }\n append = true;\n }\n\n Getters[] g = Reflection.Instance.GetGetters(t, /*_params.ShowReadOnlyProperties,*/ _params.IgnoreAttributes);\n int c = g.Length;\n for (int ii = 0; ii < c; ii++)\n {\n var p = g[ii];\n if (_params.ShowReadOnlyProperties == false && p.ReadOnly)\n continue;\n object o = p.Getter(obj);\n if (_params.SerializeNullValues == false && (o == null || o is DBNull))\n {\n //append = false;\n }\n else\n {\n if (append)\n _output.Append(',');\n if (p.memberName != null)\n WritePair(p.memberName, o);\n else if (_params.SerializeToLowerCaseNames)\n WritePair(p.lcName, o);\n else\n WritePair(p.Name, o);\n if (o != null && _params.UseExtensions)\n {\n Type tt = o.GetType();\n if (tt == typeof(System.Object))\n map.Add(p.Name, tt.ToString());\n }\n append = true;\n }\n }\n if (map.Count > 0 && _params.UseExtensions)\n {\n _output.Append(\",\\\"$map\\\":\");\n WriteStringDictionary(map);\n }\n _output.Append('}');\n _current_depth--;\n }\n\n private void WritePairFast(string name, string value)\n {\n WriteStringFast(name);\n\n _output.Append(':');\n\n WriteStringFast(value);\n }\n\n private void WritePair(string name, object value)\n {\n WriteString(name);\n\n _output.Append(':');\n\n WriteValue(value);\n }\n\n private void WriteArray(IEnumerable array)\n {\n _output.Append('[');\n\n bool pendingSeperator = false;\n\n foreach (object obj in array)\n {\n if (pendingSeperator) _output.Append(',');\n\n WriteValue(obj);\n\n pendingSeperator = true;\n }\n _output.Append(']');\n }\n\n private void WriteStringDictionary(IDictionary dic)\n {\n _output.Append('{');\n\n bool pendingSeparator = false;\n\n foreach (DictionaryEntry entry in dic)\n {\n if (_params.SerializeNullValues == false && (entry.Value == null))\n {\n }\n else\n {\n if (pendingSeparator) _output.Append(',');\n\n string k = (string)entry.Key;\n if (_params.SerializeToLowerCaseNames)\n WritePair(k.ToLowerInvariant(), entry.Value);\n else\n WritePair(k, entry.Value);\n pendingSeparator = true;\n }\n }\n _output.Append('}');\n }\n\n private void WriteStringDictionary(IEnumerable> dic)\n {\n _output.Append('{');\n bool pendingSeparator = false;\n foreach (KeyValuePair entry in dic)\n {\n if (_params.SerializeNullValues == false && (entry.Value == null))\n {\n }\n else\n {\n if (pendingSeparator) _output.Append(',');\n string k = entry.Key;\n\n if (_params.SerializeToLowerCaseNames)\n WritePair(k.ToLowerInvariant(), entry.Value);\n else\n WritePair(k, entry.Value);\n pendingSeparator = true;\n }\n }\n _output.Append('}');\n }\n\n private void WriteDictionary(IDictionary dic)\n {\n _output.Append('[');\n\n bool pendingSeparator = false;\n\n foreach (DictionaryEntry entry in dic)\n {\n if (pendingSeparator) _output.Append(',');\n _output.Append('{');\n WritePair(\"k\", entry.Key);\n _output.Append(\",\");\n WritePair(\"v\", entry.Value);\n _output.Append('}');\n\n pendingSeparator = true;\n }\n _output.Append(']');\n }\n\n private void WriteStringFast(string s)\n {\n _output.Append('\\\"');\n _output.Append(s);\n _output.Append('\\\"');\n }\n\n private void WriteString(string s)\n {\n _output.Append('\\\"');\n\n int runIndex = -1;\n int l = s.Length;\n for (var index = 0; index < l; ++index)\n {\n var c = s[index];\n\n if (_useEscapedUnicode)\n {\n if (c >= ' ' && c < 128 && c != '\\\"' && c != '\\\\')\n {\n if (runIndex == -1)\n runIndex = index;\n\n continue;\n }\n }\n else\n {\n if (c != '\\t' && c != '\\n' && c != '\\r' && c != '\\\"' && c != '\\\\' && c!='\\0')// && c != ':' && c!=',')\n {\n if (runIndex == -1)\n runIndex = index;\n\n continue;\n }\n }\n\n if (runIndex != -1)\n {\n _output.Append(s, runIndex, index - runIndex);\n runIndex = -1;\n }\n\n switch (c)\n {\n case '\\t': _output.Append(\"\\\\t\"); break;\n case '\\r': _output.Append(\"\\\\r\"); break;\n case '\\n': _output.Append(\"\\\\n\"); break;\n case '\"':\n case '\\\\': _output.Append('\\\\'); _output.Append(c); break;\n case '\\0': _output.Append(\"\\\\u0000\"); break;\n default:\n if (_useEscapedUnicode)\n {\n _output.Append(\"\\\\u\");\n _output.Append(((int)c).ToString(\"X4\", NumberFormatInfo.InvariantInfo));\n }\n else\n _output.Append(c);\n\n break;\n }\n }\n\n if (runIndex != -1)\n _output.Append(s, runIndex, s.Length - runIndex);\n\n _output.Append('\\\"');\n }\n }\n}\n" }, { "path": "RaptorDB.Common/fastJSON/Reflection.cs", "content": "using System;\nusing System.Collections.Generic;\nusing System.Reflection.Emit;\nusing System.Reflection;\nusing System.Collections;\nusing System.Text;\nusing System.Runtime.Serialization;\n#if NET4\nusing System.Linq;\nusing RaptorDB.Common;\n#endif\n#if !SILVERLIGHT\nusing System.Data;\n#endif\nusing System.Collections.Specialized;\n\nnamespace fastJSON\n{\n public struct Getters\n {\n public string Name;\n public string lcName;\n public string memberName;\n public Reflection.GenericGetter Getter;\n public bool ReadOnly;\n }\n\n public enum myPropInfoType\n {\n Int,\n Long,\n String,\n Bool,\n DateTime,\n Enum,\n Guid,\n\n Array,\n ByteArray,\n Dictionary,\n StringKeyDictionary,\n NameValue,\n StringDictionary,\n#if !SILVERLIGHT\n Hashtable,\n DataSet,\n DataTable,\n#endif\n Custom,\n Unknown,\n }\n\n public class myPropInfo\n {\n public Type pt;\n public Type bt;\n public Type changeType;\n public Reflection.GenericSetter setter;\n public Reflection.GenericGetter getter;\n public Type[] GenericTypes;\n public string Name;\n#if NET4\n public string memberName;\n#endif\n public myPropInfoType Type;\n public bool CanWrite;\n\n public bool IsClass;\n public bool IsValueType;\n public bool IsGenericType;\n public bool IsStruct;\n public bool IsInterface;\n }\n\n public sealed class Reflection\n {\n // Singleton pattern 4 from : http://csharpindepth.com/articles/general/singleton.aspx\n private static readonly Reflection instance = new Reflection();\n // Explicit static constructor to tell C# compiler\n // not to mark type as beforefieldinit\n static Reflection()\n {\n }\n private Reflection()\n {\n }\n public static Reflection Instance { get { return instance; } }\n\n public static bool RDBMode = false;\n\n public delegate string Serialize(object data);\n public delegate object Deserialize(string data);\n\n public delegate object GenericSetter(object target, object value);\n public delegate object GenericGetter(object obj);\n private delegate object CreateObject();\n private delegate object CreateList(int capacity);\n\n private SafeDictionary _tyname = new SafeDictionary(10);\n private SafeDictionary _typecache = new SafeDictionary(10);\n private SafeDictionary _constrcache = new SafeDictionary(10);\n private SafeDictionary _conlistcache = new SafeDictionary(10);\n private SafeDictionary _getterscache = new SafeDictionary(10);\n private SafeDictionary> _propertycache = new SafeDictionary>(10);\n private SafeDictionary _genericTypes = new SafeDictionary(10);\n private SafeDictionary _genericTypeDef = new SafeDictionary(10);\n private static SafeDictionary _opCodes;\n private static List _blacklistTypes = new List()\n {\n \"system.configuration.install.assemblyinstaller\",\n \"system.activities.presentation.workflowdesigner\",\n \"system.windows.resourcedictionary\",\n \"system.windows.data.objectdataprovider\",\n \"system.windows.forms.bindingsource\",\n \"microsoft.exchange.management.systemmanager.winforms.exchangesettingsprovider\"\n };\n\n private static bool TryGetOpCode(short code, out OpCode opCode)\n {\n if (_opCodes != null)\n return _opCodes.TryGetValue(code, out opCode);\n var dict = new SafeDictionary();\n foreach (var fi in typeof(OpCodes).GetFields(BindingFlags.Public | BindingFlags.Static))\n {\n if (!typeof(OpCode).IsAssignableFrom(fi.FieldType)) continue;\n var innerOpCode = (OpCode)fi.GetValue(null);\n if (innerOpCode.OpCodeType != OpCodeType.Nternal)\n dict.Add(innerOpCode.Value, innerOpCode);\n }\n _opCodes = dict;\n return _opCodes.TryGetValue(code, out opCode);\n }\n\n #region bjson custom types\n //internal UnicodeEncoding unicode = new UnicodeEncoding();\n private static UTF8Encoding utf8 = new UTF8Encoding();\n\n // TODO : optimize utf8 \n public static byte[] UTF8GetBytes(string str)\n {\n return utf8.GetBytes(str);\n }\n\n public static string UTF8GetString(byte[] bytes, int offset, int len)\n {\n return utf8.GetString(bytes, offset, len);\n }\n\n public unsafe static byte[] UnicodeGetBytes(string str)\n {\n int len = str.Length * 2;\n byte[] b = new byte[len];\n fixed (void* ptr = str)\n {\n System.Runtime.InteropServices.Marshal.Copy(new IntPtr(ptr), b, 0, len);\n }\n return b;\n }\n\n public static string UnicodeGetString(byte[] b)\n {\n return UnicodeGetString(b, 0, b.Length);\n }\n\n public unsafe static string UnicodeGetString(byte[] bytes, int offset, int buflen)\n {\n string str = \"\";\n fixed (byte* bptr = bytes)\n {\n char* cptr = (char*)(bptr + offset);\n str = new string(cptr, 0, buflen / 2);\n }\n return str;\n }\n #endregion\n\n #region json custom types\n // JSON custom\n internal SafeDictionary _customSerializer = new SafeDictionary();\n internal SafeDictionary _customDeserializer = new SafeDictionary();\n\n internal object CreateCustom(string v, Type type)\n {\n Deserialize d;\n _customDeserializer.TryGetValue(type, out d);\n return d(v);\n }\n\n internal void RegisterCustomType(Type type, Serialize serializer, Deserialize deserializer)\n {\n if (type != null && serializer != null && deserializer != null)\n {\n _customSerializer.Add(type, serializer);\n _customDeserializer.Add(type, deserializer);\n // reset property cache\n Instance.ResetPropertyCache();\n }\n }\n\n internal bool IsTypeRegistered(Type t)\n {\n if (_customSerializer.Count() == 0)\n return false;\n Serialize s;\n return _customSerializer.TryGetValue(t, out s);\n }\n #endregion\n\n public Type GetGenericTypeDefinition(Type t)\n {\n Type tt = null;\n if (_genericTypeDef.TryGetValue(t, out tt))\n return tt;\n else\n {\n tt = t.GetGenericTypeDefinition();\n _genericTypeDef.Add(t, tt);\n return tt;\n }\n }\n\n public Type[] GetGenericArguments(Type t)\n {\n Type[] tt = null;\n if (_genericTypes.TryGetValue(t, out tt))\n return tt;\n else\n {\n tt = t.GetGenericArguments();\n _genericTypes.Add(t, tt);\n return tt;\n }\n }\n\n public Dictionary Getproperties(Type type, string typename, bool ShowReadOnlyProperties)\n {\n Dictionary sd = null;\n if (_propertycache.TryGetValue(typename, out sd))\n {\n return sd;\n }\n else\n {\n sd = new Dictionary(10);\n var bf = BindingFlags.Public | BindingFlags.Instance | BindingFlags.Static;\n PropertyInfo[] pr = type.GetProperties(bf);\n foreach (PropertyInfo p in pr)\n {\n if (p.GetIndexParameters().Length > 0)// Property is an indexer\n continue;\n\n myPropInfo d = CreateMyProp(p.PropertyType, p.Name);\n d.setter = Reflection.CreateSetMethod(type, p, ShowReadOnlyProperties);\n if (d.setter != null)\n d.CanWrite = true;\n d.getter = Reflection.CreateGetMethod(type, p);\n#if NET4\n var att = p.GetCustomAttributes(true);\n foreach (var at in att)\n {\n if (at is DataMemberAttribute)\n {\n var dm = (DataMemberAttribute)at;\n if (dm.Name != \"\")\n d.memberName = dm.Name;\n }\n }\n if (d.memberName != null)\n sd.Add(d.memberName, d);\n else\n#endif\n sd.Add(p.Name.ToLowerInvariant(), d);\n }\n FieldInfo[] fi = type.GetFields(bf);\n foreach (FieldInfo f in fi)\n {\n myPropInfo d = CreateMyProp(f.FieldType, f.Name);\n if (f.IsLiteral == false)\n {\n if (f.IsInitOnly == false)\n d.setter = Reflection.CreateSetField(type, f);\n if (d.setter != null)\n d.CanWrite = true;\n d.getter = Reflection.CreateGetField(type, f);\n#if NET4\n var att = f.GetCustomAttributes(true);\n foreach (var at in att)\n {\n if (at is DataMemberAttribute)\n {\n var dm = (DataMemberAttribute)at;\n if (dm.Name != \"\")\n d.memberName = dm.Name;\n }\n }\n if (d.memberName != null)\n sd.Add(d.memberName, d);\n else\n#endif\n sd.Add(f.Name.ToLowerInvariant(), d);\n }\n }\n\n _propertycache.Add(typename, sd);\n return sd;\n }\n }\n\n private myPropInfo CreateMyProp(Type t, string name)\n {\n myPropInfo d = new myPropInfo();\n myPropInfoType d_type = myPropInfoType.Unknown;\n\n if (t == typeof(int) || t == typeof(int?)) d_type = myPropInfoType.Int;\n else if (t == typeof(long) || t == typeof(long?)) d_type = myPropInfoType.Long;\n else if (t == typeof(string)) d_type = myPropInfoType.String;\n else if (t == typeof(bool) || t == typeof(bool?)) d_type = myPropInfoType.Bool;\n else if (t == typeof(DateTime) || t == typeof(DateTime?)) d_type = myPropInfoType.DateTime;\n else if (t.IsEnum) d_type = myPropInfoType.Enum;\n else if (t == typeof(Guid) || t == typeof(Guid?)) d_type = myPropInfoType.Guid;\n else if (t == typeof(StringDictionary)) d_type = myPropInfoType.StringDictionary;\n else if (t == typeof(NameValueCollection)) d_type = myPropInfoType.NameValue;\n else if (t.IsArray)\n {\n d.bt = t.GetElementType();\n if (t == typeof(byte[]))\n d_type = myPropInfoType.ByteArray;\n else\n d_type = myPropInfoType.Array;\n }\n else if (t.Name.Contains(\"Dictionary\"))\n {\n d.GenericTypes = Reflection.Instance.GetGenericArguments(t);\n if (d.GenericTypes.Length > 0 && d.GenericTypes[0] == typeof(string))\n d_type = myPropInfoType.StringKeyDictionary;\n else\n d_type = myPropInfoType.Dictionary;\n }\n#if !SILVERLIGHT\n else if (t == typeof(Hashtable)) d_type = myPropInfoType.Hashtable;\n else if (t == typeof(DataSet)) d_type = myPropInfoType.DataSet;\n else if (t == typeof(DataTable)) d_type = myPropInfoType.DataTable;\n#endif\n else if (IsTypeRegistered(t))\n d_type = myPropInfoType.Custom;\n\n if (t.IsValueType && !t.IsPrimitive && !t.IsEnum && t != typeof(decimal))\n d.IsStruct = true;\n\n d.IsInterface = t.IsInterface;\n d.IsClass = t.IsClass;\n d.IsValueType = t.IsValueType;\n if (t.IsGenericType)\n {\n d.IsGenericType = true;\n d.bt = Reflection.Instance.GetGenericArguments(t)[0];\n }\n\n d.pt = t;\n d.Name = name;\n d.changeType = GetChangeType(t);\n d.Type = d_type;\n\n return d;\n }\n\n private Type GetChangeType(Type conversionType)\n {\n if (conversionType.IsGenericType && conversionType.GetGenericTypeDefinition().Equals(typeof(Nullable<>)))\n return Reflection.Instance.GetGenericArguments(conversionType)[0];\n\n return conversionType;\n }\n\n #region [ PROPERTY GET SET ]\n\n public string GetTypeAssemblyName(Type t)\n {\n string val = \"\";\n if (_tyname.TryGetValue(t, out val))\n return val;\n else\n {\n string s = t.AssemblyQualifiedName;\n _tyname.Add(t, s);\n return s;\n }\n }\n\n internal Type GetTypeFromCache(string typename, bool blacklistChecking)\n {\n Type val = null;\n if (_typecache.TryGetValue(typename, out val))\n return val;\n else\n {\n // check for BLACK LIST types -> more secure when using $type\n if (blacklistChecking)\n {\n var tn = typename.Trim().ToLowerInvariant();\n foreach (var s in _blacklistTypes)\n if (tn.StartsWith(s, StringComparison.Ordinal))\n throw new Exception(\"Black list type encountered, possible attack vector when using $type : \" + typename);\n }\n Type t = Type.GetType(typename);\n#if NET4\n if (RDBMode)\n {\n if (t == null) // RaptorDB : loading runtime assemblies\n {\n t = Type.GetType(typename, (name) =>\n {\n return AppDomain.CurrentDomain.GetAssemblies().Where(z => z.FullName == name.FullName).FirstOrDefault();\n }, null, true);\n }\n }\n#endif\n _typecache.Add(typename, t);\n return t;\n }\n }\n\n internal object FastCreateList(Type objtype, int capacity)\n {\n try\n {\n int count = 10;\n if (capacity > 10)\n count = capacity;\n CreateList c = null;\n if (_conlistcache.TryGetValue(objtype, out c))\n {\n if (c != null) // kludge : non capacity lists\n return c(count);\n else\n return FastCreateInstance(objtype);\n }\n else\n {\n var cinfo = objtype.GetConstructor(new Type[] { typeof(int) });\n if (cinfo != null)\n {\n DynamicMethod dynMethod = new DynamicMethod(\"_fcil\", objtype, new Type[] { typeof(int) }, true);\n ILGenerator ilGen = dynMethod.GetILGenerator();\n ilGen.Emit(OpCodes.Ldarg_0);\n ilGen.Emit(OpCodes.Newobj, objtype.GetConstructor(new Type[] { typeof(int) }));\n ilGen.Emit(OpCodes.Ret);\n c = (CreateList)dynMethod.CreateDelegate(typeof(CreateList));\n _conlistcache.Add(objtype, c);\n return c(count);\n }\n else\n {\n _conlistcache.Add(objtype, null);// kludge : non capacity lists\n return FastCreateInstance(objtype);\n }\n }\n }\n catch (Exception exc)\n {\n throw new Exception(string.Format(\"Failed to fast create instance for type '{0}' from assembly '{1}'\",\n objtype.FullName, objtype.AssemblyQualifiedName), exc);\n }\n }\n\n internal object FastCreateInstance(Type objtype)\n {\n try\n {\n CreateObject c = null;\n if (_constrcache.TryGetValue(objtype, out c))\n {\n return c();\n }\n else\n {\n if (objtype.IsClass)\n {\n DynamicMethod dynMethod = new DynamicMethod(\"_fcic\", objtype, null, true);\n ILGenerator ilGen = dynMethod.GetILGenerator();\n ilGen.Emit(OpCodes.Newobj, objtype.GetConstructor(Type.EmptyTypes));\n ilGen.Emit(OpCodes.Ret);\n c = (CreateObject)dynMethod.CreateDelegate(typeof(CreateObject));\n _constrcache.Add(objtype, c);\n }\n else // structs\n {\n DynamicMethod dynMethod = new DynamicMethod(\"_fcis\", typeof(object), null, true);\n ILGenerator ilGen = dynMethod.GetILGenerator();\n var lv = ilGen.DeclareLocal(objtype);\n ilGen.Emit(OpCodes.Ldloca_S, lv);\n ilGen.Emit(OpCodes.Initobj, objtype);\n ilGen.Emit(OpCodes.Ldloc_0);\n ilGen.Emit(OpCodes.Box, objtype);\n ilGen.Emit(OpCodes.Ret);\n c = (CreateObject)dynMethod.CreateDelegate(typeof(CreateObject));\n _constrcache.Add(objtype, c);\n }\n return c();\n }\n }\n catch (Exception exc)\n {\n throw new Exception(string.Format(\"Failed to fast create instance for type '{0}' from assembly '{1}'\",\n objtype.FullName, objtype.AssemblyQualifiedName), exc);\n }\n }\n\n internal static GenericSetter CreateSetField(Type type, FieldInfo fieldInfo)\n {\n Type[] arguments = new Type[2];\n arguments[0] = arguments[1] = typeof(object);\n\n DynamicMethod dynamicSet = new DynamicMethod(\"_csf\", typeof(object), arguments, type, true);\n\n ILGenerator il = dynamicSet.GetILGenerator();\n\n if (!type.IsClass) // structs\n {\n var lv = il.DeclareLocal(type);\n il.Emit(OpCodes.Ldarg_0);\n il.Emit(OpCodes.Unbox_Any, type);\n il.Emit(OpCodes.Stloc_0);\n il.Emit(OpCodes.Ldloca_S, lv);\n il.Emit(OpCodes.Ldarg_1);\n if (fieldInfo.FieldType.IsClass)\n il.Emit(OpCodes.Castclass, fieldInfo.FieldType);\n else\n il.Emit(OpCodes.Unbox_Any, fieldInfo.FieldType);\n il.Emit(OpCodes.Stfld, fieldInfo);\n il.Emit(OpCodes.Ldloc_0);\n il.Emit(OpCodes.Box, type);\n il.Emit(OpCodes.Ret);\n }\n else\n {\n il.Emit(OpCodes.Ldarg_0);\n il.Emit(OpCodes.Ldarg_1);\n if (fieldInfo.FieldType.IsValueType)\n il.Emit(OpCodes.Unbox_Any, fieldInfo.FieldType);\n il.Emit(OpCodes.Stfld, fieldInfo);\n il.Emit(OpCodes.Ldarg_0);\n il.Emit(OpCodes.Ret);\n }\n return (GenericSetter)dynamicSet.CreateDelegate(typeof(GenericSetter));\n }\n\n internal static FieldInfo GetGetterBackingField(PropertyInfo autoProperty)\n {\n var getMethod = autoProperty.GetGetMethod();\n // Restrict operation to auto properties to avoid risking errors if a getter does not contain exactly one field read instruction (such as with calculated properties).\n if (!getMethod.IsDefined(typeof(System.Runtime.CompilerServices.CompilerGeneratedAttribute), false)) return null;\n\n var byteCode = getMethod.GetMethodBody()?.GetILAsByteArray() ?? new byte[0];\n //var byteCode = getMethod.GetMethodBody().GetILAsByteArray();\n int pos = 0;\n // Find the first LdFld instruction and parse its operand to a FieldInfo object.\n while (pos < byteCode.Length)\n {\n // Read and parse the OpCode (it can be 1 or 2 bytes in size).\n byte code = byteCode[pos++];\n if (!(TryGetOpCode(code, out var opCode) || pos < byteCode.Length && TryGetOpCode((short)(code * 0x100 + byteCode[pos++]), out opCode)))\n throw new NotSupportedException(\"Unknown IL code detected.\");\n // If it is a LdFld, read its operand, parse it to a FieldInfo and return it.\n if (opCode == OpCodes.Ldfld && opCode.OperandType == OperandType.InlineField && pos + sizeof(int) <= byteCode.Length)\n {\n return getMethod.Module.ResolveMember(BitConverter.ToInt32(byteCode, pos), getMethod.DeclaringType?.GetGenericArguments(), null) as FieldInfo;\n }\n // Otherwise, set the current position to the start of the next instruction, if any (we need to know how much bytes are used by operands).\n pos += opCode.OperandType == OperandType.InlineNone\n ? 0\n : opCode.OperandType == OperandType.ShortInlineBrTarget ||\n opCode.OperandType == OperandType.ShortInlineI ||\n opCode.OperandType == OperandType.ShortInlineVar\n ? 1\n : opCode.OperandType == OperandType.InlineVar\n ? 2\n : opCode.OperandType == OperandType.InlineI8 ||\n opCode.OperandType == OperandType.InlineR\n ? 8\n : opCode.OperandType == OperandType.InlineSwitch\n ? 4 * (BitConverter.ToInt32(byteCode, pos) + 1)\n : 4;\n }\n return null;\n }\n\n internal static GenericSetter CreateSetMethod(Type type, PropertyInfo propertyInfo, bool ShowReadOnlyProperties)\n {\n MethodInfo setMethod = propertyInfo.GetSetMethod(ShowReadOnlyProperties);\n if (setMethod == null)\n {\n if (!ShowReadOnlyProperties) return null;\n // If the property has no setter and it is an auto property, try and create a setter for its backing field instead \n var fld = GetGetterBackingField(propertyInfo);\n return fld != null ? CreateSetField(type, fld) : null;\n }\n\n Type[] arguments = new Type[2];\n arguments[0] = arguments[1] = typeof(object);\n\n DynamicMethod setter = new DynamicMethod(\"_csm\", typeof(object), arguments, true);// !setMethod.IsPublic); // fix: skipverify\n ILGenerator il = setter.GetILGenerator();\n\n if (!type.IsClass) // structs\n {\n var lv = il.DeclareLocal(type);\n il.Emit(OpCodes.Ldarg_0);\n il.Emit(OpCodes.Unbox_Any, type);\n il.Emit(OpCodes.Stloc_0);\n il.Emit(OpCodes.Ldloca_S, lv);\n il.Emit(OpCodes.Ldarg_1);\n if (propertyInfo.PropertyType.IsClass)\n il.Emit(OpCodes.Castclass, propertyInfo.PropertyType);\n else\n il.Emit(OpCodes.Unbox_Any, propertyInfo.PropertyType);\n il.EmitCall(OpCodes.Call, setMethod, null);\n il.Emit(OpCodes.Ldloc_0);\n il.Emit(OpCodes.Box, type);\n }\n else\n {\n if (!setMethod.IsStatic)\n {\n il.Emit(OpCodes.Ldarg_0);\n il.Emit(OpCodes.Castclass, propertyInfo.DeclaringType);\n il.Emit(OpCodes.Ldarg_1);\n if (propertyInfo.PropertyType.IsClass)\n il.Emit(OpCodes.Castclass, propertyInfo.PropertyType);\n else\n il.Emit(OpCodes.Unbox_Any, propertyInfo.PropertyType);\n il.EmitCall(OpCodes.Callvirt, setMethod, null);\n il.Emit(OpCodes.Ldarg_0);\n }\n else\n {\n il.Emit(OpCodes.Ldarg_0);\n il.Emit(OpCodes.Ldarg_1);\n if (propertyInfo.PropertyType.IsClass)\n il.Emit(OpCodes.Castclass, propertyInfo.PropertyType);\n else\n il.Emit(OpCodes.Unbox_Any, propertyInfo.PropertyType);\n il.Emit(OpCodes.Call, setMethod);\n }\n }\n\n il.Emit(OpCodes.Ret);\n\n return (GenericSetter)setter.CreateDelegate(typeof(GenericSetter));\n }\n\n internal static GenericGetter CreateGetField(Type type, FieldInfo fieldInfo)\n {\n DynamicMethod dynamicGet = new DynamicMethod(\"_cgf\", typeof(object), new Type[] { typeof(object) }, type, true);\n\n ILGenerator il = dynamicGet.GetILGenerator();\n\n if (!type.IsClass) // structs\n {\n var lv = il.DeclareLocal(type);\n il.Emit(OpCodes.Ldarg_0);\n il.Emit(OpCodes.Unbox_Any, type);\n il.Emit(OpCodes.Stloc_0);\n il.Emit(OpCodes.Ldloca_S, lv);\n il.Emit(OpCodes.Ldfld, fieldInfo);\n if (fieldInfo.FieldType.IsValueType)\n il.Emit(OpCodes.Box, fieldInfo.FieldType);\n }\n else\n {\n il.Emit(OpCodes.Ldarg_0);\n il.Emit(OpCodes.Ldfld, fieldInfo);\n if (fieldInfo.FieldType.IsValueType)\n il.Emit(OpCodes.Box, fieldInfo.FieldType);\n }\n\n il.Emit(OpCodes.Ret);\n\n return (GenericGetter)dynamicGet.CreateDelegate(typeof(GenericGetter));\n }\n\n internal static GenericGetter CreateGetMethod(Type type, PropertyInfo propertyInfo)\n {\n MethodInfo getMethod = propertyInfo.GetGetMethod();\n if (getMethod == null)\n return null;\n\n DynamicMethod getter = new DynamicMethod(\"_cgm\", typeof(object), new Type[] { typeof(object) }, type, true);\n\n ILGenerator il = getter.GetILGenerator();\n\n if (!type.IsClass) // structs\n {\n var lv = il.DeclareLocal(type);\n il.Emit(OpCodes.Ldarg_0);\n il.Emit(OpCodes.Unbox_Any, type);\n il.Emit(OpCodes.Stloc_0);\n il.Emit(OpCodes.Ldloca_S, lv);\n il.EmitCall(OpCodes.Call, getMethod, null);\n if (propertyInfo.PropertyType.IsValueType)\n il.Emit(OpCodes.Box, propertyInfo.PropertyType);\n }\n else\n {\n if (!getMethod.IsStatic)\n {\n il.Emit(OpCodes.Ldarg_0);\n il.Emit(OpCodes.Castclass, propertyInfo.DeclaringType);\n il.EmitCall(OpCodes.Callvirt, getMethod, null);\n }\n else\n il.Emit(OpCodes.Call, getMethod);\n\n if (propertyInfo.PropertyType.IsValueType)\n il.Emit(OpCodes.Box, propertyInfo.PropertyType);\n }\n\n il.Emit(OpCodes.Ret);\n\n return (GenericGetter)getter.CreateDelegate(typeof(GenericGetter));\n }\n\n public Getters[] GetGetters(Type type, /*bool ShowReadOnlyProperties,*/ List IgnoreAttributes)\n {\n Getters[] val = null;\n if (_getterscache.TryGetValue(type, out val))\n return val;\n\n var bf = BindingFlags.Public | BindingFlags.Instance | BindingFlags.Static;\n //if (ShowReadOnlyProperties)\n // bf |= BindingFlags.NonPublic;\n PropertyInfo[] props = type.GetProperties(bf);\n List getters = new List();\n foreach (PropertyInfo p in props)\n {\n bool read_only = false;\n if (p.GetIndexParameters().Length > 0)\n {// Property is an indexer\n continue;\n }\n if (!p.CanWrite)// && (ShowReadOnlyProperties == false))//|| isAnonymous == false))\n read_only = true; //continue;\n if (IgnoreAttributes != null)\n {\n bool found = false;\n foreach (var ignoreAttr in IgnoreAttributes)\n {\n if (p.IsDefined(ignoreAttr, false))\n {\n found = true;\n break;\n }\n }\n if (found)\n continue;\n }\n string mName = null;\n#if NET4\n var att = p.GetCustomAttributes(true);\n foreach (var at in att)\n {\n if (at is DataMemberAttribute)\n {\n var dm = (DataMemberAttribute)at;\n if (dm.Name != \"\")\n {\n mName = dm.Name;\n }\n }\n }\n#endif\n GenericGetter g = CreateGetMethod(type, p);\n if (g != null)\n getters.Add(new Getters { Getter = g, Name = p.Name, lcName = p.Name.ToLowerInvariant(), memberName = mName, ReadOnly = read_only });\n }\n\n FieldInfo[] fi = type.GetFields(bf);\n foreach (var f in fi)\n {\n bool read_only = false;\n if (f.IsInitOnly) // && (ShowReadOnlyProperties == false))//|| isAnonymous == false))\n read_only = true;//continue;\n if (IgnoreAttributes != null)\n {\n bool found = false;\n foreach (var ignoreAttr in IgnoreAttributes)\n {\n if (f.IsDefined(ignoreAttr, false))\n {\n found = true;\n break;\n }\n }\n if (found)\n continue;\n }\n string mName = null;\n#if NET4\n var att = f.GetCustomAttributes(true);\n foreach (var at in att)\n {\n if (at is DataMemberAttribute)\n {\n var dm = (DataMemberAttribute)at;\n if (dm.Name != \"\")\n {\n mName = dm.Name;\n }\n }\n }\n#endif\n if (f.IsLiteral == false)\n {\n GenericGetter g = CreateGetField(type, f);\n if (g != null)\n getters.Add(new Getters { Getter = g, Name = f.Name, lcName = f.Name.ToLowerInvariant(), memberName = mName, ReadOnly = read_only });\n }\n }\n val = getters.ToArray();\n _getterscache.Add(type, val);\n return val;\n }\n\n //private static bool IsAnonymousType(Type type)\n //{\n // // may break in the future if compiler defined names change...\n // const string CS_ANONYMOUS_PREFIX = \"<>f__AnonymousType\";\n // const string VB_ANONYMOUS_PREFIX = \"VB$AnonymousType\";\n\n // if (type == null)\n // throw new ArgumentNullException(\"type\");\n\n // if (type.Name.StartsWith(CS_ANONYMOUS_PREFIX, StringComparison.Ordinal) || type.Name.StartsWith(VB_ANONYMOUS_PREFIX, StringComparison.Ordinal))\n // {\n // return type.IsDefined(typeof(CompilerGeneratedAttribute), false);\n // }\n\n // return false;\n //}\n #endregion\n\n internal void ResetPropertyCache()\n {\n _propertycache = new SafeDictionary>();\n }\n\n internal void ClearReflectionCache()\n {\n _tyname = new SafeDictionary(10);\n _typecache = new SafeDictionary(10);\n _constrcache = new SafeDictionary(10);\n _getterscache = new SafeDictionary(10);\n _propertycache = new SafeDictionary>(10);\n _genericTypes = new SafeDictionary(10);\n _genericTypeDef = new SafeDictionary(10);\n }\n }\n}\n" }, { "path": "RaptorDB.Common/fastJSON/dynamic.cs", "content": "#if NET4\nusing System;\nusing System.Collections;\nusing System.Collections.Generic;\nusing System.Dynamic;\nusing System.Linq;\n\nnamespace fastJSON\n{\n internal class DynamicJson : DynamicObject, IEnumerable\n {\n private IDictionary _dictionary { get; set; }\n private List _list { get; set; }\n\n public DynamicJson(string json)\n {\n var parse = fastJSON.JSON.Parse(json);\n\n if (parse is IDictionary)\n _dictionary = (IDictionary)parse;\n else\n _list = (List)parse;\n }\n\n private DynamicJson(object dictionary)\n {\n if (dictionary is IDictionary)\n _dictionary = (IDictionary)dictionary;\n }\n\n public override IEnumerable GetDynamicMemberNames()\n {\n return _dictionary.Keys.ToList();\n }\n\n public override bool TryGetIndex(GetIndexBinder binder, Object[] indexes, out Object result)\n {\n var index = indexes[0];\n if (index is int)\n {\n result = _list[(int) index];\n }\n else\n {\n result = _dictionary[(string) index];\n } \n if (result is IDictionary)\n result = new DynamicJson(result as IDictionary);\n return true;\n }\n\n public override bool TryGetMember(GetMemberBinder binder, out object result)\n {\n if (_dictionary.TryGetValue(binder.Name, out result) == false)\n if (_dictionary.TryGetValue(binder.Name.ToLowerInvariant(), out result) == false)\n return false;// throw new Exception(\"property not found \" + binder.Name);\n\n if (result is IDictionary)\n {\n result = new DynamicJson(result as IDictionary);\n }\n else if (result is List)\n {\n List list = new List();\n foreach (object item in (List)result)\n {\n if (item is IDictionary)\n list.Add(new DynamicJson(item as IDictionary));\n else\n list.Add(item);\n }\n result = list;\n }\n\n return _dictionary.ContainsKey(binder.Name);\n }\n\n IEnumerator IEnumerable.GetEnumerator()\n {\n foreach(var o in _list)\n {\n yield return new DynamicJson(o as IDictionary);\n }\n }\n }\n}\n#endif" }, { "path": "RaptorDBCore/RaptorDB/RaptorDB.csproj", "content": "\n \n netstandard2.0\n true\n ..\\..\\raptordb.snk\n \n\n \n netstandard2.0\n 3.3.19.1\n 3.3.19.1\n 3.3.19\n \n\n \n ..\\..\\output\n true\n true\n \n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n \n \n \n \n \n \n \n\n \n \n \n \n \n\n \n \n ResXFileCodeGenerator\n \n \n\n \n \n \n \n \n \n\n \n \n \n\n \n \n \n \n \n \n \n \n \n \n\n\n" }, { "path": "RaptorDBCore/RaptorDb.Common/RaptorDb.Common.csproj", "content": "\n\n \n netstandard2.0\n true\n ..\\..\\raptordb.snk\n 3.3.19.1\n 3.3.19.1\n 3.3.19\n \n\n \n true\n ..\\..\\output\\\n true\n TRACE;NET4\n \n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n \n \n \n \n\n \n \n \n \n\n\n" }, { "path": "RaptorDBCore/test/sample.cs", "content": "// ref : ..\\output\\raptordb.dll\n// ref : ..\\output\\raptordb.common.dll\n// ref : ..\\faker.dll\nusing System;\nusing System.Collections.Generic;\nusing System.Linq;\nusing System.Text;\nusing System.Threading.Tasks;\n\nusing RaptorDB;\nusing RaptorDB.Common;\n\nnamespace rdbtest\n{\n\n #region [ entities ]\n public class LineItem\n {\n public decimal QTY { get; set; }\n public string Product { get; set; }\n public decimal Price { get; set; }\n public decimal Discount { get; set; }\n }\n\n public class SalesInvoice\n {\n public SalesInvoice()\n {\n ID = Guid.NewGuid();\n }\n\n public Guid ID { get; set; }\n public string CustomerName { get; set; }\n public string NoCase { get; set; }\n public string Address { get; set; }\n public List Items { get; set; }\n public DateTime Date { get; set; }\n public int Serial { get; set; }\n public byte Status { get; set; }\n public bool Approved { get; set; }\n }\n #endregion\n\n #region [ view definition ]\n public class SalesInvoiceViewRowSchema : RDBSchema\n {\n public string CustomerName;\n public string NoCase;\n public DateTime Date;\n public string Address;\n public int Serial;\n public byte Status;\n public bool? Approved;\n }\n\n [RegisterView]\n public class SalesInvoiceView : View\n {\n public SalesInvoiceView()\n {\n this.Name = \"SalesInvoice\";\n this.Description = \"A primary view for SalesInvoices\";\n this.isPrimaryList = true;\n this.isActive = true;\n this.BackgroundIndexing = true;\n this.Version = 3;\n\n this.Schema = typeof(SalesInvoiceViewRowSchema);\n this.FullTextColumns.Add(\"CustomerName\");\n this.FullTextColumns.Add(\"Address\");\n\n\n this.Mapper = (api, docid, doc) =>\n {\n //if (doc.Status == 0)\n // return;\n \n api.EmitObject(docid, doc);\n };\n }\n }\n #endregion\n\n class Program\n {\n static RaptorDB.RaptorDB rdb; // 1 instance\n\n static void Main(string[] args)\n {\n rdb = RaptorDB.RaptorDB.Open(\"data\"); // a \"data\" folder beside the executable\n RaptorDB.Global.RequirePrimaryView = false;\n\n Console.WriteLine(\"Registering views..\");\n rdb.RegisterView(new SalesInvoiceView());\n\n DoWork();\n\n Console.WriteLine(\"press any key...\");\n Console.ReadKey();\n Console.WriteLine(\"\\r\\nShutting down...\");\n rdb.Shutdown(); // explicit shutdown\n }\n\n static void DoWork()\n {\n long c = rdb.DocumentCount();\n if (c > 0) // not the first time running\n {\n var result = rdb.Query(x => x.Serial < 100);\n // show the rows\n Console.WriteLine(fastJSON.JSON.ToNiceJSON(result.Rows, new fastJSON.JSONParameters { UseExtensions = false, UseFastGuid = false }));\n // show the count\n Console.WriteLine(\"Query result count = \" + result.Count);\n return;\n }\n\n Console.Write(\"Inserting 100,000 documents...\");\n int count = 100000;\n\n for (int i = 0; i < count; i++)\n { \n var inv = CreateInvoice(i);\n\n // save here\n rdb.Save(inv.ID, inv);\n }\n\n Console.WriteLine(\"done.\");\n }\n\n static SalesInvoice CreateInvoice(int counter)\n {\n // new invoice\n var inv = new SalesInvoice()\n {\n Date = Faker.DateTimeFaker.BirthDay(),\n Serial = counter % 10000,\n CustomerName = Faker.NameFaker.Name(),\n NoCase = \"Me \" + counter % 10,\n Status = (byte)(counter % 4),\n Address = Faker.LocationFaker.Street(),\n Approved = counter % 100 == 0 ? true : false\n };\n // new line items\n inv.Items = new List();\n for (int k = 0; k < 5; k++)\n inv.Items.Add(new LineItem() { Product = \"prod \" + k, Discount = 0, Price = 10 + k, QTY = 1 + k });\n\n return inv;\n }\n }\n}" }, { "path": "RaptorDBCore/test/test.csproj", "content": "\n\n \n Exe\n netcoreapp2.0\n \n\n \n \n \n \n\n \n \n ..\\..\\Faker.dll\n \n \n\n\n" }, { "path": "RaptorDBServer/Installer.cs", "content": "using System.ComponentModel;\nusing System.Configuration.Install;\nusing System.ServiceProcess;\n\nnamespace RaptorDBServer\n{\n [RunInstaller(true)]\n public class CustomServiceInstaller : Installer\n {\n private ServiceProcessInstaller process;\n private ServiceInstaller service;\n\n public CustomServiceInstaller()\n {\n process = new ServiceProcessInstaller();\n\n process.Account = ServiceAccount.LocalSystem;\n\n service = new ServiceInstaller();\n service.ServiceName = Program.InstallServiceName;\n \n Installers.Add(process);\n Installers.Add(service);\n }\n\n protected override void OnBeforeInstall(System.Collections.IDictionary savedState)\n {\n Context.Parameters[\"assemblypath\"] = \"\\\"\" + this.GetType().Assembly.Location + \"\\\" -p \" + Program.Port + \" -f \\\"\" + Program.Path + \"\\\"\";\n base.OnBeforeInstall(savedState);\n }\n }\n}\n" }, { "path": "RaptorDBServer/Program.cs", "content": "using System;\nusing System.ServiceProcess;\nusing System.IO;\nusing System.Reflection;\nusing System.Configuration.Install;\n\nnamespace RaptorDBServer\n{\n static class Program\n {\n public static string InstallServiceName;\n public static int Port = 90;\n public static string Path = \"\";\n /// \n /// The main entry point for the application.\n /// \n static void Main(string[] args)\n {\n if (args.Length == 0)\n {\n Console.WriteLine(@\"\nRun with : \n -i install service\n -u uninstall service\n -n [default = RaptorDB]\n -p [default = 90]\n -f \n\");\n return;\n }\n\n string name = \"RaptorDB\";\n string path = Directory.GetCurrentDirectory();\n int port = 90;\n bool install = false;\n bool uninstall = false;\n\n for (int i = 0; i < args.Length; i++)\n {\n if (args[i].Trim() == \"-i\") install = true;\n if (args[i].Trim() == \"-u\") uninstall = true;\n if (args[i].Trim() == \"-p\") port = int.Parse(args[++i]);\n if (args[i].Trim() == \"-f\") path = args[++i].Trim();\n if (args[i].Trim() == \"-n\") name = \"RaptorDB - \" + args[++i].Trim();\n }\n\n InstallServiceName = name;\n Port = port;\n Path = path;\n\n if (install)\n {\n if (IsServiceInstalled(name))\n {\n Console.WriteLine();\n Console.WriteLine(\"Service exists : \" + name);\n return;\n }\n // Install service\n ManagedInstallerClass.InstallHelper(new string[] { Assembly.GetExecutingAssembly().Location });\n return;\n }\n else if (uninstall)\n {\n if (IsServiceInstalled(name) == false)\n return;\n // Uninstall service\n ManagedInstallerClass.InstallHelper(new string[] { \"/u\", Assembly.GetExecutingAssembly().Location });\n return;\n }\n\n if (Environment.UserInteractive == false)\n ServiceBase.Run(new Service1());\n else\n Dostart();\n }\n\n private static void Dostart()\n {\n var _raptor = new RaptorDB.RaptorDBServer(Port, Path);\n Console.WriteLine(\"Press Enter to shutdown...\");\n Console.ReadLine();\n _raptor.Shutdown();\n }\n\n private static bool IsServiceInstalled(string serviceName)\n {\n // Get a list of current services\n ServiceController[] services = ServiceController.GetServices();\n\n // Look for our service\n foreach (ServiceController service in services)\n if (String.Compare(serviceName, service.ServiceName, true) == 0) \n return true;\n\n // Return\n return false;\n }\n }\n}\n" }, { "path": "RaptorDBServer/Properties/AssemblyInfo.cs", "content": "using System.Reflection;\n\n[assembly: AssemblyTitle(\"RaptorDBServer\")]\n[assembly: AssemblyDescription(\"Stand alone server or Windows service loader\")]\n[assembly: AssemblyProduct(\"RaptorDBServer\")]\n\n\n\n\n" }, { "path": "RaptorDBServer/RaptorDBServer.csproj", "content": "\n\n \n Debug\n x86\n 8.0.30703\n 2.0\n {3EEB5C76-8216-4013-915D-94402BB320F6}\n Exe\n Properties\n RaptorDBServer\n RaptorDBServer\n v4.0\n 512\n \n publish\\\n true\n Disk\n false\n Foreground\n 7\n Days\n false\n false\n true\n 0\n 1.0.0.%2a\n false\n false\n true\n \n \n AnyCPU\n true\n full\n true\n ..\\Output\\server\\\n DEBUG;TRACE\n prompt\n 4\n false\n false\n \n \n AnyCPU\n pdbonly\n true\n bin\\Release\\\n TRACE\n prompt\n 4\n false\n \n \n \n \n \n true\n \n \n ..\\raptordb.snk\n \n \n \n \n \n \n \n \n BuildVersion.cs\n \n \n Component\n \n \n Component\n \n \n Service1.cs\n \n \n \n \n \n \n Service1.cs\n Designer\n \n \n \n \n {45F6BE30-989A-4749-B6A0-69099C8661F4}\n RaptorDB\n \n \n \n \n False\n .NET Framework 3.5 SP1\n false\n \n \n \n \n md \"$(SolutionDir)output\"\ncopy \"$(TargetPath)\" \"$(SolutionDir)output\\$(TargetFileName)\"\n \n \n" }, { "path": "RaptorDBServer/Service1.Designer.cs", "content": "namespace RaptorDBServer\n{\n partial class Service1\n {\n /// \n /// Required designer variable.\n /// \n private System.ComponentModel.IContainer components = null;\n\n /// \n /// Clean up any resources being used.\n /// \n /// true if managed resources should be disposed; otherwise, false.\n protected override void Dispose(bool disposing)\n {\n if (disposing && (components != null))\n {\n components.Dispose();\n }\n base.Dispose(disposing);\n }\n\n #region Component Designer generated code\n\n /// \n /// Required method for Designer support - do not modify \n /// the contents of this method with the code editor.\n /// \n private void InitializeComponent()\n {\n // \n // Service1\n // \n this.ServiceName = \"RaptorDB\";\n\n }\n\n #endregion\n }\n}\n" }, { "path": "RaptorDBServer/Service1.cs", "content": "using System.ServiceProcess;\nusing System.IO;\n\nnamespace RaptorDBServer\n{\n public partial class Service1 : ServiceBase\n {\n public Service1()\n {\n InitializeComponent();\n }\n\n RaptorDB.RaptorDBServer _raptor;\n\n protected override void OnStart(string[] args)\n {\n Directory.SetCurrentDirectory(Path.GetDirectoryName(this.GetType().Assembly.Location));\n _raptor = new RaptorDB.RaptorDBServer(Program.Port, Program.Path);\n }\n\n protected override void OnStop()\n {\n _raptor.Shutdown();\n }\n\n protected override void OnShutdown()\n {\n _raptor.Shutdown();\n }\n }\n}\n" }, { "path": "RaptorDBServer/Service1.resx", "content": "\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n text/microsoft-resx\n \n \n 2.0\n \n \n System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089\n \n \n System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089\n \n \n False\n \n" }, { "path": "RaptorDBTest.sln", "content": "\nMicrosoft Visual Studio Solution File, Format Version 12.00\n# Visual Studio 14\nVisualStudioVersion = 14.0.25420.1\nMinimumVisualStudioVersion = 10.0.40219.1\nProject(\"{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}\") = \"RaptorDB\", \"RaptorDB\\RaptorDB.csproj\", \"{45F6BE30-989A-4749-B6A0-69099C8661F4}\"\nEndProject\nProject(\"{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}\") = \"tests\", \"testing\\tests.csproj\", \"{C6DA7503-3BCF-4688-ADD7-1CB6EDCE5E90}\"\nEndProject\nProject(\"{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}\") = \"datagridbinding\", \"datagridbinding\\datagridbinding.csproj\", \"{4B90D800-C8C9-45CA-BD8C-DD5B47F78C41}\"\nEndProject\nProject(\"{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}\") = \"RaptorDB.Common\", \"RaptorDB.Common\\RaptorDB.Common.csproj\", \"{32331D51-5BE0-41E2-AF1A-9B086C5AE809}\"\nEndProject\nProject(\"{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}\") = \"Views\", \"Views\\Views.csproj\", \"{A1347486-8D54-4E17-8A22-76EFE61BF37B}\"\nEndProject\nProject(\"{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}\") = \"RaptorDBServer\", \"RaptorDBServer\\RaptorDBServer.csproj\", \"{3EEB5C76-8216-4013-915D-94402BB320F6}\"\nEndProject\nGlobal\n\tGlobalSection(SolutionConfigurationPlatforms) = preSolution\n\t\tDebug|Any CPU = Debug|Any CPU\n\t\tDebug|Mixed Platforms = Debug|Mixed Platforms\n\t\tDebug|x86 = Debug|x86\n\t\tRelease|Any CPU = Release|Any CPU\n\t\tRelease|Mixed Platforms = Release|Mixed Platforms\n\t\tRelease|x86 = Release|x86\n\tEndGlobalSection\n\tGlobalSection(ProjectConfigurationPlatforms) = postSolution\n\t\t{45F6BE30-989A-4749-B6A0-69099C8661F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU\n\t\t{45F6BE30-989A-4749-B6A0-69099C8661F4}.Debug|Any CPU.Build.0 = Debug|Any CPU\n\t\t{45F6BE30-989A-4749-B6A0-69099C8661F4}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU\n\t\t{45F6BE30-989A-4749-B6A0-69099C8661F4}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU\n\t\t{45F6BE30-989A-4749-B6A0-69099C8661F4}.Debug|x86.ActiveCfg = Debug|Any CPU\n\t\t{45F6BE30-989A-4749-B6A0-69099C8661F4}.Release|Any CPU.ActiveCfg = Release|Any CPU\n\t\t{45F6BE30-989A-4749-B6A0-69099C8661F4}.Release|Any CPU.Build.0 = Release|Any CPU\n\t\t{45F6BE30-989A-4749-B6A0-69099C8661F4}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU\n\t\t{45F6BE30-989A-4749-B6A0-69099C8661F4}.Release|Mixed Platforms.Build.0 = Release|Any CPU\n\t\t{45F6BE30-989A-4749-B6A0-69099C8661F4}.Release|x86.ActiveCfg = Release|Any CPU\n\t\t{C6DA7503-3BCF-4688-ADD7-1CB6EDCE5E90}.Debug|Any CPU.ActiveCfg = Debug|Any CPU\n\t\t{C6DA7503-3BCF-4688-ADD7-1CB6EDCE5E90}.Debug|Any CPU.Build.0 = Debug|Any CPU\n\t\t{C6DA7503-3BCF-4688-ADD7-1CB6EDCE5E90}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU\n\t\t{C6DA7503-3BCF-4688-ADD7-1CB6EDCE5E90}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU\n\t\t{C6DA7503-3BCF-4688-ADD7-1CB6EDCE5E90}.Debug|x86.ActiveCfg = Debug|Any CPU\n\t\t{C6DA7503-3BCF-4688-ADD7-1CB6EDCE5E90}.Release|Any CPU.ActiveCfg = Release|Any CPU\n\t\t{C6DA7503-3BCF-4688-ADD7-1CB6EDCE5E90}.Release|Any CPU.Build.0 = Release|Any CPU\n\t\t{C6DA7503-3BCF-4688-ADD7-1CB6EDCE5E90}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU\n\t\t{C6DA7503-3BCF-4688-ADD7-1CB6EDCE5E90}.Release|Mixed Platforms.Build.0 = Release|Any CPU\n\t\t{C6DA7503-3BCF-4688-ADD7-1CB6EDCE5E90}.Release|x86.ActiveCfg = Release|Any CPU\n\t\t{4B90D800-C8C9-45CA-BD8C-DD5B47F78C41}.Debug|Any CPU.ActiveCfg = Debug|Any CPU\n\t\t{4B90D800-C8C9-45CA-BD8C-DD5B47F78C41}.Debug|Any CPU.Build.0 = Debug|Any CPU\n\t\t{4B90D800-C8C9-45CA-BD8C-DD5B47F78C41}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU\n\t\t{4B90D800-C8C9-45CA-BD8C-DD5B47F78C41}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU\n\t\t{4B90D800-C8C9-45CA-BD8C-DD5B47F78C41}.Debug|x86.ActiveCfg = Debug|Any CPU\n\t\t{4B90D800-C8C9-45CA-BD8C-DD5B47F78C41}.Release|Any CPU.ActiveCfg = Release|Any CPU\n\t\t{4B90D800-C8C9-45CA-BD8C-DD5B47F78C41}.Release|Any CPU.Build.0 = Release|Any CPU\n\t\t{4B90D800-C8C9-45CA-BD8C-DD5B47F78C41}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU\n\t\t{4B90D800-C8C9-45CA-BD8C-DD5B47F78C41}.Release|Mixed Platforms.Build.0 = Release|Any CPU\n\t\t{4B90D800-C8C9-45CA-BD8C-DD5B47F78C41}.Release|x86.ActiveCfg = Release|Any CPU\n\t\t{32331D51-5BE0-41E2-AF1A-9B086C5AE809}.Debug|Any CPU.ActiveCfg = Debug|Any CPU\n\t\t{32331D51-5BE0-41E2-AF1A-9B086C5AE809}.Debug|Any CPU.Build.0 = Debug|Any CPU\n\t\t{32331D51-5BE0-41E2-AF1A-9B086C5AE809}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU\n\t\t{32331D51-5BE0-41E2-AF1A-9B086C5AE809}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU\n\t\t{32331D51-5BE0-41E2-AF1A-9B086C5AE809}.Debug|x86.ActiveCfg = Debug|Any CPU\n\t\t{32331D51-5BE0-41E2-AF1A-9B086C5AE809}.Release|Any CPU.ActiveCfg = Release|Any CPU\n\t\t{32331D51-5BE0-41E2-AF1A-9B086C5AE809}.Release|Any CPU.Build.0 = Release|Any CPU\n\t\t{32331D51-5BE0-41E2-AF1A-9B086C5AE809}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU\n\t\t{32331D51-5BE0-41E2-AF1A-9B086C5AE809}.Release|Mixed Platforms.Build.0 = Release|Any CPU\n\t\t{32331D51-5BE0-41E2-AF1A-9B086C5AE809}.Release|x86.ActiveCfg = Release|Any CPU\n\t\t{A1347486-8D54-4E17-8A22-76EFE61BF37B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU\n\t\t{A1347486-8D54-4E17-8A22-76EFE61BF37B}.Debug|Any CPU.Build.0 = Debug|Any CPU\n\t\t{A1347486-8D54-4E17-8A22-76EFE61BF37B}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU\n\t\t{A1347486-8D54-4E17-8A22-76EFE61BF37B}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU\n\t\t{A1347486-8D54-4E17-8A22-76EFE61BF37B}.Debug|x86.ActiveCfg = Debug|Any CPU\n\t\t{A1347486-8D54-4E17-8A22-76EFE61BF37B}.Release|Any CPU.ActiveCfg = Release|Any CPU\n\t\t{A1347486-8D54-4E17-8A22-76EFE61BF37B}.Release|Any CPU.Build.0 = Release|Any CPU\n\t\t{A1347486-8D54-4E17-8A22-76EFE61BF37B}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU\n\t\t{A1347486-8D54-4E17-8A22-76EFE61BF37B}.Release|Mixed Platforms.Build.0 = Release|Any CPU\n\t\t{A1347486-8D54-4E17-8A22-76EFE61BF37B}.Release|x86.ActiveCfg = Release|Any CPU\n\t\t{3EEB5C76-8216-4013-915D-94402BB320F6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU\n\t\t{3EEB5C76-8216-4013-915D-94402BB320F6}.Debug|Any CPU.Build.0 = Debug|Any CPU\n\t\t{3EEB5C76-8216-4013-915D-94402BB320F6}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU\n\t\t{3EEB5C76-8216-4013-915D-94402BB320F6}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU\n\t\t{3EEB5C76-8216-4013-915D-94402BB320F6}.Debug|x86.ActiveCfg = Debug|Any CPU\n\t\t{3EEB5C76-8216-4013-915D-94402BB320F6}.Release|Any CPU.ActiveCfg = Release|Any CPU\n\t\t{3EEB5C76-8216-4013-915D-94402BB320F6}.Release|Any CPU.Build.0 = Release|Any CPU\n\t\t{3EEB5C76-8216-4013-915D-94402BB320F6}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU\n\t\t{3EEB5C76-8216-4013-915D-94402BB320F6}.Release|Mixed Platforms.Build.0 = Release|Any CPU\n\t\t{3EEB5C76-8216-4013-915D-94402BB320F6}.Release|x86.ActiveCfg = Release|Any CPU\n\tEndGlobalSection\n\tGlobalSection(SolutionProperties) = preSolution\n\t\tHideSolutionNode = FALSE\n\tEndGlobalSection\nEndGlobal\n" }, { "path": "RaptorDB_Doc.nuspec", "content": "\n\n \n RaptorDB_doc\n 4.0.10\n RaptorDB Document Database\n mgholam\n \n https://www.codeproject.com/Articles/375413/RaptorDB-The-Document-Store\n false\n NoSql, JSON based, Document store database with compiled .net map functions and automatic hybrid bitmap indexing and LINQ query filters (now with standalone Server mode, Backup and Active Restore, Transactions, Server side queries, MonoDroid support, HQ-Branch Replication, works on Linux)\n \n \n \n \n \n \n \n \n \n \n \n" }, { "path": "RaptorDbCore.sln", "content": "\nMicrosoft Visual Studio Solution File, Format Version 12.00\n# Visual Studio 15\nVisualStudioVersion = 15.0.27130.2024\nMinimumVisualStudioVersion = 10.0.40219.1\nProject(\"{9A19103F-16F7-4668-BE54-9A1E7A4F7556}\") = \"RaptorDb.Common\", \"RaptorDbCore\\RaptorDb.Common\\RaptorDb.Common.csproj\", \"{A2AC2F5C-917E-40B4-9D78-83B6F3B84E74}\"\nEndProject\nProject(\"{9A19103F-16F7-4668-BE54-9A1E7A4F7556}\") = \"RaptorDB\", \"RaptorDbCore\\RaptorDB\\RaptorDB.csproj\", \"{CD626E71-6EE5-4B89-AD51-B09717C203C0}\"\nEndProject\nProject(\"{9A19103F-16F7-4668-BE54-9A1E7A4F7556}\") = \"test\", \"RaptorDBCore\\test\\test.csproj\", \"{4FC9D294-4318-4B4B-9AB4-B6907F0E8409}\"\n\tProjectSection(ProjectDependencies) = postProject\n\t\t{A2AC2F5C-917E-40B4-9D78-83B6F3B84E74} = {A2AC2F5C-917E-40B4-9D78-83B6F3B84E74}\n\t\t{CD626E71-6EE5-4B89-AD51-B09717C203C0} = {CD626E71-6EE5-4B89-AD51-B09717C203C0}\n\tEndProjectSection\nEndProject\nGlobal\n\tGlobalSection(SolutionConfigurationPlatforms) = preSolution\n\t\tDebug|Any CPU = Debug|Any CPU\n\t\tRelease|Any CPU = Release|Any CPU\n\tEndGlobalSection\n\tGlobalSection(ProjectConfigurationPlatforms) = postSolution\n\t\t{A2AC2F5C-917E-40B4-9D78-83B6F3B84E74}.Debug|Any CPU.ActiveCfg = Debug|Any CPU\n\t\t{A2AC2F5C-917E-40B4-9D78-83B6F3B84E74}.Debug|Any CPU.Build.0 = Debug|Any CPU\n\t\t{A2AC2F5C-917E-40B4-9D78-83B6F3B84E74}.Release|Any CPU.ActiveCfg = Release|Any CPU\n\t\t{A2AC2F5C-917E-40B4-9D78-83B6F3B84E74}.Release|Any CPU.Build.0 = Release|Any CPU\n\t\t{CD626E71-6EE5-4B89-AD51-B09717C203C0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU\n\t\t{CD626E71-6EE5-4B89-AD51-B09717C203C0}.Debug|Any CPU.Build.0 = Debug|Any CPU\n\t\t{CD626E71-6EE5-4B89-AD51-B09717C203C0}.Release|Any CPU.ActiveCfg = Release|Any CPU\n\t\t{CD626E71-6EE5-4B89-AD51-B09717C203C0}.Release|Any CPU.Build.0 = Release|Any CPU\n\t\t{4FC9D294-4318-4B4B-9AB4-B6907F0E8409}.Debug|Any CPU.ActiveCfg = Debug|Any CPU\n\t\t{4FC9D294-4318-4B4B-9AB4-B6907F0E8409}.Debug|Any CPU.Build.0 = Debug|Any CPU\n\t\t{4FC9D294-4318-4B4B-9AB4-B6907F0E8409}.Release|Any CPU.ActiveCfg = Release|Any CPU\n\t\t{4FC9D294-4318-4B4B-9AB4-B6907F0E8409}.Release|Any CPU.Build.0 = Release|Any CPU\n\tEndGlobalSection\n\tGlobalSection(SolutionProperties) = preSolution\n\t\tHideSolutionNode = FALSE\n\tEndGlobalSection\n\tGlobalSection(ExtensibilityGlobals) = postSolution\n\t\tSolutionGuid = {79525774-233F-4ED6-82E2-0A92423403DE}\n\tEndGlobalSection\nEndGlobal\n" }, { "path": "Tools/buildversion.ncs", "content": "using System;\nusing System.IO;\nusing System.Windows.Forms;\nusing System.Text.RegularExpressions;\n\nnamespace pp\n{\n\tpublic class pp\n\t{\n\t\tpublic static void Main(string[] args)\n\t\t{\n\t\t\tif(args.Length<1) return;\n\t\t\t\n\t\t\tstring filename = args[0];\n\t\t\tstring buildversion = \"\";\n\t\t\tint buildnum = 0;\n\t\t\t\n\t\t\t\n\t\t\tStreamReader sr = new StreamReader(filename);\n\t\t\t\n\t\t\tstring s = sr.ReadToEnd();\n\t\t\tsr.Close();\n\t\t\tstring buildnumregex = @\"\\s*//\\s*build\\s*number\\s*=\\s*(?\\w*)\";\n\t\t\tstring buildverregex = @\"\\s*//\\s*build\\s*version\\s*=\\s*(?[\\w.]*)\";\n\t\t\tRegex mr = new Regex(\n\t\t\t\tbuildnumregex,\n\t\t\t\tRegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace\n\t\t\t\t);\n\t\t\t\n\t\t\tMatch m = mr.Match(s);\n\t\t\tstring ss= m.Groups[\"num\"].Value;\n\t\t\tif(ss !=\"\") \n\t\t\t{\n\t\t\t\tbuildnum = int.Parse(ss);\n\t\t\t\tbuildnum++;\n\t\t\t\ts=mr.Replace(s,\"\\r\\n// build number = \"+buildnum.ToString());\n\t\t\t}\n\t\t\t\n\t\t\tmr = new Regex(\n\t\t\t\tbuildverregex,\n\t\t\t\tRegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace\n\t\t\t\t);\n\t\t\t\n\t\t\tm = mr.Match(s);\n\t\t\tss =m.Groups[\"vers\"].Value;\n\t\t\tif(ss !=\"\") buildversion = ss;\n\t\t\t\n\t\t\tstring currbuild = buildversion+\".\"+buildnum.ToString();\n\t\t\t\n\t\t\tmr = new Regex(\n\t\t\t\t@\"AssemblyFileVersion\\s*\\(\"\"(?[\\w.]*\"\")\",\n\t\t\t\tRegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace\n\t\t\t\t);\n\t\t\t\n\t\t\tm=mr.Match(s);\n\t\t\ts=mr.Replace(s,\"AssemblyFileVersion(\\\"\"+ currbuild + \"\\\"\");\n\n\t\t\t//MessageBox.Show(s);\n\t\t\t\n\t\t\tStreamWriter sw = new StreamWriter(args[0]);\n\t\t\tsw.Write(s);\n\t\t\tsw.Close();\n\t\t}\n\t}\n}" }, { "path": "Views/Class1.cs", "content": "using System;\nusing System.Collections.Generic;\nusing RaptorDB;\n\n\nnamespace SampleViews\n{\n #region [ class definitions ]\n //public enum State\n //{\n // Open,\n // Closed,\n // Approved\n //}\n\n public class LineItem\n {\n public decimal QTY { get; set; }\n public string Product { get; set; }\n public decimal Price { get; set; }\n public decimal Discount { get; set; }\n }\n\n public class SalesInvoice\n {\n public SalesInvoice()\n {\n ID = Guid.NewGuid();\n }\n\n public Guid ID { get; set; }\n public string CustomerName { get; set; }\n public string NoCase { get; set; }\n public string Address { get; set; }\n public List Items { get; set; }\n public DateTime Date { get; set; }\n public int Serial { get; set; }\n public byte Status { get; set; }\n public bool Approved { get; set; }\n //public State InvoiceState { get; set; }\n }\n #endregion\n\n #region [ views ]\n\n public class TestData\n {\n public Guid id;\n public string username;\n public string password;\n }\n\n public class TestSchema : RDBSchema\n {\n public string username;\n public string password;\n }\n\n public class TestView : View\n {\n public TestView()\n {\n this.Name = \"test\";\n this.Schema = typeof(TestSchema);\n this.Version = 1;\n this.isPrimaryList = true;\n this.isActive = true;\n this.BackgroundIndexing = true;\n\n this.Mapper = (api, docid, doc) =>\n {\n api.EmitObject(docid, doc);\n };\n }\n }\n\n [RegisterView]\n public class SalesInvoiceViewRowSchema : RDBSchema\n {\n //[FullText]\n public string CustomerName;\n [CaseInsensitive]\n [StringIndexLength(255)]\n public string NoCase;\n public DateTime Date;\n public string Address;\n public int Serial;\n public byte Status;//{ get; set; }\n public bool? Approved;// { get; set; }\n //public State InvoiceState;\n }\n\n [RegisterView]\n public class SalesInvoiceView : View\n {\n public SalesInvoiceView()\n {\n this.Name = \"SalesInvoice\";\n this.Description = \"A primary view for SalesInvoices\";\n this.isPrimaryList = true;\n this.isActive = true;\n this.BackgroundIndexing = true;\n this.Version = 6;\n //// uncomment the following for transaction mode\n //this.TransactionMode = true;\n\n this.Schema = typeof(SalesInvoiceViewRowSchema);\n\n this.FullTextColumns.Add(nameof(SalesInvoiceViewRowSchema.CustomerName));// \"customername\"); // this or the attribute\n this.FullTextColumns.Add(nameof(SalesInvoiceViewRowSchema.Address));// \"address\");\n\n this.CaseInsensitiveColumns.Add(nameof(SalesInvoiceViewRowSchema.NoCase));// \"nocase\"); // this or the attribute\n //this.StringIndexLength.Add(\"nocase\", 255);\n\n this.Mapper = (api, docid, doc) =>\n {\n //int c = api.Count(\"SalesItemRows\", \"product = \\\"prod 1\\\"\");\n if (doc.Serial == 0)\n api.RollBack();\n api.EmitObject(docid, doc);\n };\n }\n }\n\n public class SalesItemRowsViewRowSchema : RDBSchema\n {\n public string Product;\n public decimal QTY;\n public decimal Price;\n public decimal Discount;\n }\n\n [RegisterView]\n public class SalesItemRowsView : View\n {\n public SalesItemRowsView()\n {\n this.Name = \"SalesItemRows\";\n this.Description = \"\";\n this.isPrimaryList = false;\n this.isActive = true;\n this.BackgroundIndexing = true;\n\n this.Schema = typeof(SalesItemRowsViewRowSchema);\n\n this.Mapper = (api, docid, doc) =>\n {\n if (doc.Status == 3 && doc.Items != null)\n foreach (var item in doc.Items)\n api.EmitObject(docid, item);\n };\n }\n }\n\n public class NewViewRowSchema : RDBSchema\n {\n public string Product;\n public decimal QTY;\n public decimal Price;\n public decimal Discount;\n }\n\n [RegisterView]\n public class newview : View\n {\n public newview()\n {\n this.Name = \"newview\";\n this.Description = \"\";\n this.isPrimaryList = false;\n this.isActive = true;\n this.BackgroundIndexing = true;\n this.Version = 1;\n\n this.Schema = typeof(NewViewRowSchema);\n\n this.Mapper = (api, docid, doc) =>\n {\n if (doc.Status == 3 && doc.Items != null)\n foreach (var i in doc.Items)\n api.EmitObject(docid, i);\n };\n }\n }\n #endregion\n}\n" }, { "path": "Views/Properties/AssemblyInfo.cs", "content": "using System.Reflection;\nusing System.Runtime.InteropServices;\n\n// General Information about an assembly is controlled through the following \n// set of attributes. Change these attribute values to modify the information\n// associated with an assembly.\n[assembly: AssemblyTitle(\"Views\")]\n[assembly: AssemblyDescription(\"\")]\n[assembly: AssemblyConfiguration(\"\")]\n[assembly: AssemblyCompany(\"Microsoft\")]\n[assembly: AssemblyProduct(\"Views\")]\n[assembly: AssemblyCopyright(\"Copyright © Microsoft 2012\")]\n[assembly: AssemblyTrademark(\"\")]\n[assembly: AssemblyCulture(\"\")]\n\n// Setting ComVisible to false makes the types in this assembly not visible \n// to COM components. If you need to access a type in this assembly from \n// COM, set the ComVisible attribute to true on that type.\n[assembly: ComVisible(false)]\n\n// The following GUID is for the ID of the typelib if this project is exposed to COM\n[assembly: Guid(\"b3b61807-a394-421f-88f3-a340dc93f53b\")]\n\n// Version information for an assembly consists of the following four values:\n//\n// Major Version\n// Minor Version \n// Build Number\n// Revision\n//\n// You can specify all the values or you can default the Build and Revision Numbers \n// by using the '*' as shown below:\n// [assembly: AssemblyVersion(\"1.0.*\")]\n[assembly: AssemblyVersion(\"1.0.0.0\")]\n[assembly: AssemblyFileVersion(\"1.0.0.0\")]\n" }, { "path": "Views/ServerSide.cs", "content": "using System.Collections.Generic;\nusing System.Linq;\nusing RaptorDB.Common;\nusing SampleViews;\n\nnamespace Views\n{\n public class ServerSide\n {\n // so the result can be serialized and is not an anonymous type\n // since this uses fields, derive from the BindableFields for data binding to work\n public class sumtype : RaptorDB.BindableFields\n {\n public string Product;\n public decimal TotalPrice;\n public decimal TotalQTY;\n }\n\n public static List Sum_Products_based_on_filter(IRaptorDB rap, string filter)\n {\n var q = rap.Query(filter);\n\n var res = from x in q.Rows\n group x by x.Product into g\n select new sumtype // avoid anonymous types\n {\n Product = g.Key,\n TotalPrice = g.Sum(p => p.Price),\n TotalQTY = g.Sum(p => p.QTY)\n };\n\n return res.ToList();\n }\n\n public static List Sum_Products_based_on_filter_args(IRaptorDB rap, string filter, params object[] args)\n {\n if (args != null)\n {\n // get args here\n }\n var q = rap.Query(filter);\n\n var res = from x in q.Rows\n group x by x.Product into g\n select new sumtype // avoid anonymous types\n {\n Product = g.Key,\n TotalPrice = g.Sum(p => p.Price),\n TotalQTY = g.Sum(p => p.QTY)\n };\n\n return res.ToList();\n }\n }\n}\n" }, { "path": "Views/Views.csproj", "content": "\n\n \n Debug\n AnyCPU\n 8.0.30703\n 2.0\n {A1347486-8D54-4E17-8A22-76EFE61BF37B}\n Library\n Properties\n Views\n Views\n v4.0\n 512\n \n \n \n true\n full\n false\n bin\\Debug\\\n DEBUG;TRACE\n prompt\n 4\n false\n \n \n pdbonly\n true\n bin\\Release\\\n TRACE\n prompt\n 4\n false\n \n \n false\n \n \n ..\\key.snk\n \n \n \n \n \n \n \n \n \n \n \n {32331D51-5BE0-41E2-AF1A-9B086C5AE809}\n RaptorDB.Common\n \n \n \n \n md \"$(SolutionDir)Output\\server\\Extensions\"\nxcopy /q /y \"$(TargetDir)$(TargetFileName)\" \"$(SolutionDir)Output\\server\\Extensions\\\"\n \n \n" }, { "path": "WebStudio/README.md", "content": "# RaptorDB WebStudio\n\n\n## Get started\n\nInstall the dependencies...\n\n```bash\nnpm install\n```\n\n...then start [Rollup](https://rollupjs.org):\n\n```bash\nnpm run dev\n```\n\nNavigate to [localhost:5000](http://localhost:5000). You should see your app running. Edit a component file in `src`, save it, and reload the page to see your changes.\n" }, { "path": "WebStudio/build.cmd", "content": "md dist\nmd public\ncopy src\\index.html public\\\ncopy src\\index.html dist\\\ncopy src\\global.css public\\\ncopy src\\global.css dist\\\n\nnpm run build\n" }, { "path": "WebStudio/deploy.cmd", "content": "copy dist\\*.* ..\\raptordb\\web\npause" }, { "path": "WebStudio/package.json", "content": "{\n \"name\": \"RaptorDB-WebStudio\",\n \"version\": \"1.0.0\",\n \"devDependencies\": {\n \"npm-run-all\": \"^4.1.5\",\n \"rollup\": \"^1.10.1\",\n \"rollup-plugin-commonjs\": \"^9.3.4\",\n \"rollup-plugin-livereload\": \"^1.0.0\",\n \"rollup-plugin-node-resolve\": \"^4.2.3\",\n \"rollup-plugin-svelte\": \"^5.0.3\",\n \"rollup-plugin-terser\": \"^4.0.4\",\n \"svelte\": \"^3.0.0\"\n },\n \"dependencies\": {\n \"sirv-cli\": \"^0.4.4\"\n },\n \"scripts\": {\n \"build\": \"rollup -c --compact\",\n \"autobuild\": \"rollup -c -w\",\n \"dev\": \"run-p start:dev autobuild\",\n \"start\": \"sirv public --single\",\n \"start:dev\": \"sirv public --single --dev\"\n }\n}\n" }, { "path": "WebStudio/rollup.config.js", "content": "import svelte from 'rollup-plugin-svelte';\nimport resolve from 'rollup-plugin-node-resolve';\nimport commonjs from 'rollup-plugin-commonjs';\nimport livereload from 'rollup-plugin-livereload';\nimport { terser } from 'rollup-plugin-terser';\n\nconst production = !process.env.ROLLUP_WATCH;\nconst dir = production ? \"dist\" : \"public\";\n\nexport default {\n input: production ? 'src/main.js' : 'src/debug.js',\n\n output: {\n sourcemap: !production,\n format: 'iife',\n name: 'app',\n file: dir + '/bundle.js'\n },\n plugins: [\n svelte({\n // enable run-time checks when not in production\n dev: !production,\n // we'll extract any component CSS out into\n // a separate file — better for performance\n css: css => {\n css.write(dir + '/bundle.css', !production); // disable sourcemap\n }\n }),\n\n // If you have external dependencies installed from\n // npm, you'll most likely need these plugins. In\n // some cases you'll need additional configuration —\n // consult the documentation for details:\n // https://github.com/rollup/rollup-plugin-commonjs\n resolve({ browser: true }),\n commonjs(),\n\n // Watch the `public` directory and refresh the\n // browser on changes when not in production\n !production && livereload(dir), // 'public'\n\n // If we're building for production (npm run build\n // instead of npm run dev), minify\n\n production && terser()\n ],\n watch: {\n clearScreen: true\n }\n};" }, { "path": "WebStudio/src/App.svelte", "content": "\n\n\n\n
\n
\n showtab(event.detail)} />\n
\n\n
\n
    \n X\n {#each tabs as tab (tab.id)}\n changetab(tab.id)}>\n
    \n \n {#if tab.title !== 'Help'}\n \n {/if}\n
    \n \n {/each}\n
\n\n
\n\n
\n
\n{#if showmsg}\n (showmsg = false)} {title}>\n
\n {#if modalyes}\n \n {/if}\n \n
\n \n{/if}\n" }, { "path": "WebStudio/src/UI/Button.svelte", "content": "\n\n\n\n{#if href}\n \n \n \n{:else}\n \n{/if}\n" }, { "path": "WebStudio/src/UI/Modal.svelte", "content": "\n\n\n\n
\n
\n

{title}

\n
\n \n
\n
\n \n \n \n
\n
\n" }, { "path": "WebStudio/src/UI/datatable.svelte", "content": "\n\n\n\n{#if rows !== null && rows.length > 0}\n 
Rows : {rows.length} of {totalrows.toLocaleString()}
\n
\n \n \n \n \n \n \n \n \n
\n\n
\n \n \n {#each Object.keys(rows[0]) as col (col)}\n \n {/each}\n \n {#each rows as row}\n \n {#each Object.keys(row) as r (r)}\n \n {/each}\n \n {/each}\n
sortby(col)}>{col}
\n {#if r === 'docid'}\n \n {:else}{row[r]}{/if}\n
\n
\n\n
\n \n \n \n
\n{:else}\n

No data

\n{/if}\n" }, { "path": "WebStudio/src/UI/nav.svelte", "content": "\n\n\n\n
\n \"logo\"\n\n {#each Object.keys(navdata) as name}\n togglepanel(name)}>\n {name}\n \n
\n {#each navdata[name] as panel}\n \n {/each}\n
\n {/each}\n
\n" }, { "path": "WebStudio/src/debug.js", "content": "import './main.js'\nwindow.ServerURL = \"http://localhost:91/\";" }, { "path": "WebStudio/src/global.css", "content": "body {\n font-family: sans-serif;\n color: #222;\n background: #414141;\n}\n\n.tab-content {\n background-color: white;\n padding: 5px;\n padding-top: 15px;\n min-height: 90vh;\n}\n\n.LogArea {\n border-style: solid;\n border-color: gray;\n background-color: #e0e0e0;\n padding: 5px;\n}\n\n.LogItems {\n font-family: \"Courier New\", monospace;\n white-space: pre-wrap;\n font-size: smaller;\n}\n\n.JSONArea {\n border-style: solid;\n border-color: gray;\n background-color: white;\n padding: 5px;\n min-height: 500px;\n}\n\n.JSON {\n font-family: \"Courier New\", monospace;\n white-space: pre-wrap;\n font-size: smaller;\n}" }, { "path": "WebStudio/src/index.html", "content": "\n\n\t\n\t\n\n\tRaptorDB WebStudio\n\n\t\n\t\n\t\n\n\n\n\t\n\n" }, { "path": "WebStudio/src/main.js", "content": "import App from './App.svelte';\n\nconst app = new App({\n target: document.body,\n});\n\nwindow.ServerURL = document.location.protocol + \"//\" + document.location.hostname + \":\" + document.location.port + \"/\";\nwindow.GET = function(url, callback, error) {\n var xhr = new XMLHttpRequest();\n xhr.open('GET', window.ServerURL + url);\n\n xhr.onreadystatechange = function() {\n if (xhr.readyState === 1) { return; } else if (xhr.readyState === 4) {\n if (xhr.status < 400) {\n var data = null;\n if (xhr.responseText !== \"\")\n data = JSON.parse(xhr.responseText);\n else\n data = \"\";\n if (callback)\n callback(data);\n } else if (xhr.status === 401) {\n xhr.abort();\n if (callback)\n callback(\"\");\n } else if (error !== null)\n error(xhr.responseText);\n }\n };\n xhr.onerror = function(err) {\n // $.showDialog(\"Sorry\", \"

Connection Failed!

\");\n console.log(err);\n };\n xhr.withCredentials = false;\n xhr.send();\n}\nwindow.LOAD = function(url, callback, error) {\n var xhr = new XMLHttpRequest();\n xhr.open('GET', window.ServerURL + url);\n\n xhr.onreadystatechange = function() {\n if (xhr.readyState === 1) { return; } else if (xhr.readyState === 4) {\n // console.log(xhr);\n if (xhr.status < 400) {\n var data = null;\n if (xhr.responseText !== \"\")\n data = xhr.responseText;\n else\n data = \"\";\n if (callback)\n callback(data);\n } else if (xhr.status === 401) {\n xhr.abort();\n if (callback)\n callback(\"\");\n } else if (error !== null)\n error(xhr.responseText);\n }\n };\n xhr.onerror = function(err) {\n // $.showDialog(\"Sorry\", \"

Connection Failed!

\");\n console.log(err);\n };\n xhr.withCredentials = false;\n xhr.send();\n}\n\nexport default app;" }, { "path": "WebStudio/src/pages/dochistory.svelte", "content": "\n\n\n\n\n\n{#if active}\n
\n \n \n {\n if ((event.keyCode || event.which) == 13) fillhistory();\n }}>\n Get\n \n
\n
\n \n \n {#if versions.length > 0}\n \n \n \n \n \n
\n
\n {#each versions as v (v)}\n \n {/each}\n
\n 		\n
\n
{jsondata}
\n
\n
\n {/if}\n
\n{/if}\n" }, { "path": "WebStudio/src/pages/docsearch.svelte", "content": "\n\n\n\n\n\n{#if active}\n
\n \n {\n if ((event.keyCode || event.which) == 13) docsearch();\n }}\n bind:value={find} />\n \n
\n
\n \n \n
\n{/if}\n" }, { "path": "WebStudio/src/pages/docview.svelte", "content": "\n\n\n\n\n\n{#if active}\n
\n \n {\n if ((event.keyCode || event.which) == 13) docview();\n }}\n bind:value={docid} />\n \n
\n
\n {#if doc !== ''}\n
\n \n
\n {/if}\n
\n{/if}\n" }, { "path": "WebStudio/src/pages/help.svelte", "content": "\n\n\n\n{#if active}\n
\n

Welcome to the RaptorDB Web Studio

\n You can use this Web interface to query and see configuration information.\n \n
\n{/if}\n" }, { "path": "WebStudio/src/pages/hfbrowser.svelte", "content": "\n\n\n\n\n\n{#if active}\n
\n \n
\n
\n \n {\n if ((event.keyCode || event.which) == 13) showkey(key);\n }}\n bind:value={key} />\n \n
\n
\n \n \n
\n{/if}\n" }, { "path": "WebStudio/src/pages/query.svelte", "content": "\n\n\n\n\n\n{#if active}\n
\n \n \n
\n
\n\n \n \n \n \n \n \n
\n \n {#each colnames as col (col)}\n \n {/each}\n \n \n \n \n \n \n \n \n \n (filter = '')}>\n Clear\n \n
\n \n
\n {\n page = 1;\n start = 0;\n run();\n }}>\n Run\n \n \n
\n\n {#if errormsg === ''}\n {#if data != null}\n 
Query time (+render) : {qtime}
\n {/if}\n refresh(event)}\n on:showdoc\n bind:rows={data}\n bind:totalrows\n bind:page />\n {:else}\n

{errormsg}

\n {/if}\n\n
\n{/if}\n" }, { "path": "WebStudio/src/pages/schema.svelte", "content": "\n\n\n\n\n\n{#if active}\n
\n \n \n
\n
\n
\n \n \n
\n \n \n
\n \n \n
\n \n \n
\n \n \n
\n \n \n
\n \n \n
\n \n \n
\n \n \n
\n \n \n \n \n {#each Object.keys(schema) as sc (sc)}\n \n \n \n \n {/each}\n
Column NameType
{sc}{schema[sc]}
\n \n \n \n \n \n \n \n \n \n \n \n \n \n
Full Text Columns :Case Insensitive Columns :String Index Length :No Indexing Columns :
\n
\n {#each FullTextColumns as fc (fc)}\n \n {/each}\n
\n 		\n
\n {#each CaseInsensitiveColumns as fc (fc)}\n \n {/each}\n
\n 		\n
\n {#each StringIndexLength as fc (fc)}\n \n {/each}\n
\n 		\n
\n {#each NoIndexingColumns as fc (fc)}\n \n {/each}\n
\n
\n
\n
\n{/if}\n" }, { "path": "WebStudio/src/pages/sysconfig.svelte", "content": "\n\n\n\n\n\n{#if active}\n
\n

raptordb.config

\n
\n 
{configs}
\n
\n
\n{/if}\n" }, { "path": "WebStudio/src/pages/sysinfo.svelte", "content": "\n\n\n\n\n\n{#if active}\n
\n \n \n
\n
\n\n
\n \n \n
\n \n \n
\n \n \n
\n \n \n
\n \n \n
\n \n \n
\n \n \n
\n \n \n
\n \n \n
\n \n
\n {#each logs as l (l)}\n
{l}
\n {/each}\n
\n
\n
\n{/if}\n" }, { "path": "build.cmd", "content": "\"C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\MSBuild\\15.0\\Bin\\msbuild.exe\" /p:buildmode=Debug /t:rebuild raptordbtest.sln\ndotnet build raptordbcore.sln\n.nuget\\NuGet.exe pack raptordb_doc.nuspec" }, { "path": "datagridbinding/Program.cs", "content": "using System;\nusing System.Windows.Forms;\nusing System.Reflection;\n\nnamespace datagridbinding\n{\n\n static class Program\n {\n public static void DoubleBuffered(this DataGridView dgv, bool setting)\n {\n Type dgvType = dgv.GetType();\n PropertyInfo pi = dgvType.GetProperty(\"DoubleBuffered\", BindingFlags.Instance | BindingFlags.NonPublic);\n pi.SetValue(dgv, setting, null);\n }\n /// \n /// The main entry point for the application.\n /// \n [STAThread]\n static void Main()\n {\n Application.EnableVisualStyles();\n Application.SetCompatibleTextRenderingDefault(false);\n Application.Run(new frmMain());\n }\n }\n}\n" }, { "path": "datagridbinding/Properties/AssemblyInfo.cs", "content": "using System.Reflection;\nusing System.Runtime.InteropServices;\n\n// General Information about an assembly is controlled through the following \n// set of attributes. Change these attribute values to modify the information\n// associated with an assembly.\n[assembly: AssemblyTitle(\"datagridbinding\")]\n[assembly: AssemblyDescription(\"\")]\n[assembly: AssemblyConfiguration(\"\")]\n[assembly: AssemblyCompany(\"Microsoft\")]\n[assembly: AssemblyProduct(\"datagridbinding\")]\n[assembly: AssemblyCopyright(\"Copyright © Microsoft 2012\")]\n[assembly: AssemblyTrademark(\"\")]\n[assembly: AssemblyCulture(\"\")]\n\n// Setting ComVisible to false makes the types in this assembly not visible \n// to COM components. If you need to access a type in this assembly from \n// COM, set the ComVisible attribute to true on that type.\n[assembly: ComVisible(false)]\n\n// The following GUID is for the ID of the typelib if this project is exposed to COM\n[assembly: Guid(\"7231f003-9b1a-444b-b0db-a4109f304fd1\")]\n\n// Version information for an assembly consists of the following four values:\n//\n// Major Version\n// Minor Version \n// Build Number\n// Revision\n//\n// You can specify all the values or you can default the Build and Revision Numbers \n// by using the '*' as shown below:\n// [assembly: AssemblyVersion(\"1.0.*\")]\n[assembly: AssemblyVersion(\"1.0.0.0\")]\n[assembly: AssemblyFileVersion(\"1.0.0.0\")]\n" }, { "path": "datagridbinding/datagridbinding.csproj", "content": "\n\n \n Debug\n AnyCPU\n 9.0.30729\n 2.0\n {4B90D800-C8C9-45CA-BD8C-DD5B47F78C41}\n WinExe\n Properties\n datagridbinding\n datagridbinding\n v4.0\n 512\n \n \n \n true\n full\n false\n ..\\Output\\\n DEBUG;TRACE\n prompt\n 4\n false\n \n \n pdbonly\n true\n bin\\Release\\\n TRACE\n prompt\n 4\n false\n \n \n \n ..\\Faker.dll\n \n \n \n \n \n \n \n Form\n \n \n frmMain.cs\n \n \n Form\n \n \n frmStartup.cs\n \n \n \n \n frmMain.cs\n \n \n frmStartup.cs\n \n \n \n \n {32331D51-5BE0-41E2-AF1A-9B086C5AE809}\n RaptorDB.Common\n \n \n {45F6BE30-989A-4749-B6A0-69099C8661F4}\n RaptorDB\n \n \n {A1347486-8D54-4E17-8A22-76EFE61BF37B}\n Views\n \n \n \n \n" }, { "path": "datagridbinding/frmMain.Designer.cs", "content": "namespace datagridbinding\n{\n partial class frmMain\n {\n /// \n /// Required designer variable.\n /// \n private System.ComponentModel.IContainer components = null;\n\n /// \n /// Clean up any resources being used.\n /// \n /// true if managed resources should be disposed; otherwise, false.\n protected override void Dispose(bool disposing)\n {\n if (disposing && (components != null))\n {\n components.Dispose();\n }\n base.Dispose(disposing);\n }\n\n #region Windows Form Designer generated code\n\n /// \n /// Required method for Designer support - do not modify\n /// the contents of this method with the code editor.\n /// \n private void InitializeComponent()\n {\n this.components = new System.ComponentModel.Container();\n System.Windows.Forms.DataGridViewCellStyle dataGridViewCellStyle1 = new System.Windows.Forms.DataGridViewCellStyle();\n this.bindingSource1 = new System.Windows.Forms.BindingSource(this.components);\n this.statusStrip1 = new System.Windows.Forms.StatusStrip();\n this.toolStripStatusLabel1 = new System.Windows.Forms.ToolStripStatusLabel();\n this.toolStripStatusLabel2 = new System.Windows.Forms.ToolStripStatusLabel();\n this.toolStripProgressBar1 = new System.Windows.Forms.ToolStripProgressBar();\n this.stsError = new System.Windows.Forms.ToolStripStatusLabel();\n this.menuStrip1 = new System.Windows.Forms.MenuStrip();\n this.sumQueryToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();\n this.serverSideSumQueryToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();\n this.insert100000DocumentsToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();\n this.backupToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();\n this.restoreToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();\n this.exitToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();\n this.testToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();\n this.textBox1 = new System.Windows.Forms.TextBox();\n this.dataGridView1 = new System.Windows.Forms.DataGridView();\n this.freememoryToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem();\n ((System.ComponentModel.ISupportInitialize)(this.bindingSource1)).BeginInit();\n this.statusStrip1.SuspendLayout();\n this.menuStrip1.SuspendLayout();\n ((System.ComponentModel.ISupportInitialize)(this.dataGridView1)).BeginInit();\n this.SuspendLayout();\n // \n // statusStrip1\n // \n this.statusStrip1.Items.AddRange(new System.Windows.Forms.ToolStripItem[] {\n this.toolStripStatusLabel1,\n this.toolStripStatusLabel2,\n this.toolStripProgressBar1,\n this.stsError});\n this.statusStrip1.Location = new System.Drawing.Point(0, 364);\n this.statusStrip1.Name = \"statusStrip1\";\n this.statusStrip1.Padding = new System.Windows.Forms.Padding(1, 0, 7, 0);\n this.statusStrip1.Size = new System.Drawing.Size(693, 22);\n this.statusStrip1.TabIndex = 3;\n this.statusStrip1.Text = \"statusStrip1\";\n // \n // toolStripStatusLabel1\n // \n this.toolStripStatusLabel1.Name = \"toolStripStatusLabel1\";\n this.toolStripStatusLabel1.Size = new System.Drawing.Size(60, 17);\n this.toolStripStatusLabel1.Text = \"Count = 0\";\n // \n // toolStripStatusLabel2\n // \n this.toolStripStatusLabel2.Name = \"toolStripStatusLabel2\";\n this.toolStripStatusLabel2.Size = new System.Drawing.Size(48, 17);\n this.toolStripStatusLabel2.Text = \"time =0\";\n // \n // toolStripProgressBar1\n // \n this.toolStripProgressBar1.Name = \"toolStripProgressBar1\";\n this.toolStripProgressBar1.Size = new System.Drawing.Size(100, 16);\n // \n // stsError\n // \n this.stsError.DisplayStyle = System.Windows.Forms.ToolStripItemDisplayStyle.Text;\n this.stsError.Font = new System.Drawing.Font(\"Segoe UI\", 9F, System.Drawing.FontStyle.Bold);\n this.stsError.ForeColor = System.Drawing.Color.Red;\n this.stsError.Name = \"stsError\";\n this.stsError.Size = new System.Drawing.Size(0, 17);\n // \n // menuStrip1\n // \n this.menuStrip1.Items.AddRange(new System.Windows.Forms.ToolStripItem[] {\n this.sumQueryToolStripMenuItem,\n this.serverSideSumQueryToolStripMenuItem,\n this.insert100000DocumentsToolStripMenuItem,\n this.backupToolStripMenuItem,\n this.restoreToolStripMenuItem,\n this.exitToolStripMenuItem,\n this.testToolStripMenuItem,\n this.freememoryToolStripMenuItem});\n this.menuStrip1.Location = new System.Drawing.Point(0, 0);\n this.menuStrip1.Name = \"menuStrip1\";\n this.menuStrip1.Size = new System.Drawing.Size(693, 24);\n this.menuStrip1.TabIndex = 5;\n this.menuStrip1.Text = \"menuStrip1\";\n // \n // sumQueryToolStripMenuItem\n // \n this.sumQueryToolStripMenuItem.Name = \"sumQueryToolStripMenuItem\";\n this.sumQueryToolStripMenuItem.Size = new System.Drawing.Size(78, 20);\n this.sumQueryToolStripMenuItem.Text = \"Sum Query\";\n this.sumQueryToolStripMenuItem.Click += new System.EventHandler(this.sumQueryToolStripMenuItem_Click);\n // \n // serverSideSumQueryToolStripMenuItem\n // \n this.serverSideSumQueryToolStripMenuItem.Name = \"serverSideSumQueryToolStripMenuItem\";\n this.serverSideSumQueryToolStripMenuItem.Size = new System.Drawing.Size(138, 20);\n this.serverSideSumQueryToolStripMenuItem.Text = \"Server Side Sum Query\";\n this.serverSideSumQueryToolStripMenuItem.Click += new System.EventHandler(this.serverSideSumQueryToolStripMenuItem_Click);\n // \n // insert100000DocumentsToolStripMenuItem\n // \n this.insert100000DocumentsToolStripMenuItem.Name = \"insert100000DocumentsToolStripMenuItem\";\n this.insert100000DocumentsToolStripMenuItem.Size = new System.Drawing.Size(154, 20);\n this.insert100000DocumentsToolStripMenuItem.Text = \"Insert 100,000 Documents\";\n this.insert100000DocumentsToolStripMenuItem.Click += new System.EventHandler(this.insert100000DocumentsToolStripMenuItem_Click);\n // \n // backupToolStripMenuItem\n // \n this.backupToolStripMenuItem.Name = \"backupToolStripMenuItem\";\n this.backupToolStripMenuItem.Size = new System.Drawing.Size(58, 20);\n this.backupToolStripMenuItem.Text = \"Backup\";\n this.backupToolStripMenuItem.Click += new System.EventHandler(this.backupToolStripMenuItem_Click);\n // \n // restoreToolStripMenuItem\n // \n this.restoreToolStripMenuItem.Name = \"restoreToolStripMenuItem\";\n this.restoreToolStripMenuItem.Size = new System.Drawing.Size(58, 20);\n this.restoreToolStripMenuItem.Text = \"Restore\";\n this.restoreToolStripMenuItem.Click += new System.EventHandler(this.restoreToolStripMenuItem_Click);\n // \n // exitToolStripMenuItem\n // \n this.exitToolStripMenuItem.Name = \"exitToolStripMenuItem\";\n this.exitToolStripMenuItem.Size = new System.Drawing.Size(37, 20);\n this.exitToolStripMenuItem.Text = \"Exit\";\n this.exitToolStripMenuItem.Click += new System.EventHandler(this.exitToolStripMenuItem_Click);\n // \n // testToolStripMenuItem\n // \n this.testToolStripMenuItem.Name = \"testToolStripMenuItem\";\n this.testToolStripMenuItem.Size = new System.Drawing.Size(38, 20);\n this.testToolStripMenuItem.Text = \"test\";\n this.testToolStripMenuItem.Click += new System.EventHandler(this.testToolStripMenuItem_Click);\n // \n // textBox1\n // \n this.textBox1.Dock = System.Windows.Forms.DockStyle.Top;\n this.textBox1.Location = new System.Drawing.Point(0, 24);\n this.textBox1.Margin = new System.Windows.Forms.Padding(3, 2, 3, 2);\n this.textBox1.Name = \"textBox1\";\n this.textBox1.Size = new System.Drawing.Size(693, 23);\n this.textBox1.TabIndex = 6;\n this.textBox1.Text = \"salesinvoice, serial<100\";\n this.textBox1.KeyPress += new System.Windows.Forms.KeyPressEventHandler(this.TextBox1KeyPress);\n // \n // dataGridView1\n // \n dataGridViewCellStyle1.BackColor = System.Drawing.Color.FromArgb(((int)(((byte)(192)))), ((int)(((byte)(255)))), ((int)(((byte)(192)))));\n this.dataGridView1.AlternatingRowsDefaultCellStyle = dataGridViewCellStyle1;\n this.dataGridView1.AutoSizeColumnsMode = System.Windows.Forms.DataGridViewAutoSizeColumnsMode.DisplayedCells;\n this.dataGridView1.ColumnHeadersHeightSizeMode = System.Windows.Forms.DataGridViewColumnHeadersHeightSizeMode.AutoSize;\n this.dataGridView1.Dock = System.Windows.Forms.DockStyle.Fill;\n this.dataGridView1.EditMode = System.Windows.Forms.DataGridViewEditMode.EditProgrammatically;\n this.dataGridView1.Location = new System.Drawing.Point(0, 47);\n this.dataGridView1.Margin = new System.Windows.Forms.Padding(3, 2, 3, 2);\n this.dataGridView1.Name = \"dataGridView1\";\n this.dataGridView1.Size = new System.Drawing.Size(693, 317);\n this.dataGridView1.TabIndex = 7;\n this.dataGridView1.CellDoubleClick += new System.Windows.Forms.DataGridViewCellEventHandler(this.dataGridView1_CellDoubleClick);\n // \n // freememoryToolStripMenuItem\n // \n this.freememoryToolStripMenuItem.Name = \"freememoryToolStripMenuItem\";\n this.freememoryToolStripMenuItem.Size = new System.Drawing.Size(84, 20);\n this.freememoryToolStripMenuItem.Text = \"freememory\";\n this.freememoryToolStripMenuItem.Click += new System.EventHandler(this.freememoryToolStripMenuItem_Click);\n // \n // frmMain\n // \n this.AutoScaleDimensions = new System.Drawing.SizeF(7F, 16F);\n this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;\n this.ClientSize = new System.Drawing.Size(693, 386);\n this.Controls.Add(this.dataGridView1);\n this.Controls.Add(this.textBox1);\n this.Controls.Add(this.statusStrip1);\n this.Controls.Add(this.menuStrip1);\n this.Font = new System.Drawing.Font(\"Tahoma\", 9.75F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(178)));\n this.MainMenuStrip = this.menuStrip1;\n this.Margin = new System.Windows.Forms.Padding(3, 4, 3, 4);\n this.Name = \"frmMain\";\n this.Text = \"Query Viewer\";\n this.FormClosing += new System.Windows.Forms.FormClosingEventHandler(this.frmMain_FormClosing);\n this.Load += new System.EventHandler(this.Form1_Load);\n ((System.ComponentModel.ISupportInitialize)(this.bindingSource1)).EndInit();\n this.statusStrip1.ResumeLayout(false);\n this.statusStrip1.PerformLayout();\n this.menuStrip1.ResumeLayout(false);\n this.menuStrip1.PerformLayout();\n ((System.ComponentModel.ISupportInitialize)(this.dataGridView1)).EndInit();\n this.ResumeLayout(false);\n this.PerformLayout();\n\n }\n private System.Windows.Forms.ToolStripStatusLabel toolStripStatusLabel2;\n private System.Windows.Forms.ToolStripStatusLabel toolStripStatusLabel1;\n private System.Windows.Forms.StatusStrip statusStrip1;\n\n #endregion\n\n private System.Windows.Forms.BindingSource bindingSource1;\n private System.Windows.Forms.MenuStrip menuStrip1;\n private System.Windows.Forms.ToolStripMenuItem sumQueryToolStripMenuItem;\n private System.Windows.Forms.TextBox textBox1;\n private System.Windows.Forms.DataGridView dataGridView1;\n private System.Windows.Forms.ToolStripMenuItem exitToolStripMenuItem;\n private System.Windows.Forms.ToolStripMenuItem insert100000DocumentsToolStripMenuItem;\n private System.Windows.Forms.ToolStripProgressBar toolStripProgressBar1;\n private System.Windows.Forms.ToolStripMenuItem backupToolStripMenuItem;\n private System.Windows.Forms.ToolStripMenuItem restoreToolStripMenuItem;\n private System.Windows.Forms.ToolStripMenuItem serverSideSumQueryToolStripMenuItem;\n private System.Windows.Forms.ToolStripMenuItem testToolStripMenuItem;\n private System.Windows.Forms.ToolStripStatusLabel stsError;\n private System.Windows.Forms.ToolStripMenuItem freememoryToolStripMenuItem;\n }\n}\n\n" }, { "path": "datagridbinding/frmMain.cs", "content": "using RaptorDB.Common;\nusing SampleViews;\nusing System;\nusing System.Collections.Generic;\nusing System.Linq;\nusing System.Windows.Forms;\n\nnamespace datagridbinding\n{\n public partial class frmMain : Form\n {\n public frmMain()\n {\n InitializeComponent();\n }\n\n\n IRaptorDB rap;\n\n private void Form1_Load(object sender, EventArgs e)\n {\n dataGridView1.DoubleBuffered(true);\n frmStartup f = new frmStartup();\n if (f.ShowDialog() == DialogResult.OK)\n {\n rap = f._rap;\n\n Query();\n }\n }\n\n void TextBox1KeyPress(object sender, KeyPressEventArgs e)\n {\n if (e.KeyChar == (char)Keys.Return)\n Query();\n }\n\n private void Query()\n {\n int c = textBox1.Text.IndexOf(',');\n string viewname = textBox1.Text.Substring(0, c);\n string args = textBox1.Text.Substring(c + 1);\n\n try\n {\n DateTime dt = FastDateTime.Now;\n var q = rap.Query(viewname, args.Trim());\n toolStripStatusLabel2.Text = \"Query time (sec) = \" + FastDateTime.Now.Subtract(dt).TotalSeconds;\n dataGridView1.DataSource = q.Rows;\n toolStripStatusLabel1.Text = \"Count = \" + q.Count.ToString(\"#,0\");\n stsError.Text = \"\";\n }\n catch (Exception ex)\n {\n stsError.Text = ex.Message;\n dataGridView1.DataSource = null;\n toolStripStatusLabel1.Text = \"Count = 0\";\n toolStripStatusLabel2.Text = \"Query time (sec) = 0\";\n }\n }\n\n private void sumQueryToolStripMenuItem_Click(object sender, EventArgs e)\n {\n int c = rap.Count(\"SalesItemRows\", \"product = \\\"prod 1\\\"\");\n\n DateTime dt = FastDateTime.Now;\n var q = //rap.Query(typeof(SalesItemRowsView), (LineItem l) => (l.Product == \"prod 1\" || l.Product == \"prod 3\"));\n rap.Query(x => x.Product == \"prod 1\" || x.Product == \"prod 3\");\n //List list = q.Rows.Cast().ToList();\n var res = from item in q.Rows//list\n group item by item.Product into grouped\n select new\n {\n Product = grouped.Key,\n TotalPrice = grouped.Sum(product => product.Price),\n TotalQTY = grouped.Sum(product => product.QTY)\n };\n\n var reslist = res.ToList();\n dataGridView1.DataSource = reslist;\n toolStripStatusLabel2.Text = \"Query time (sec) = \" + FastDateTime.Now.Subtract(dt).TotalSeconds;\n toolStripStatusLabel1.Text = \"Count = \" + q.Count.ToString(\"#,0\");\n }\n\n private void exitToolStripMenuItem_Click(object sender, EventArgs e)\n {\n this.Close();// shutdown();\n }\n\n private void shutdown()\n {\n if (rap != null)\n rap.Shutdown();\n //this.Close();\n }\n\n private object _lock = new object();\n private void insert100000DocumentsToolStripMenuItem_Click(object sender, EventArgs e)\n {\n //RaptorDB.Global.SplitStorageFilesMegaBytes = 50;\n lock (_lock)\n {\n DialogResult dr = MessageBox.Show(\"Do you want to insert?\", \"Continue?\", MessageBoxButtons.OKCancel, MessageBoxIcon.Stop, MessageBoxDefaultButton.Button2);\n if (dr == System.Windows.Forms.DialogResult.Cancel)\n return;\n toolStripProgressBar1.Value = 0;\n DateTime dt = FastDateTime.Now;\n int count = 100000;\n int step = 5000;\n toolStripProgressBar1.Maximum = (count / step) + 1;\n Random r = new Random();\n for (int i = 0; i < count; i++)\n {\n var inv = CreateInvoice(i);\n if (i % step == 0)\n toolStripProgressBar1.Value++;\n rap.Save(inv.ID, inv);\n }\n MessageBox.Show(\"Insert done in (sec) : \" + FastDateTime.Now.Subtract(dt).TotalSeconds);\n toolStripProgressBar1.Value = 0;\n }\n }\n\n private static SalesInvoice CreateInvoice(int i)\n {\n var inv = new SalesInvoice()\n {\n Date = Faker.DateTimeFaker.BirthDay(),// FastDateTime.Now.AddMinutes(r.Next(60)),\n Serial = i % 10000,\n CustomerName = Faker.NameFaker.Name(),// \"Me \" + i % 10,\n NoCase = \"Me \" + i % 10,\n Status = (byte)(i % 4),\n Address = Faker.LocationFaker.Street(), //\"df asd sdf asdf asdf\",\n Approved = i % 100 == 0 ? true : false\n };\n inv.Items = new List();\n for (int k = 0; k < 5; k++)\n inv.Items.Add(new LineItem() { Product = \"prod \" + k, Discount = 0, Price = 10 + k, QTY = 1 + k });\n return inv;\n }\n\n private void backupToolStripMenuItem_Click(object sender, EventArgs e)\n {\n bool b = rap.Backup();\n MessageBox.Show(\"Backup done\");\n }\n\n private void restoreToolStripMenuItem_Click(object sender, EventArgs e)\n {\n rap.Restore();\n }\n\n public class objclass\n {\n public string val;\n }\n string prod3 = \"prod 3\";\n private void serverSideSumQueryToolStripMenuItem_Click(object sender, EventArgs e)\n {\n //string prod1 = \"prod 1\";\n objclass c = new objclass() { val = \"prod 3\" };\n //decimal i = 20;\n\n //var q = rap.Count(typeof(SalesItemRowsView), \n // (LineItem l) => (l.Product == prod1 || l.Product == prod3) && l.Price.Between(10,i)\n // );\n\n DateTime dt = FastDateTime.Now;\n\n var qq = rap.ServerSide(Views.ServerSide.Sum_Products_based_on_filter_args,\n //\"product = \\\"prod 1\\\"\"\n //(LineItem l) => (l.Product == c.val || l.Product == prod3 ) \n x => x.Product == c.val || x.Product == prod3\n ).ToList();\n dataGridView1.DataSource = qq;\n toolStripStatusLabel2.Text = \"Query time (sec) = \" + FastDateTime.Now.Subtract(dt).TotalSeconds;\n toolStripStatusLabel1.Text = \"Count = \" + qq.Count.ToString(\"#,0\");\n }\n\n private void KVHFtest()\n {\n //var r = (rap as RaptorDB.RaptorDB);\n var kv = rap.GetKVHF();\n\n int c = kv.CountHF();\n\n //if (c == 0)\n //{\n // DateTime dt = DateTime.Now;\n // for (int i = 0; i < 1000; i++)\n // {\n // var o = CreateInvoice(i);\n // kv.SetObjectHF(i.ToString(), o);// new byte[100000]);\n // }\n // MessageBox.Show(\"time = \" + DateTime.Now.Subtract(dt).TotalSeconds);\n //}\n //else\n //{\n // for(int i = 0; i < 1000; i++)\n // {\n // var o = (SalesInvoice) kv.GetObjectHF(\"\" + i);\n // var id = o.Serial;\n // if(id != i)\n // {\n // MessageBox.Show(\"not equal\");\n // break;\n // }\n // }\n //}\n if (c == 0)\n {\n kv.SetObjectHF(\"00\", 100);\n kv.SetObjectHF(\"01\", 101);\n }\n else\n {\n kv.SetObjectHF(\"00\", 102);\n }\n\n var g = kv.GetObjectHF(\"00\");\n\n //for (int i = 0; i < 100; i++)\n //kv.DeleteKeyHF(i.ToString());\n\n //g = kv.GetObjectHF(\"1009\");\n //MessageBox.Show(\"\"+kv.CountHF());\n\n //foreach (var f in Directory.GetFiles(\"d:\\\\pp\", \"*.*\"))\n //{\n //kv.SetObjectHF(f, File.ReadAllBytes(f));\n //}\n\n //kv.CompactStorageHF();\n\n //foreach (var f in Directory.GetFiles(\"d:\\\\pp\", \"*.*\"))\n //{\n // var o = kv.GetObjectHF(f);\n // File.WriteAllBytes(f.Replace(\"\\\\pp\\\\\", \"\\\\ppp\\\\\"), o as byte[]);\n //}\n //bool b = kv.ContainsHF(\"aa\");\n //var keys = kv.GetKeysHF();\n //foreach(var o in r.KVHF.EnumerateObjects())\n //{\n // string s = o.GetType().ToString();\n //}\n }\n\n class ppp\n {\n public int i = 100;\n }\n private void testToolStripMenuItem_Click(object sender, EventArgs e)\n {\n var tc = rap.Count(\"test\");\n if (tc == 0)\n {\n for (int i = 0; i < 3; i++)\n {\n var u = new TestData()\n {\n id = Guid.NewGuid(),\n username = \"\" + i,\n password = \"\" + i\n };\n rap.Save(u.id, u);\n }\n }\n var ppppp = rap.Query(x => x.username == \"1\" && x.password == \"1\");\n\n GC.Collect();\n var llll = rap.Query(x => x.CustomerName == \"ruth\" && x.Serial.Between(1600, 1630), 0, 100);\n\n KVHFtest();\n var ind = rap.Query(x =>\n x.Date.Year.In(2000)\n &&\n x.Date.Month.In(4));\n\n var id = new int[] { 20, 30, 40 };\n var iin = rap.Query(x => x.Serial.In(\n // new int[] { 20,30, 40} \n id\n //20,30,40\n ));\n var p = new ppp[2];\n p[0] = new ppp();\n p[1] = new ppp();\n var pp = new ppp();\n var d1 = DateTime.Parse(\"2001-1-1\");\n var d2 = DateTime.Parse(\"2010-1-1\");\n var ooooooooo = rap.Query(x => x.Date.Between(\"2001-1-1\", \"2010-1-1\") && x.Status == 2);\n ooooooooo = rap.Query(x => x.Date.Between(d1, d2) && x.Status == 2);\n ooooooooo = rap.Query(x => x.Serial.Between(1, 3));\n\n int cc = rap.Count(x => x.Serial < pp.i);\n var tt = p[1].i;\n cc = rap.Count(x => x.Serial < tt);\n\n var t = rap.Query(x => false);\n var ss = rap.FullTextSearch(\"woodland -oak\");\n\n int c = rap.Count(x => x.Serial < 100);\n c = rap.Count(x => x.Serial != 100);\n c = rap.Count(\"SalesInvoice\", \"serial != 100\");\n var q = rap.Query(x => x.Serial < 100, 0, 10, \"serial desc\");\n //var p = rap.Query(\"SalesInvoice\");\n //var pp = rap.Query(typeof(SalesInvoiceView));\n //var ppp = rap.Query(typeof(SalesItemRowsView.RowSchema));\n //var pppp = rap.Query(typeof(SalesInvoiceView), (SalesInvoiceView.RowSchema r) => r.Serial < 10);\n //var ppppp = rap.Query(typeof(SalesInvoiceView.RowSchema), (SalesInvoiceView.RowSchema r) => r.Serial < 10);\n //var pppppp = rap.Query(\"serial <10\");\n //Guid g = new Guid(\"82997e60-f8f4-4b37-ae35-02d033512673\");\n var qq = rap.Query(x => x.docid == new Guid(\"82997e60-f8f4-4b37-ae35-02d033512673\"));\n dataGridView1.DataSource = q.Rows;\n\n //int i = rap.ViewDelete(x => x.Serial == 0);\n //var qqq= rap.Query(x => );\n //SalesInvoiceViewRowSchema s = new SalesInvoiceViewRowSchema();\n //s.docid = Guid.NewGuid();\n //s.CustomerName = \"hello\";\n //rap.ViewInsert(s.docid, s);\n //q= rap.Query(\"serial <100\");\n //string s = q.Rows[0].CustomerName;\n\n //perftest();\n\n }\n\n private void frmMain_FormClosing(object sender, FormClosingEventArgs e)\n {\n shutdown();\n }\n\n private void freememoryToolStripMenuItem_Click(object sender, EventArgs e)\n {\n rap.FreeMemory();\n }\n\n private void dataGridView1_CellDoubleClick(object sender, DataGridViewCellEventArgs e)\n {\n var id = (Guid)dataGridView1.Rows[e.RowIndex].Cells[\"docid\"].Value;\n\n var s = fastJSON.JSON.ToNiceJSON(rap.Fetch(id), new fastJSON.JSONParameters { UseExtensions = false, UseFastGuid = false });\n\n MessageBox.Show(s, \"JSON Value\");\n }\n\n //private void perftest()\n //{\n // DateTime dt = DateTime.Now;\n\n // for (int i = 0; i < 100000; i++)\n // {\n // var s = new SalesInvoiceViewRowSchema();\n // s.docid = Guid.NewGuid();\n // s.Address = Faker.LocationFaker.Street();\n // s.CustomerName = Faker.NameFaker.Name();\n // s.Date = Faker.DateTimeFaker.BirthDay();\n // s.Serial = i % 1000;\n // s.Status = (byte)(i % 5);\n // rap.ViewInsert(s.docid, s);\n // }\n // MessageBox.Show(\"time = \" + DateTime.Now.Subtract(dt).TotalSeconds);\n //}\n }\n}" }, { "path": "datagridbinding/frmMain.resx", "content": "\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n text/microsoft-resx\n \n \n 2.0\n \n \n System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089\n \n \n System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089\n \n \n 17, 17\n \n \n 147, 17\n \n \n 263, 17\n \n" }, { "path": "datagridbinding/frmStartup.Designer.cs", "content": "namespace datagridbinding\n{\n partial class frmStartup\n {\n /// \n /// Required designer variable.\n /// \n private System.ComponentModel.IContainer components = null;\n\n /// \n /// Clean up any resources being used.\n /// \n /// true if managed resources should be disposed; otherwise, false.\n protected override void Dispose(bool disposing)\n {\n if (disposing && (components != null))\n {\n components.Dispose();\n }\n base.Dispose(disposing);\n }\n\n #region Windows Form Designer generated code\n\n /// \n /// Required method for Designer support - do not modify\n /// the contents of this method with the code editor.\n /// \n private void InitializeComponent()\n {\n this.groupBox1 = new System.Windows.Forms.GroupBox();\n this.label1 = new System.Windows.Forms.Label();\n this.button3 = new System.Windows.Forms.Button();\n this.txtFolder = new System.Windows.Forms.TextBox();\n this.groupBox2 = new System.Windows.Forms.GroupBox();\n this.txtPassword = new System.Windows.Forms.TextBox();\n this.label5 = new System.Windows.Forms.Label();\n this.txtUser = new System.Windows.Forms.TextBox();\n this.label4 = new System.Windows.Forms.Label();\n this.txtPort = new System.Windows.Forms.TextBox();\n this.label3 = new System.Windows.Forms.Label();\n this.txtServer = new System.Windows.Forms.TextBox();\n this.label2 = new System.Windows.Forms.Label();\n this.button1 = new System.Windows.Forms.Button();\n this.button2 = new System.Windows.Forms.Button();\n this.groupBox3 = new System.Windows.Forms.GroupBox();\n this.radioButton2 = new System.Windows.Forms.RadioButton();\n this.radioButton1 = new System.Windows.Forms.RadioButton();\n this.groupBox1.SuspendLayout();\n this.groupBox2.SuspendLayout();\n this.groupBox3.SuspendLayout();\n this.SuspendLayout();\n // \n // groupBox1\n // \n this.groupBox1.Controls.Add(this.label1);\n this.groupBox1.Controls.Add(this.button3);\n this.groupBox1.Controls.Add(this.txtFolder);\n this.groupBox1.Dock = System.Windows.Forms.DockStyle.Top;\n this.groupBox1.Location = new System.Drawing.Point(0, 45);\n this.groupBox1.Name = \"groupBox1\";\n this.groupBox1.Size = new System.Drawing.Size(341, 63);\n this.groupBox1.TabIndex = 0;\n this.groupBox1.TabStop = false;\n // \n // label1\n // \n this.label1.AutoSize = true;\n this.label1.Location = new System.Drawing.Point(6, 18);\n this.label1.Name = \"label1\";\n this.label1.Size = new System.Drawing.Size(67, 14);\n this.label1.TabIndex = 3;\n this.label1.Text = \"Data folder\";\n // \n // button3\n // \n this.button3.Location = new System.Drawing.Point(297, 35);\n this.button3.Name = \"button3\";\n this.button3.Size = new System.Drawing.Size(36, 22);\n this.button3.TabIndex = 2;\n this.button3.Text = \"...\";\n this.button3.UseVisualStyleBackColor = true;\n // \n // txtFolder\n // \n this.txtFolder.Location = new System.Drawing.Point(3, 35);\n this.txtFolder.Name = \"txtFolder\";\n this.txtFolder.Size = new System.Drawing.Size(288, 22);\n this.txtFolder.TabIndex = 1;\n this.txtFolder.Text = \"..\\\\RaptorDBdata\";\n // \n // groupBox2\n // \n this.groupBox2.Controls.Add(this.txtPassword);\n this.groupBox2.Controls.Add(this.label5);\n this.groupBox2.Controls.Add(this.txtUser);\n this.groupBox2.Controls.Add(this.label4);\n this.groupBox2.Controls.Add(this.txtPort);\n this.groupBox2.Controls.Add(this.label3);\n this.groupBox2.Controls.Add(this.txtServer);\n this.groupBox2.Controls.Add(this.label2);\n this.groupBox2.Dock = System.Windows.Forms.DockStyle.Top;\n this.groupBox2.Location = new System.Drawing.Point(0, 108);\n this.groupBox2.Name = \"groupBox2\";\n this.groupBox2.Size = new System.Drawing.Size(341, 137);\n this.groupBox2.TabIndex = 1;\n this.groupBox2.TabStop = false;\n this.groupBox2.Visible = false;\n // \n // txtPassword\n // \n this.txtPassword.Location = new System.Drawing.Point(79, 99);\n this.txtPassword.Name = \"txtPassword\";\n this.txtPassword.PasswordChar = '*';\n this.txtPassword.Size = new System.Drawing.Size(240, 22);\n this.txtPassword.TabIndex = 11;\n this.txtPassword.Text = \"admin\";\n // \n // label5\n // \n this.label5.AutoSize = true;\n this.label5.Location = new System.Drawing.Point(15, 102);\n this.label5.Name = \"label5\";\n this.label5.Size = new System.Drawing.Size(58, 14);\n this.label5.TabIndex = 10;\n this.label5.Text = \"Password\";\n // \n // txtUser\n // \n this.txtUser.Location = new System.Drawing.Point(79, 71);\n this.txtUser.Name = \"txtUser\";\n this.txtUser.Size = new System.Drawing.Size(240, 22);\n this.txtUser.TabIndex = 9;\n this.txtUser.Text = \"admin\";\n // \n // label4\n // \n this.label4.AutoSize = true;\n this.label4.Location = new System.Drawing.Point(12, 74);\n this.label4.Name = \"label4\";\n this.label4.Size = new System.Drawing.Size(61, 14);\n this.label4.TabIndex = 8;\n this.label4.Text = \"Username\";\n // \n // txtPort\n // \n this.txtPort.Location = new System.Drawing.Point(79, 43);\n this.txtPort.Name = \"txtPort\";\n this.txtPort.Size = new System.Drawing.Size(240, 22);\n this.txtPort.TabIndex = 7;\n this.txtPort.Text = \"90\";\n // \n // label3\n // \n this.label3.AutoSize = true;\n this.label3.Location = new System.Drawing.Point(38, 46);\n this.label3.Name = \"label3\";\n this.label3.Size = new System.Drawing.Size(30, 14);\n this.label3.TabIndex = 6;\n this.label3.Text = \"Port\";\n // \n // txtServer\n // \n this.txtServer.Location = new System.Drawing.Point(79, 15);\n this.txtServer.Name = \"txtServer\";\n this.txtServer.Size = new System.Drawing.Size(240, 22);\n this.txtServer.TabIndex = 5;\n this.txtServer.Text = \"127.0.0.1\";\n // \n // label2\n // \n this.label2.AutoSize = true;\n this.label2.Location = new System.Drawing.Point(31, 18);\n this.label2.Name = \"label2\";\n this.label2.Size = new System.Drawing.Size(42, 14);\n this.label2.TabIndex = 4;\n this.label2.Text = \"Server\";\n // \n // button1\n // \n this.button1.DialogResult = System.Windows.Forms.DialogResult.OK;\n this.button1.Location = new System.Drawing.Point(7, 254);\n this.button1.Name = \"button1\";\n this.button1.Size = new System.Drawing.Size(117, 34);\n this.button1.TabIndex = 2;\n this.button1.Text = \"Start\";\n this.button1.UseVisualStyleBackColor = true;\n this.button1.Click += new System.EventHandler(this.button1_Click);\n // \n // button2\n // \n this.button2.DialogResult = System.Windows.Forms.DialogResult.Cancel;\n this.button2.Location = new System.Drawing.Point(217, 254);\n this.button2.Name = \"button2\";\n this.button2.Size = new System.Drawing.Size(117, 34);\n this.button2.TabIndex = 3;\n this.button2.Text = \"Cancel\";\n this.button2.UseVisualStyleBackColor = true;\n this.button2.Click += new System.EventHandler(this.button2_Click);\n // \n // groupBox3\n // \n this.groupBox3.Controls.Add(this.radioButton2);\n this.groupBox3.Controls.Add(this.radioButton1);\n this.groupBox3.Dock = System.Windows.Forms.DockStyle.Top;\n this.groupBox3.Location = new System.Drawing.Point(0, 0);\n this.groupBox3.Name = \"groupBox3\";\n this.groupBox3.Size = new System.Drawing.Size(341, 45);\n this.groupBox3.TabIndex = 4;\n this.groupBox3.TabStop = false;\n // \n // radioButton2\n // \n this.radioButton2.AutoSize = true;\n this.radioButton2.Dock = System.Windows.Forms.DockStyle.Left;\n this.radioButton2.Location = new System.Drawing.Point(121, 18);\n this.radioButton2.Name = \"radioButton2\";\n this.radioButton2.Size = new System.Drawing.Size(94, 24);\n this.radioButton2.TabIndex = 2;\n this.radioButton2.Text = \"Server Mode\";\n this.radioButton2.UseVisualStyleBackColor = true;\n // \n // radioButton1\n // \n this.radioButton1.AutoSize = true;\n this.radioButton1.Checked = true;\n this.radioButton1.Dock = System.Windows.Forms.DockStyle.Left;\n this.radioButton1.Location = new System.Drawing.Point(3, 18);\n this.radioButton1.Name = \"radioButton1\";\n this.radioButton1.Size = new System.Drawing.Size(118, 24);\n this.radioButton1.TabIndex = 1;\n this.radioButton1.TabStop = true;\n this.radioButton1.Text = \"Embedded Mode\";\n this.radioButton1.UseVisualStyleBackColor = true;\n this.radioButton1.CheckedChanged += new System.EventHandler(this.radioButton1_CheckedChanged);\n // \n // Form2\n // \n this.AcceptButton = this.button1;\n this.AutoScaleDimensions = new System.Drawing.SizeF(7F, 14F);\n this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;\n this.CancelButton = this.button2;\n this.ClientSize = new System.Drawing.Size(341, 293);\n this.Controls.Add(this.button2);\n this.Controls.Add(this.button1);\n this.Controls.Add(this.groupBox2);\n this.Controls.Add(this.groupBox1);\n this.Controls.Add(this.groupBox3);\n this.Font = new System.Drawing.Font(\"Tahoma\", 9F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(178)));\n this.Name = \"Form2\";\n this.StartPosition = System.Windows.Forms.FormStartPosition.CenterScreen;\n this.Text = \"Startup\";\n this.groupBox1.ResumeLayout(false);\n this.groupBox1.PerformLayout();\n this.groupBox2.ResumeLayout(false);\n this.groupBox2.PerformLayout();\n this.groupBox3.ResumeLayout(false);\n this.groupBox3.PerformLayout();\n this.ResumeLayout(false);\n\n }\n\n #endregion\n\n private System.Windows.Forms.GroupBox groupBox1;\n private System.Windows.Forms.Button button3;\n private System.Windows.Forms.TextBox txtFolder;\n private System.Windows.Forms.GroupBox groupBox2;\n private System.Windows.Forms.Button button1;\n private System.Windows.Forms.Button button2;\n private System.Windows.Forms.Label label1;\n private System.Windows.Forms.TextBox txtPassword;\n private System.Windows.Forms.Label label5;\n private System.Windows.Forms.TextBox txtUser;\n private System.Windows.Forms.Label label4;\n private System.Windows.Forms.TextBox txtPort;\n private System.Windows.Forms.Label label3;\n private System.Windows.Forms.TextBox txtServer;\n private System.Windows.Forms.Label label2;\n private System.Windows.Forms.GroupBox groupBox3;\n private System.Windows.Forms.RadioButton radioButton2;\n private System.Windows.Forms.RadioButton radioButton1;\n }\n}" }, { "path": "datagridbinding/frmStartup.cs", "content": "using System;\nusing System.Windows.Forms;\nusing RaptorDB.Common;\nusing SampleViews;\n\nnamespace datagridbinding\n{\n public partial class frmStartup : Form\n {\n public frmStartup()\n {\n InitializeComponent();\n }\n\n public IRaptorDB _rap;\n\n private void button2_Click(object sender, EventArgs e)\n {\n this.Close();\n Application.Exit();\n }\n\n private void button1_Click(object sender, EventArgs e)\n {\n if (radioButton1.Checked)\n {\n var p = RaptorDB.RaptorDB.Open(txtFolder.Text);\n p.RegisterView(new SalesInvoiceView());\n p.RegisterView(new SalesItemRowsView());\n p.RegisterView(new newview());\n p.RegisterView(new TestView());\n _rap = p;\n }\n else\n {\n _rap = new RaptorDB.RaptorDBClient(txtServer.Text, int.Parse(txtPort.Text), txtUser.Text, txtPassword.Text);\n //fastBinaryJSON.BJSON.Parameters.v1_4TypedArray = true;\n }\n }\n\n private void radioButton1_CheckedChanged(object sender, EventArgs e)\n {\n if (radioButton1.Checked)\n {\n groupBox1.Visible = true;\n groupBox2.Visible = false;\n }\n else\n {\n groupBox1.Visible = false;\n groupBox2.Visible = true;\n }\n }\n }\n}\n" }, { "path": "datagridbinding/frmStartup.resx", "content": "\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n text/microsoft-resx\n \n \n 2.0\n \n \n System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089\n \n \n System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089\n \n" }, { "path": "history.txt", "content": "v4.0.10\n-------\n- upgrade to fastJSON v2.3.0\n- upgrade to fastBinaryJSON v1.6.0\n\nv4.0.9\n------\n- rewritten the web studio with Svelte\n\nv4.0.8\n------\n- bug fix .string index files growing over time\n- upgrade to fastJSON v2.2.5\n- upgrade to fastBinaryJSON v1.5.3\n\nv4.0.7\n------\n- bug fix StorageFileHF reloading free list from previous failed shutdown\n\nv4.0.6\n------\n- upgrade to fastBinaryJSON v1.5.2\n- bug fix loading empty hOOt words file\n- bug fix query MGRB index with less than 64 rows \n\nv4.0.5\n------\n- bug fix MGRB.Fill(length) respect length value\n- bug fix MGRB.CountOnes() when length = 0\n- MGRB added locks for integrity\n- MGRB added InvertedContainer for better memory usage\n- MGRB respect lengths in computations\n- truncate StorageFileHF end free list data on read\n\nv4.0.2\n------\n- bug fix MGRB.Fill() \n\nv4.0.1\n------\n- bug fix missing raptordb.version file in data folder\n\nv4.0.0\n-------\n- upgrade to fastJSON v2.2.0\n- upgrade to fastBinaryJSON v1.5.1\n- changed to faster unicode <-> bytes conversion\n- optimized view data bytes size without bjson typed arrays\n- new sparse bitmap index for less memory usage (MGRB based on roaring bitmap)\n- * possible breaking change if you are using High Frequency key store *\n- bug fix full text search in columns for case insensitive words\n- parallel shutdown indexes for better performance\n- fixed .strings files kept growing\n\nv3.3.19.1\n---------\n- bug fix shutting down in .net core on ubuntu\n\nv3.3.19\n-------\n- upgrade to fastJSON v2.1.28\n- upgrade to fastBinaryJSON v1.4.21\n- support for .netcore 2.0 and netstandard 2.0\n\nv3.3.18\n-------\n- upgrade to fastJSON v2.1.26\n- upgrade to fastBinaryJSON v1.4.20\n- possible edge case currupt deleted rows index on FreeMemory() fix\n- \n\nv3.3.17\n-------\n- added ITokenizer full text search overload to RaptorDB.Open()\n- tokenizer rewrite\n- bug fix HFKV rebuild \n\nv3.3.16\n-------\n- bug fix HFKV rebuild \n\nv3.3.15\n-------\n- upgrade to fastBinaryJSON v1.4.19\n- bug fix logger zip file names\n- bug fix local/UTC time in FastDateTime\n- changed to public WAHBitArray\n- changed to public Hoot\n- changed to public CronDaemon\n- Zipstorer default to UTF8 filenames\n- bug fix ViewInsert() and rebuild index *old data is not recoverable, rebuild your database if you can* \n- remove query bitmap cache because of incorrect results\n- upgrade to fastJSON v2.1.25\n- bug fix HFKV store rebuild on failed shutdown\n\nv3.3.14\n-------\n- upgrade to fastJSON v2.1.21\n- upgrade to fastBinaryJSON v1.4.18\n- bug fix doc full text search results < paging count\n\nv3.3.13\n-------\n- bug fix x=>true predicate case\n- upgrade to fastJSON v2.1.19\n- upgrade to fastBinaryJSON v1.4.16\n- bug fix string keys indexes with Global.EnableOptimizedStringIndex = true\n- ** incremented RaptorDBVersion ** will rebuild old index data if exists\n- bug fix WAHBitArray.Fill()\n\nv3.3.12\n-------\n- added In() e.g. x => x.Serial.In(1,3,5,7) or \"serial.in(1,3,5,7)\"\n- query cache for Count() and all other overloads\n- code refactoring/cleanup in ViewHandler.cs\n- date parts work with In() e.g. x => x.Date.Year.In(2000,2001)\n- date parts work with Between() e.g. x => x.Date.Year.Between(2000,2010)\n\nv3.3.11\n-------\n- added query bitmap caching for faster queries\n\nv3.3.10\n-------\n- code cleanup\n- generic Between()\n- bug fix BitmapIndex.FreeMemory() \n- bug fix MGIndex.FreeMemory()\n- added generic Fetch() (thanks to Norbert Haberl)\n- added ZipStorer from [https://github.com/jaime-olivares/zipstorer] (thanks to Jaime Olivares)\n- added auto compress old log files\n\nv3.3.9\n------\n- query date parts e.g. x => x.Date.Year == 2016 or \"date.year = 2016\"\n- Between() for dates e.g. x => x.Date.Between(\"2000-1-1\",\"2002-6-16\")\n- optimized internal query engine for ranges\n- internal query from to implementation \n- Between() for int, long, decimal\n\nv3.3.8\n------\n- synced with hOOt\n- added nscript.exe tool\n- added sample.cs test script with command line run\n- changed log message types\n- bug fix fulltext search \n- fulltext tokenizer breaks a.b.c words and numbers\n\nv3.3.6\n------\n- upgrade to fastJSON v2.1.18\n- upgrade to fastBinaryJSON v1.4.15\n- changed default view.ConsistentSaveToThisView = true\n- changed default view.isPrimaryList = true\n- re-factored web server\n- restructured nav.js\n\nv3.3.5\n------\n- bug fix != in query return all rows if RH expression not found (thanks to Lutz Wellhausen)\n- linq (x => true) predicate returns all rows\n- linq (x => false) predicate returns no rows\n\nv3.3.4\n------\n- streamlined shutdown process (no extra index saves)\n- web ui content height set to browser height\n- memory usage reduction in BimapIndex (removed record cache)\n- memory usage reduction in WAHBitArray (offset Dictionary -> SortedList)\n- memory usage reduction in MGIndex (cache Dictionary -> SortedList)\n- memory usage reduction in Hoot (words Dictionary -> SortedList)\n- log error if web server started when not run as administrator\n- frmMain form close handles shutdown correctly\n- changed default Globals.FreeMemoryTimerSeconds to 5 mins from 30 mins\n\nv3.3.3\n------\n- bug fix SafeSortedList.Add()\n- bug fix WAHBitArray.FreeMemory()\n\nv3.3.2\n------\n- bug fix edge case duplicates showing in queries \n\nv3.3.1\n------\n- bug fix mixed property and fields in view schema definition (invalid cast exception on query)\n\nv3.3.0\n------\n- added WEB Studio interface\n- logger keeps last 100 logs for recall\n- bug fix HF storage file seek position overflow \n\nv3.2.18\n-------\n- usings cleanup\n- FreeMemory() in indexes, saves data first\n- added indexed word list to hOOt index\n- hOOt optimize index saves data first\n- hOOt optimize index will block until done\n- hOOt FindDocuments() is now generic\n- added server side function with object[] args\n- added auto build number incrementor\n- bug fix not equal != convert linq to string in queries\n- upgrade to fastJSON v2.1.15\n- upgrade to fastBJSON v1.4.12\n\nv3.2.17\n-------\n- bug fix empty rows in GenericResult() in client/server mode\n- bug fix if property/field defined in schema but not in entity hence null error when getting rows\n\nv3.2.16\n-------\n- added support for ushort in views (thanks to dragosc)\n- bug fix FetchHistory() return values in server mode (thanks to DrDeadCrash)\n- bug fix edge case bitmap index commits (thanks to dragosc)\n- bug fix hOOt fulltext index shutdown\n- bug fix FullTextSerach() to filter out json characters before indexing\n\nv3.2.15\n-------\n- *breaking change* + in full text search means OR now\n- full text search bug fix edge case\n\nv3.2.14\n-------\n- server keeps track of connected clients count with auto clear every 30 secs\n- added atomic Increment() Decrement() to HFKV int,decimal values\n- bug fix wildcard search in full text indexes\n- added not wildcard search in full text -> \"-oak -*l\"\n- fulltext indexes break text on spaces and char.IsPunctuation()\n- api.NextRowNumber() blocks until rebuild finishes\n- view map functions has access to high frequency KV through the api interface\n\nv3.2.13\n-------\n- code refactoring\n- bug fix full text index search with leading not \"-oak hill\"\n- fix time output in logs\n- upgrade to fastJSON v2.1.14\n- upgrade to fastBinaryJSON v1.4.11\n\nv3.2.12\n-------\n- code cleanup\n- bug fix full text searching with + - prefixes\n- upgrade to fastJSON v2.1.13\n- upgrade to fastBinaryJSON v1.4.10\n\nv3.2.11\n-------\n- bug fix sorting cache\n\nv3.2.10\n-------\n- renamed Form1 to frmMain\n- added sortable fulltext indexes\n- fixed path names for linux systems\n- changed default save and free memory timers to 30 min instead of 60 sec\n- optimized query sorting with internal cache ~100x faster\n\nv3.2.9\n------\n- upgrade to fastJSON v2.1.11\n- upgrade to fastBinaryJSON v1.4.8\n- added support for vb.net string linq queries \n- added vb test project\n\nv3.2.8\n------\n- bug fix duplicates showing in queries related to the deleted bitmap index \n\nv3.2.7\n------\n- bug fix wait on view rebuild while Shutdown() was being cut off in 2 secs mid process (ProcessExit)\n\nv3.2.6\n------\n- optimizations done by Stainslav Lukeš\n- upgrade to fastJSON v2.1.10\n- upgrade to fastBinaryJSON v1.4.7\n- bug fix bitmap indexes \n- bug fix file name conflicts with deleted bitmap indexes\n- added version checking of views and RaptorDB engine with auto rebuild for engine upgrades\n- changed deleted bitmap indexes to .deleted extension\n- changed version number files to text mode with .version extension\n\nv3.2.5\n------\n- new optimized storage for string key MGIndex file \n- added Global.EnableOptimizedStringIndex flag to control the new index usage\n\nv3.2.0\n------\n- you can compress the documents in the storage file with Global.CompressDocumentOverKiloBytes configuration\n- Upgrade to fastJSON v2.1.9\n- added integrity check for views with auto rebuild if not shutdown cleanly\n- bug fix disable timers before Shutdown()\n- added high frequency update key/value storage file\n\n\nv3.1.6\n------\n- document storage files can now be split with Global.SplitStorageFilesMegaBytes configuration\n- refactoring StorageFile.cs\n- Upgrade to fastJSON v2.1.8\n- Upgrade to fastBinaryJSON v1.4.6\n- bug fix .config files were not saved correctly\n\nv3.1.5\n------\n- added View.NoIndexingColumns definition to override indexing of selected columns\n- Upgrade to fastJSON v2.1.7\n- Upgrade to fastBinaryJSON v1.4.5\n- added DocumentCount() to get how many items in the storage file\n- Shutdown() now waits for View rebuilds to finish\n- more intellisense help\n\n\nv3.1.4\n------\n- added StringIndexLength attribute for view schema to control string index size for the index file\n- added ViewDelete() to delete directly from views \n- added ViewInsert() to insert directly into views\n- added Faker.dll (http://faker.codeplex.com) to generate nicer data\n- FreeMemory() will save indexes to disk also\n- moved server mode files to output\\server so you don't get conflicts loading views.dll\n- page list is also saved to disk on SaveIndex()\n- bug fix view schema when not inheriting from RDBSchema\n- replaced T with more meaningful TRowSchema in code intellisense \n\nv3.1.3\n------\n- added FetchHistoryInfo() and FetchBytesHistoryInfo() with date change information\n- added api.NextRowNumber() \n- moved all config files to the data folder which you should have write access to (thanks to Detlef Kroll)\n- bug fixed delete before insert with no rows \n\nv3.1.2\n------\n- Upgrade to fastJSON v2.1.1\n- Upgrade to fastBinaryJSON v1.4.1\n- bug fixes in WAH and Query2 from Richard Hauer\n- changed all singleton implementations\n- bug fix indexing String.Empty\n- *breaking change* removed FireOnType from view definitions\n- Views can now correctly work with subclass of the T defined (i.e. SpecialInvoice : Invoice)\n- bug fix index bitmap.Not(size)\n\nv3.1.1\n------\n- added signed assemblies the assembly version will stay at 3.0.0.0 and the file version will increment\n- added nuget build \n\nv3.1.0\n------\n- added sort for queries \n- removed extra query overloads in favour of the new model\n\nv3.0.6\n------\n- Result.TotalCount reflects the original row count and differs from Result.Count when paging\n- internal changed FireOnType to handle Type instead of strings\n- Query() can now handle empty filter strings correctly\n- Upgrade to fastJSON v2.0.24\n- Upgrade to fastBinaryJSON v1.3.11\n\nv3.0.5\n------\n- bug fix saving page list to disk for counts > 50 million items\n\nv3.0.1\n------\n- upgrade to fastJSON v2.0.22\n- upgrade to fastBinaryJSON v1.3.10\n- detect process exit and shutdown cleanly so you can omit the explicit Shutdown()\n- bug fix WAH bitarray \n\nv3.0.0\n------\n- index files are opened in share mode for online copy\n- add cron daemon (thanks to Kevin Coylar)\n- backups are now on a cron schedule\n- restructured storage file for future proofing and replication support\n- storage files now store meta data about objects stored\n- * storage files are not backward compatible *\n- dirty index pages are sorted on save for read performance\n- restore is now resumable after a shutdown\n- you can disable the primary view to be defined on save with Global.RequirePrimaryView (K/V mode) \n- view rebuilds are now done in the background (non-blocking on restart)\n- you can define views in c# script format (*.view) to be compiled at runtime in 'datafolder\\views'\n- row schema defined in script views will be transferred to the client if they don't exist\n- fastJSON now serializes static properties\n- upgrade to fastJSON v2.0.18\n- upgrade to fastBinaryJSON v1.3.8\n- added HQ-Branch replication feature\n- automatic generate config files if they don't exist with a '-' prefix\n- 'output' in the root of the solution folder is the new build destination of projects for easy access\n\nv2.0.6.1\n--------\n- bug fix WAHBitArray\n\nv2.0.6\n------\n- bug fix WAHBitArray\n- upgrade to fastJSON v2.0.15\n- bug fix hoot fulltext index on last word \n- save deleted items bitmap on save timer\n\nv2.0.5\n------\n- added FreeMemory to classes\n- memory limiting and free memory timer added\n- views background save indexes to disk on timer\n- fixed RaptorDBServer.csproj to AnyCPU build\n\nv2.0.0\n------\n- added more method documentations\n- * breaking change in doc storage file from hashed guid to guid keys *\n- added FetchHistory() and FetchVersion() for docs and files to get revisions \n- upgrade to fastJSON v2.0.14\n- upgrade to fastBinaryJSON v1.3.7\n- full text indexing and search for the entire original document\n- bug fix linq query with boolean parameter\n\nv1.9.2\n------\n- SafeDictionary.Add() will update if item exists\n- BitmapIndex using new lock mechanism\n- CaseInsensitive attribute\n- bug fix lowercase hoot indexing\n- case insensitive string indexing and searching\n- nocase samples\n- fixed handling != (not equal) in linq query\n\nv1.9.1\n------\n- bug fix edge case WAHBitarray\n- sync code with changes in hOOt\n- bug fix missing server mode SaveBytes()\n- bug fix server side queries in server mode\n- bug fix embedded guid in query : v => v.docid == new Guid(\"...\")\n\nv1.9.0\n------\n- speed increase writing bitmap indexes to disk\n- bug fix hoot search with wildcards\n- bug fix datetime indexing with UTC time (all times are localtime)\n- upgrade to fastJSON v2.0.9\n- upgrade to fastBinaryJSON v1.3.5\n- changed CodeDOM to Reflection.Emit for MonoDroid compatibility\n- more optimized bitmap storage format (save offsets if smaller than WAH)\n- fixed path seperator character for monodroid and windows compatibility changed to Path.DirectorySeparatorChar \n- new generic Query interface with typed results (thanks to seer_tenedos2)\n- changed to Result\n- WAH bitcount speed increase\n- bitmap index uses buffered stream for speed\n- added between query (work in progress)\n- bug fix storage file and deleted items\n- new query model for mapper api interface\n- you can now define your own schema for rows with caveats\n- bug fix NOT on bitmap indexes to resize to the total row count first\n- when defining your own schema you can define the fulltext columns in the view without attributes\n\nv1.8.3\n------\n- upgrade to fastJSON v2.0.6\n- upgrade to fastBinaryJSON v1.3.4\n- bug fix linq2string with date,guid parameters\n- added double,float types to the indexer valid data types\n- added a lock to the IndexFile for concurrency issues (thanks to Antonello Oliveri)\n- fixed lock on _que in the logger for concurrency (thanks to Antonello Oliveri)\n- fixed the reflection binding to the insert method (thanks to Antonello Oliveri)\n- added Count() on views\n- added support for paging of results\n- the mapper can now see changes it has made in it's own thread in transaction mode while quering\n\nv1.8.2\n------\n- bug fix linq binding -> c.val == obj.property (Thanks to Joe Dluzen) \n- added lock to the bitmap index for concurrency\n- optimized $types output in JSON and BJSON\n- bug fix null check for SafeSortedList.Remove\n- bug fix server mode data transfer\n\nv1.8.1\n------\n- speed increase WAH bitmap Set() code \n- bug fix concurrent save bitmap index to disk\n- upgrade to fastBinaryJSON v1.3\n- upgrade to fastJSON v2.0.1\n\nv1.8\n----\n- upgrade to fastBinaryJSON v1.2 with struct support\n- upgrade to fastJSON v1.9.9 with struct support\n- bug fix hoot index loadwords when file size is zero\n- bug fix linq binding ServerSide -> c.val == stringvariable\n- bug fix linq binding -> c.val == stringvariable\n- bug fix reflection code in serializers\n\nv1.7\n----\n- server side aggregate queries\n- fixed the build script for views to copy the dll to the extensions folder\n- server side queries can have filters\n- login form default buttons fix\n\nv1.6\n----\n- query lambda caching\n- transaction support\n- bug in datetime serialization\n\nv1.5\n----\n- compressing network traffic over Param.CompressDataOver limit with MiniLZO\n- added Delete(docid) and DeleteBytes(fileid)\n- added ability to query Guid and DateTime in string form\n- bug fix reading boolean indexes\n- rebuild view and background indexer handles deleted docs\n- added authentication via users.config file in server mode\n- Backup & Restore data\n- AddUser() method for user\n- handle isDeleted when restoring data and rebuilding View\n- Auto backup in server mode @ 00:00 time\n\nv1.4\n----\n- break up the source into projects\n- created client, server dlls\n- upgrade to fastBinaryJSON v1.1\n- changed to SafeSortedList for thread safe indexes\n- add auto installer RaptorDBServer service\n- performance optimized tcp network layer\n- added dual mode usage to the windows application (embedded, server)\n- code cleanup\n- added IRaptorDB interface to allow you to switch between embedded and client seamlessly\n- load views from the Extensions folder in server mode\n\nv1.3\n----\n- Results.Rows are now row schema objects and bindable (even when fields)\n- View.Schema must now derive from RaptorDB.RDBSchema\n- removed columns from Result (not needed anymore)\n- RegisterView throws exceptions instead of returning a Result\n- added a rudimentary query viewer project\n- null values are ignored when indexing\n- bool index filename will end in \".idx\"\n- sample apps will create data files in the main soultion folder for easy sharing\n- you can now do aggregate queries on the results on the client side\n- added api.EmitObject for easier mapping (less code to write)\n- upgrade to fastJSON v1.9.8\n- bug fix datetime in fastBinaryJSON\n- added ConsistentView \n\nv1.2\n----\n- View versioning and rebuild\n- code cleanup\n- removed indent logic from fastJSON\n- added schema of the query to the Result \n\nv1.1\n----\n- fulltext indexing via attribute\n- string query parser \n- fix shutdown flusing indexes to disk\n- rudimentary console application\n- lowercase viewnames for string queries\n- fulltext search defaults to AND if + - characters not present in query\n- Query now works when suppling the view type\n- save pauses indexer for better insert performance ~30% faster\n\nv1.0\n----\n- initial release" }, { "path": "test script/run.cmd", "content": "..\\tools\\nscript.exe sample.cs" }, { "path": "test script/sample.cs", "content": "// ref : ..\\output\\raptordb.dll\n// ref : ..\\output\\raptordb.common.dll\n// ref : ..\\faker.dll\nusing System;\nusing System.Collections.Generic;\nusing System.Linq;\nusing System.Text;\nusing System.Threading.Tasks;\n\nusing RaptorDB;\nusing RaptorDB.Common;\n\nnamespace rdbtest\n{\n\n #region [ entities ]\n public class LineItem\n {\n public decimal QTY { get; set; }\n public string Product { get; set; }\n public decimal Price { get; set; }\n public decimal Discount { get; set; }\n }\n\n public class SalesInvoice\n {\n public SalesInvoice()\n {\n ID = Guid.NewGuid();\n }\n\n public Guid ID { get; set; }\n public string CustomerName { get; set; }\n public string NoCase { get; set; }\n public string Address { get; set; }\n public List Items { get; set; }\n public DateTime Date { get; set; }\n public int Serial { get; set; }\n public byte Status { get; set; }\n public bool Approved { get; set; }\n }\n #endregion\n\n #region [ view definition ]\n public class SalesInvoiceViewRowSchema : RDBSchema\n {\n public string CustomerName;\n public string NoCase;\n public DateTime Date;\n public string Address;\n public int Serial;\n public byte Status;\n public bool? Approved;\n }\n\n [RegisterView]\n public class SalesInvoiceView : View\n {\n public SalesInvoiceView()\n {\n this.Name = \"SalesInvoice\";\n this.Description = \"A primary view for SalesInvoices\";\n this.isPrimaryList = true;\n this.isActive = true;\n this.BackgroundIndexing = true;\n this.Version = 1;\n\n this.Schema = typeof(SalesInvoiceViewRowSchema);\n\n this.Mapper = (api, docid, doc) =>\n {\n //if (doc.Status == 0)\n // return;\n\n api.EmitObject(docid, doc);\n };\n }\n }\n #endregion\n\n class Program\n {\n static RaptorDB.RaptorDB rdb; // 1 instance\n\n static void Main(string[] args)\n {\n rdb = RaptorDB.RaptorDB.Open(\"data\"); // a \"data\" folder beside the executable\n RaptorDB.Global.RequirePrimaryView = false;\n\n Console.WriteLine(\"Registering views..\");\n rdb.RegisterView(new SalesInvoiceView());\n\n DoWork();\n\n Console.WriteLine(\"press any key...\");\n Console.ReadKey();\n Console.WriteLine(\"\\r\\nShutting down...\");\n rdb.Shutdown(); // explicit shutdown\n }\n\n static void DoWork()\n {\n long c = rdb.DocumentCount();\n if (c > 0) // not the first time running\n {\n var result = rdb.Query(x => x.Serial < 100);\n // show the rows\n Console.WriteLine(fastJSON.JSON.ToNiceJSON(result.Rows, new fastJSON.JSONParameters { UseExtensions = false, UseFastGuid = false }));\n // show the count\n Console.WriteLine(\"Query result count = \" + result.Count);\n return;\n }\n\n Console.Write(\"Inserting 100,000 documents...\");\n int count = 100000;\n\n for (int i = 0; i < count; i++)\n { \n var inv = CreateInvoice(i);\n\n // save here\n rdb.Save(inv.ID, inv);\n }\n\n Console.WriteLine(\"done.\");\n }\n\n static SalesInvoice CreateInvoice(int counter)\n {\n // new invoice\n var inv = new SalesInvoice()\n {\n Date = Faker.DateTimeFaker.BirthDay(),\n Serial = counter % 10000,\n CustomerName = Faker.NameFaker.Name(),\n NoCase = \"Me \" + counter % 10,\n Status = (byte)(counter % 4),\n Address = Faker.LocationFaker.Street(),\n Approved = counter % 100 == 0 ? true : false\n };\n // new line items\n inv.Items = new List();\n for (int k = 0; k < 5; k++)\n inv.Items.Add(new LineItem() { Product = \"prod \" + k, Discount = 0, Price = 10 + k, QTY = 1 + k });\n\n return inv;\n }\n }\n}" }, { "path": "testing/AssemblyInfo.cs", "content": "using System.Reflection;\nusing System.Security;\n\n[assembly: AssemblyTitle(\"\")]\n[assembly: AssemblyDescription(\"\")]\n[assembly: AssemblyConfiguration(\"\")]\n[assembly: AssemblyCompany(\"\")]\n[assembly: AssemblyProduct(\"\")]\n[assembly: AssemblyCopyright(\"\")]\n[assembly: AssemblyTrademark(\"\")]\n[assembly: AssemblyCulture(\"\")]\n[assembly: AssemblyVersion(\"1.0.*\")]\n[assembly: AssemblyDelaySign(false)]\n[assembly: AssemblyKeyFile(\"\")]\n[assembly: AssemblyKeyName(\"\")]\n" }, { "path": "testing/Class1.cs", "content": "using System;\nusing System.Diagnostics;\nusing System.Collections;\nusing System.IO;\nusing System.Text;\nusing System.Threading;\nusing RaptorDB;\nusing System.Collections.Generic;\nusing System.Runtime.InteropServices;\nusing System.Reflection;\nusing RaptorDB.Views;\nusing System.Linq.Expressions;\nusing System.Linq;\nusing System.Collections.ObjectModel;\nusing System.Dynamic;\nusing RaptorDB.Common;\nusing System.Threading.Tasks;\n\n\nnamespace testing\n{\n //class Program3\n //{\n // static int waitTime = 5;\n // static string dataPath = \"_data\";\n // static RaptorDB.RaptorDB DB = null;\n // static Sample[] SampleArray = \n // {\n // new Sample\n // {\n // ID = Guid.NewGuid(),\n // Name = \"Object 1\",\n // OtherValue = \"other value for object 1\"\n // },\n // new Sample\n // {\n // ID = Guid.NewGuid(),\n // Name = \"Object 2\",\n // OtherValue = \"other value for object 2\"\n // }\n // };\n\n // public static void Main3(string[] args)\n // {\n // //start with a clean slate\n // if (Directory.Exists(dataPath))\n // {\n // Console.WriteLine(\"Delete database directory if it already exists\");\n // Directory.Delete(dataPath, true);\n // }\n // DBInit();\n\n // //Write initial data and verify\n // WriteSampleData();\n // Wait();\n // ReadAllObjects();\n\n // //Delete 1 object and verify\n // if (DB.Delete(SampleArray[0].ID))\n // Console.WriteLine(\"Deleted {0}\", SampleArray[0].Name);\n // else\n // {\n // Console.WriteLine(\"Unable to delete object\");\n // return;\n // }\n // Wait();\n // ReadAllObjects();\n\n // //Remove view folder and demonstrate the bug\n // RemoveViewFolder();\n // Wait();\n // ReadAllObjects();\n\n // Console.Write(\"Press any key to continue . . . \");\n // Console.ReadKey(true);\n // }\n\n // static void DBInit()\n // {\n // Console.WriteLine(\"Initialise database\");\n // DB = RaptorDB.RaptorDB.Open(dataPath);\n // DB.RegisterView(new SampleView());\n // }\n\n // static void WriteSampleData()\n // {\n // foreach (Sample obj in SampleArray)\n // {\n // Console.WriteLine(\"{0} : {1} : {2}\", obj.ID.ToString(), obj.Name, obj.OtherValue);\n // DB.Save(obj.ID, obj);\n // }\n // }\n\n // static void ReadAllObjects()\n // {\n // Console.WriteLine(\"Read back all data from DB\");\n\n // var list = DB.Query(\"SampleView\");\n // foreach (SampleView.RowSchema row in list.Rows)\n // {\n // Console.WriteLine(\"ViewRow {0}: {1}\", row.docid.ToString(), row.Name);\n // Sample obj = (Sample)DB.Fetch(row.docid);\n // if (obj != null)\n // Console.WriteLine(\"Object {0} : {1} : {2}\", obj.ID.ToString(), obj.Name, obj.OtherValue);\n // else\n // Console.WriteLine(\"Can't retrieve original Object\");\n\n // Console.WriteLine(\"\");\n // }\n // }\n\n // static void Wait()\n // {\n // Console.Write(\"\\nWait a few seconds to make sure records are written\");\n // for (int i = 0; i < waitTime; i++)\n // {\n // Console.Write(\".\");\n // System.Threading.Thread.Sleep(1000);\n // }\n // Console.WriteLine(\"\\n\");\n // }\n\n // static void RemoveViewFolder()\n // {\n // Console.WriteLine(\"Close connection to DB\");\n // DB.Shutdown();\n // DB = null;\n\n // Console.WriteLine(\"Remove 'Views' folder from DB folder to force the views to be re-populated\");\n // Directory.Delete(dataPath + @\"\\Views\", true);\n\n // Console.WriteLine(\"Re-connect to database\");\n // //Re-initialise DB\n // DBInit();\n // }\n //}\n\n //public class Sample\n //{\n // public Guid ID;\n // public string Name;\n // public string OtherValue;\n\n // public Sample()\n // {\n // }\n //}\n\n //public class SampleView : View\n //{\n // public class RowSchema : RDBSchema\n // {\n // public string Name;\n // }\n\n // public SampleView()\n // {\n // this.Name = \"SampleView\";\n // this.Description = \"A primary view for sample objects\";\n // this.isPrimaryList = true;\n // this.isActive = true;\n // this.BackgroundIndexing = false;\n\n // this.Schema = typeof(SampleView.RowSchema);\n\n // this.AddFireOnTypes(typeof(Sample));\n\n // this.Mapper = (api, docid, doc) =>\n // {\n // api.EmitObject(docid, doc);\n // };\n // }\n //}\n \n /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////\n\n /*\n public class program2\n {\n static RaptorDB.RaptorDB rd;\n public static void Main2(string[] args)\n {\n if (args.Length < 2)\n {\n System.Console.WriteLine(\"Params: numthreads insertsforthreads (numthreads = 0 for insertion in the main thread )\");\n return;\n }\n DateTime dt = FastDateTime.Now;\n int maxThread = int.Parse(args[0]);\n int maxDataToInsert = int.Parse(args[1]);\n RepositoryStart();\n System.Console.WriteLine(\"Starting inserts \" + maxDataToInsert + \" objects for \" + maxThread + \" thread/s....\");\n if (maxThread == 0)\n InsertInvoice(maxDataToInsert);\n else\n {\n System.Threading.Tasks.Task[] tasks = new System.Threading.Tasks.Task[maxThread];\n Action act = InsertInvoice;\n\n for (int i = 0; i < maxThread; i++)\n {\n tasks[i] = System.Threading.Tasks.Task.Factory.StartNew(act, maxDataToInsert);\n }\n Task.WaitAll(tasks);\n }\n //System.Console.Write((maxDataToInsert * (maxThread == 0 ? 1 : maxThread)).ToString(\"#,###\") + \" inserted press a key to continue... \");\n //System.Console.ReadLine();\n RepositoryStop();\n System.Console.WriteLine(\"Re-opening RaptorDB for objects count....\");\n RepositoryStart();\n System.Console.WriteLine(rd.Count(typeof(InvoiceView)));\n RepositoryStop();\n Console.WriteLine(\"time = \" + FastDateTime.Now.Subtract(dt).TotalSeconds);\n return;\n }\n\n\n public static void RepositoryStart()\n {\n rd = RaptorDB.RaptorDB.Open(@\"C:\\temp\\RaptorDBData\\DMDATA\");\n System.Console.WriteLine(\"Registering indexes....\");\n rd.RegisterView(new InvoiceView());\n rd.RegisterView(new InvoiceAdditionalView());\n rd.RegisterView(new InvoiceItemView());\n System.Console.WriteLine(\"RaptorDB is Up....\");\n }\n\n public static void RepositoryStop()\n {\n System.Console.WriteLine(\"RaptorDB shutdown....\");\n rd.Shutdown();\n rd.Dispose();\n rd = null;\n System.Console.WriteLine(\"RaptorDB halted....\");\n }\n\n\n\n public static void InsertInvoice(object maxDataToInsert)\n {\n System.Console.WriteLine(\"Thread \" + System.Threading.Thread.CurrentThread.ManagedThreadId + \" started...\");\n int maxVal = (int)maxDataToInsert;\n for (int i = 1; i <= maxVal; i++)\n {\n\n Invoice invoice = new Invoice();\n invoice.UNIQUEID = Guid.NewGuid();\n invoice.Customer = \"Customer1\";\n invoice.InvoiceNumber = 10;\n invoice.InvoiceDate = DateTime.Now;\n invoice.items = new List();\n invoice.items.Add(new InvoiceItem() { SKU = \"ART\" + i.ToString(), qty = 10.50M, UnitPrice = i, TotalPrice = (10.50M * i) });\n rd.Save(invoice.UNIQUEID, invoice);\n // System.Console.WriteLine(\" Writing Thread \" + System.Threading.Thread.CurrentThread.ManagedThreadId.ToString() + \" Inserted doc \" + i.ToString());\n }\n System.Console.WriteLine(\"Thread \" + System.Threading.Thread.CurrentThread.ManagedThreadId + \" ended with \" + maxVal + \" objects\");\n }\n\n public static void SearchInvoice()\n {\n int count = rd.Query(typeof(InvoiceView)).Count;\n }\n\n\n\n public class BaseStore\n {\n public Guid UNIQUEID { get; set; }\n\n }\n\n\n\n public class Invoice : BaseStore\n {\n public DateTime InvoiceDate { get; set; }\n public String Customer { get; set; }\n public int InvoiceNumber { get; set; }\n public List items { get; set; }\n public String TAG { get; set; }\n }\n\n\n public class InvoiceItem : BaseStore\n {\n public string SKU { get; set; }\n public Decimal qty { get; set; }\n public Decimal UnitPrice { get; set; }\n public Decimal TotalPrice { get; set; }\n }\n\n\n\n\n public class InvoiceView : RaptorDB.View\n {\n public class RowSchema : RaptorDB.RDBSchema\n {\n public DateTime InvoiceDate { get; set; }\n public String Customer { get; set; }\n public int InvoiceNumber { get; set; }\n\n }\n\n public InvoiceView()\n {\n this.Name = \"InvoiceView\";\n this.Schema = typeof(InvoiceView.RowSchema);\n this.isActive = true;\n this.isPrimaryList = true;\n this.ConsistentSaveToThisView = true;\n this.AddFireOnTypes(typeof(Invoice));\n this.Mapper = (api, docid, doc) => { api.Emit(docid, doc.InvoiceDate, doc.Customer, doc.InvoiceNumber); };\n }\n }\n\n public class InvoiceAdditionalView : RaptorDB.View\n {\n public class RowSchema : RaptorDB.RDBSchema\n {\n public String TAG { get; set; }\n }\n\n public InvoiceAdditionalView()\n {\n this.Name = \"InvoiceAdditionalView\";\n this.Schema = typeof(InvoiceView.RowSchema);\n this.isActive = true;\n this.isPrimaryList = false;\n this.ConsistentSaveToThisView = true;\n this.AddFireOnTypes(typeof(Invoice));\n this.Mapper = (api, docid, doc) => { api.Emit(docid, doc.TAG); };\n }\n }\n\n [RegisterView]\n public class InvoiceItemView : RaptorDB.View\n {\n public class RowSchema : RDBSchema\n {\n public string SKU { get; set; }\n public Decimal UnitPrice { get; set; }\n public Decimal qty { get; set; }\n }\n\n public InvoiceItemView()\n {\n this.Name = \"InvoiceItem View\";\n this.Schema = typeof(InvoiceItemView.RowSchema);\n this.isActive = true;\n this.isPrimaryList = false;\n this.AddFireOnTypes(typeof(Invoice));\n this.Mapper = (api, docid, doc) =>\n {\n foreach (InvoiceItem item in doc.items)\n {\n api.Emit(docid, item.SKU, item.UnitPrice, item.qty);\n }\n };\n }\n }\n }\n */\n}" }, { "path": "testing/program.cs", "content": "using System;\nusing System.IO;\nusing RaptorDB;\n\nnamespace testing\n{\n public class program\n {\n static RaptorDBServer server;\n public static void Main(string[] args)\n {\n AppDomain.CurrentDomain.UnhandledException += new UnhandledExceptionEventHandler(CurrentDomain_UnhandledException);\n server = new RaptorDBServer(90, @\"..\\..\\RaptorDBdata\");\n //fastBinaryJSON.BJSON.Parameters.v1_4TypedArray = true;\n \n Console.WriteLine(\"Server started on port 90\");\n Console.WriteLine(\"Press Enter to exit...\");\n Console.CancelKeyPress += new ConsoleCancelEventHandler(Console_CancelKeyPress);\n Console.ReadLine();\n server.Shutdown();\n \n return;\n }\n\n static void Console_CancelKeyPress(object sender, ConsoleCancelEventArgs e)\n {\n Console.WriteLine(\"Shutting down...\");\n server.Shutdown(); \n }\n\n static void CurrentDomain_UnhandledException(object sender, UnhandledExceptionEventArgs e)\n {\n File.WriteAllText(\"error.txt\", \"\" + e.ExceptionObject);\n }\n }\n}\n" }, { "path": "testing/tests.csproj", "content": "\n\n \n Local\n 9.0.30729\n 2.0\n {C6DA7503-3BCF-4688-ADD7-1CB6EDCE5E90}\n Debug\n AnyCPU\n \n \n testing\n \n \n JScript\n Grid\n IE50\n false\n Exe\n testing\n OnBuildSuccess\n \n \n \n \n 0.0\n v4.0\n publish\\\n true\n Disk\n false\n Foreground\n 7\n Days\n false\n false\n true\n 0\n 1.0.0.%2a\n false\n false\n true\n \n \n \n \n ..\\Output\\server\\\n true\n 285212672\n false\n \n \n DEBUG;TRACE\n true\n 4096\n false\n false\n false\n false\n false\n 4\n Full\n prompt\n false\n \n \n bin\\Release\\\n false\n 285212672\n false\n \n \n TRACE\n \n \n false\n 4096\n false\n \n \n true\n false\n false\n false\n 4\n none\n prompt\n false\n \n \n \n {45F6BE30-989A-4749-B6A0-69099C8661F4}\n RaptorDB\n \n \n {A1347486-8D54-4E17-8A22-76EFE61BF37B}\n Views\n False\n \n \n System\n \n \n {32331D51-5BE0-41E2-AF1A-9B086C5AE809}\n RaptorDB.Common\n \n \n \n \n Code\n \n \n Code\n \n \n \n \n False\n .NET Framework Client Profile\n false\n \n \n False\n .NET Framework 2.0 %28x86%29\n false\n \n \n False\n .NET Framework 3.0 %28x86%29\n false\n \n \n False\n .NET Framework 3.5\n false\n \n \n False\n .NET Framework 3.5 SP1\n true\n \n \n False\n Windows Installer 3.1\n true\n \n \n \n \n \n \n \n \n \n \n Auto\n AnyCPU\n \n" }, { "path": "testing.view", "content": "// ref: views.dll\nusing System;\nusing System.Collections.Generic;\nusing RaptorDB;\n\nnamespace SampleViews\n{\n [RegisterView]\n public class testing : View\n {\n\t // row schema defined in the script file \n\t // and will be transferred to the client when needed \n\t\tpublic class RowSchema : RDBSchema\n\t\t{\n\t\t\tpublic string Product;\n\t\t\tpublic decimal QTY;\n\t\t\tpublic decimal Price;\n\t\t\tpublic decimal Discount;\n\t\t}\n\t\t\n public testing()\n {\n this.Name = \"testing\";\n this.Description = \"\";\n this.isPrimaryList = false;\n this.isActive = true;\n this.BackgroundIndexing = true;\n this.Version = 3;\n\n this.Schema = typeof(RowSchema);\n\n this.AddFireOnTypes(typeof(SalesInvoice));\n\n this.Mapper = (api, docid, doc) =>\n {\n\t\t\t\tforeach (var i in doc.Items)\n\t\t\t\t\tapi.EmitObject(docid, i);\n };\n }\n }\n}" }, { "path": "vbTestConsole/App.config", "content": "\n\n \n \n \n" }, { "path": "vbTestConsole/Module1.vb", "content": "Module Module1\n\n Sub Main()\n Dim rdb As RaptorDB.RaptorDB = RaptorDB.RaptorDB.Open(\"..\\..\\..\\RaptorDBdata\")\n rdb.RegisterView(New SampleViews.SalesInvoiceView())\n Dim r = rdb.Query(Of SampleViews.SalesInvoiceViewRowSchema)(Function(x) x.NoCase = \"Me 4\" And x.Serial < 10)\n Console.WriteLine(fastJSON.JSON.ToNiceJSON(r.Rows, New fastJSON.JSONParameters With {.UseExtensions = False}))\n End Sub\n\nEnd Module\n" }, { "path": "vbTestConsole/My Project/Application.Designer.vb", "content": "'------------------------------------------------------------------------------\n' \n' This code was generated by a tool.\n' Runtime Version:4.0.30319.34014\n'\n' Changes to this file may cause incorrect behavior and will be lost if\n' the code is regenerated.\n' \n'------------------------------------------------------------------------------\n\nOption Strict On\nOption Explicit On\n\n" }, { "path": "vbTestConsole/My Project/Application.myapp", "content": "\n\n false\n false\n 0\n true\n 0\n 2\n true\n\n" }, { "path": "vbTestConsole/My Project/AssemblyInfo.vb", "content": "Imports System\nImports System.Reflection\nImports System.Runtime.InteropServices\n\n' General Information about an assembly is controlled through the following \n' set of attributes. Change these attribute values to modify the information\n' associated with an assembly.\n\n' Review the values of the assembly attributes\n\n \n \n \n \n \n \n\n\n\n'The following GUID is for the ID of the typelib if this project is exposed to COM\n \n\n' Version information for an assembly consists of the following four values:\n'\n' Major Version\n' Minor Version \n' Build Number\n' Revision\n'\n' You can specify all the values or you can default the Build and Revision Numbers \n' by using the '*' as shown below:\n' \n\n \n \n" }, { "path": "vbTestConsole/My Project/Resources.Designer.vb", "content": "'------------------------------------------------------------------------------\n' \n' This code was generated by a tool.\n' Runtime Version:4.0.30319.34014\n'\n' Changes to this file may cause incorrect behavior and will be lost if\n' the code is regenerated.\n' \n'------------------------------------------------------------------------------\n\nOption Strict On\nOption Explicit On\n\n\nNamespace My.Resources\n \n 'This class was auto-generated by the StronglyTypedResourceBuilder\n 'class via a tool like ResGen or Visual Studio.\n 'To add or remove a member, edit your .ResX file then rerun ResGen\n 'with the /str option, or rebuild your VS project.\n '''\n ''' A strongly-typed resource class, for looking up localized strings, etc.\n '''\n _\n Friend Module Resources\n\n Private resourceMan As Global.System.Resources.ResourceManager\n\n Private resourceCulture As Global.System.Globalization.CultureInfo\n\n '''\n ''' Returns the cached ResourceManager instance used by this class.\n '''\n _\n Friend ReadOnly Property ResourceManager() As Global.System.Resources.ResourceManager\n Get\n If Object.ReferenceEquals(resourceMan, Nothing) Then\n Dim temp As Global.System.Resources.ResourceManager = New Global.System.Resources.ResourceManager(\"ConsoleApplication1.Resources\", GetType(Resources).Assembly)\n resourceMan = temp\n End If\n Return resourceMan\n End Get\n End Property\n\n '''\n ''' Overrides the current thread's CurrentUICulture property for all\n ''' resource lookups using this strongly typed resource class.\n '''\n _\n Friend Property Culture() As Global.System.Globalization.CultureInfo\n Get\n Return resourceCulture\n End Get\n Set(ByVal value As Global.System.Globalization.CultureInfo)\n resourceCulture = value\n End Set\n End Property\n End Module\nEnd Namespace\n" }, { "path": "vbTestConsole/My Project/Resources.resx", "content": "\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n text/microsoft-resx\n \n \n 2.0\n \n \n System.Resources.ResXResourceReader, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089\n \n \n System.Resources.ResXResourceWriter, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089\n \n" }, { "path": "vbTestConsole/My Project/Settings.Designer.vb", "content": "'------------------------------------------------------------------------------\n' \n' This code was generated by a tool.\n' Runtime Version:4.0.30319.34014\n'\n' Changes to this file may cause incorrect behavior and will be lost if\n' the code is regenerated.\n' \n'------------------------------------------------------------------------------\n\nOption Strict On\nOption Explicit On\n\n\nNamespace My\n\n _\n Partial Friend NotInheritable Class MySettings\n Inherits Global.System.Configuration.ApplicationSettingsBase\n\n Private Shared defaultInstance As MySettings = CType(Global.System.Configuration.ApplicationSettingsBase.Synchronized(New MySettings), MySettings)\n\n#Region \"My.Settings Auto-Save Functionality\"\n#If _MyType = \"WindowsForms\" Then\n Private Shared addedHandler As Boolean\n\n Private Shared addedHandlerLockObject As New Object\n\n _\n Private Shared Sub AutoSaveSettings(ByVal sender As Global.System.Object, ByVal e As Global.System.EventArgs)\n If My.Application.SaveMySettingsOnExit Then\n My.Settings.Save()\n End If\n End Sub\n#End If\n#End Region\n\n Public Shared ReadOnly Property [Default]() As MySettings\n Get\n\n#If _MyType = \"WindowsForms\" Then\n If Not addedHandler Then\n SyncLock addedHandlerLockObject\n If Not addedHandler Then\n AddHandler My.Application.Shutdown, AddressOf AutoSaveSettings\n addedHandler = True\n End If\n End SyncLock\n End If\n#End If\n Return defaultInstance\n End Get\n End Property\n End Class\nEnd Namespace\n\nNamespace My\n \n _\n Friend Module MySettingsProperty\n \n _\n Friend ReadOnly Property Settings() As Global.ConsoleApplication1.My.MySettings\n Get\n Return Global.ConsoleApplication1.My.MySettings.Default\n End Get\n End Property\n End Module\nEnd Namespace\n" }, { "path": "vbTestConsole/My Project/Settings.settings", "content": "\n\n \n \n \n \n\n" }, { "path": "vbTestConsole/vbtestconsole.vbproj", "content": "\n\n \n \n Debug\n AnyCPU\n {D3A7445B-4D0D-4989-9EEF-334821D70894}\n Exe\n ConsoleApplication1.Module1\n ConsoleApplication1\n ConsoleApplication1\n 512\n Console\n v4.5\n \n \n AnyCPU\n true\n full\n true\n true\n bin\\Debug\\\n ConsoleApplication1.xml\n 42016,41999,42017,42018,42019,42032,42036,42020,42021,42022\n \n \n AnyCPU\n pdbonly\n false\n true\n true\n bin\\Release\\\n ConsoleApplication1.xml\n 42016,41999,42017,42018,42019,42032,42036,42020,42021,42022\n \n \n On\n \n \n Binary\n \n \n Off\n \n \n On\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n True\n Application.myapp\n \n \n True\n True\n Resources.resx\n \n \n True\n Settings.settings\n True\n \n \n \n \n VbMyResourcesResXFileCodeGenerator\n Resources.Designer.vb\n My.Resources\n Designer\n \n \n \n \n MyApplicationCodeGenerator\n Application.Designer.vb\n \n \n SettingsSingleFileGenerator\n My\n Settings.Designer.vb\n \n \n \n \n \n {32331d51-5be0-41e2-af1a-9b086c5ae809}\n RaptorDB.Common\n \n \n {45f6be30-989a-4749-b6a0-69099c8661f4}\n RaptorDB\n \n \n {a1347486-8d54-4e17-8a22-76efe61bf37b}\n Views\n \n \n \n \n" } ]